From 2c4170e24d0127640d51d90d638481c2b82aed6c Mon Sep 17 00:00:00 2001 From: xxchan Date: Wed, 21 Apr 2021 16:09:55 +0800 Subject: [PATCH 1/3] support customer-supplied encryption keys --- generated.go | 51 +++++++++++++++++++++++++++++++++++++-------------- service.toml | 13 ++++++++++--- storage.go | 6 ++++++ utils.go | 3 +++ 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/generated.go b/generated.go index 49f6036..d267907 100644 --- a/generated.go +++ b/generated.go @@ -29,12 +29,16 @@ const ( pairDefaultStoragePairs = "gcs_default_storage_pairs" // ProjectID pairProjectID = "gcs_project_id" + // SseCustomerKey is the customer's 32-byte AES-256 key + pairSseCustomerKey = "gcs_sse_customer_key" // StorageClass pairStorageClass = "gcs_storage_class" ) // Service available metadata. const ( + MetadataSseCustomerKeySha256 = "gcs-sse_customer_key_sha256" + MetadataStorageClass = "gcs-storage-class" ) @@ -65,6 +69,15 @@ func WithProjectID(v string) Pair { } } +// WithSseCustomerKey will apply sse_customer_key value to Options +// SseCustomerKey is the customer's 32-byte AES-256 key +func WithSseCustomerKey(v string) Pair { + return Pair{ + Key: pairSseCustomerKey, + Value: v, + } +} + // WithStorageClass will apply storage_class value to Options // StorageClass func WithStorageClass(v string) Pair { @@ -578,12 +591,14 @@ type pairStorageRead struct { // Required pairs // Optional pairs - HasIoCallback bool - IoCallback func([]byte) - HasOffset bool - Offset int64 - HasSize bool - Size int64 + HasIoCallback bool + IoCallback func([]byte) + HasOffset bool + Offset int64 + HasSize bool + Size int64 + HasSseCustomerKey bool + SseCustomerKey string // Generated pairs } @@ -606,6 +621,9 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { case "size": result.HasSize = true result.Size = v.Value.(int64) + case pairSseCustomerKey: + result.HasSseCustomerKey = true + result.SseCustomerKey = v.Value.(string) // Generated pairs default: @@ -657,14 +675,16 @@ type pairStorageWrite struct { // Required pairs // Optional pairs - HasContentMd5 bool - ContentMd5 string - HasContentType bool - ContentType string - HasIoCallback bool - IoCallback func([]byte) - HasStorageClass bool - StorageClass string + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasIoCallback bool + IoCallback func([]byte) + HasSseCustomerKey bool + SseCustomerKey string + HasStorageClass bool + StorageClass string // Generated pairs } @@ -687,6 +707,9 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { case "io_callback": result.HasIoCallback = true result.IoCallback = v.Value.(func([]byte)) + case pairSseCustomerKey: + result.HasSseCustomerKey = true + result.SseCustomerKey = v.Value.(string) case pairStorageClass: result.HasStorageClass = true result.StorageClass = v.Value.(string) diff --git a/service.toml b/service.toml index bcecc15..347488f 100644 --- a/service.toml +++ b/service.toml @@ -12,10 +12,14 @@ optional = ["default_storage_pairs", "pair_policy", "work_dir"] optional = ["list_mode"] [namespace.storage.op.read] -optional = ["offset", "io_callback", "size"] +optional = ["offset", "io_callback", "size", "sse_customer_key"] [namespace.storage.op.write] -optional = ["content_md5", "content_type", "io_callback", "storage_class"] +optional = ["content_md5", "content_type", "io_callback", "storage_class", "sse_customer_key"] + +[pairs.sse_customer_key] +type = "string" +description = "is the customer's 32-byte AES-256 key" [pairs.default_service_pairs] type = "DefaultServicePairs" @@ -32,4 +36,7 @@ type = "string" type = "string" [infos.object.meta.storage-class] -type = "string" \ No newline at end of file +type = "string" + +[infos.object.meta.sse_customer_key_sha256] +type = "string" \ No newline at end of file diff --git a/storage.go b/storage.go index 5f72b1d..7fcf3f7 100644 --- a/storage.go +++ b/storage.go @@ -138,6 +138,9 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt var rc io.ReadCloser object := s.bucket.Object(rp) + if opt.HasSseCustomerKey { + object = object.Key([]byte(opt.SseCustomerKey)) + } rc, err = object.NewReader(ctx) if err != nil { return 0, err @@ -171,6 +174,9 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 rp := s.getAbsPath(path) object := s.bucket.Object(rp) + if opt.HasSseCustomerKey { + object = object.Key([]byte(opt.SseCustomerKey)) + } w := object.NewWriter(ctx) defer func() { cerr := w.Close() diff --git a/utils.go b/utils.go index 057d20f..9f4567e 100644 --- a/utils.go +++ b/utils.go @@ -277,6 +277,9 @@ func (s *Storage) formatFileObject(v *gs.ObjectAttrs) (o *typ.Object, err error) if value := v.StorageClass; value != "" { sm[MetadataStorageClass] = value } + if value := v.CustomerKeySHA256; value != "" { + sm[MetadataSseCustomerKeySha256] = value + } o.SetServiceMetadata(sm) return From 27c04c3ce28ca7062800f7cb8cb498fda946fc49 Mon Sep 17 00:00:00 2001 From: xxchan Date: Wed, 21 Apr 2021 17:46:19 +0800 Subject: [PATCH 2/3] use []byte for customer key --- generated.go | 10 +++++----- go.mod | 2 +- go.sum | 10 ++++++---- service.toml | 2 +- storage.go | 4 ++-- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/generated.go b/generated.go index d267907..59f0abd 100644 --- a/generated.go +++ b/generated.go @@ -71,7 +71,7 @@ func WithProjectID(v string) Pair { // WithSseCustomerKey will apply sse_customer_key value to Options // SseCustomerKey is the customer's 32-byte AES-256 key -func WithSseCustomerKey(v string) Pair { +func WithSseCustomerKey(v []byte) Pair { return Pair{ Key: pairSseCustomerKey, Value: v, @@ -598,7 +598,7 @@ type pairStorageRead struct { HasSize bool Size int64 HasSseCustomerKey bool - SseCustomerKey string + SseCustomerKey []byte // Generated pairs } @@ -623,7 +623,7 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { result.Size = v.Value.(int64) case pairSseCustomerKey: result.HasSseCustomerKey = true - result.SseCustomerKey = v.Value.(string) + result.SseCustomerKey = v.Value.([]byte) // Generated pairs default: @@ -682,7 +682,7 @@ type pairStorageWrite struct { HasIoCallback bool IoCallback func([]byte) HasSseCustomerKey bool - SseCustomerKey string + SseCustomerKey []byte HasStorageClass bool StorageClass string // Generated pairs @@ -709,7 +709,7 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.IoCallback = v.Value.(func([]byte)) case pairSseCustomerKey: result.HasSseCustomerKey = true - result.SseCustomerKey = v.Value.(string) + result.SseCustomerKey = v.Value.([]byte) case pairStorageClass: result.HasStorageClass = true result.StorageClass = v.Value.(string) diff --git a/go.mod b/go.mod index 2083254..c2a1c7d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( cloud.google.com/go/storage v1.14.0 github.com/aos-dev/go-integration-test/v3 v3.0.0-20210205075620-0b779f4b3afc - github.com/aos-dev/go-storage/v3 v3.4.3-0.20210417162535-67db0dd18784 + github.com/aos-dev/go-storage/v3 v3.5.1-0.20210421092104-4de248aed1f1 github.com/google/uuid v1.2.0 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 google.golang.org/api v0.44.0 diff --git a/go.sum b/go.sum index 4ff9341..cfc3867 100644 --- a/go.sum +++ b/go.sum @@ -48,11 +48,11 @@ github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1/go.mod h1:x0q github.com/aos-dev/go-integration-test/v3 v3.0.0-20210205075620-0b779f4b3afc h1:s8BQqJwWe2yoTGJFOsIxnkwwMsnDqMgkO39yzEP4smw= github.com/aos-dev/go-integration-test/v3 v3.0.0-20210205075620-0b779f4b3afc/go.mod h1:2VqxZhJVvI1jKTTERWHfCBMZQaJt10xoEJ+CxbfYZlo= github.com/aos-dev/go-storage/v3 v3.0.1-0.20210205074802-e8a5b22166c2/go.mod h1:JznhvyhPDnETfZ3RyWm3mT/S5ic+uuYFWagSIkUZujc= -github.com/aos-dev/go-storage/v3 v3.4.3-0.20210417162535-67db0dd18784 h1:eeKDDn+8RPtYuk7lrQC48OtTNeSRF/l9P5YHa/Q3omk= -github.com/aos-dev/go-storage/v3 v3.4.3-0.20210417162535-67db0dd18784/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= +github.com/aos-dev/go-storage/v3 v3.5.1-0.20210421092104-4de248aed1f1 h1:Kkb/cVJRwc77lXGcnV2ThUC4QUqybiEVrFwq8qnM4V4= +github.com/aos-dev/go-storage/v3 v3.5.1-0.20210421092104-4de248aed1f1/go.mod h1:JFshvl851ZDDXtFGWDFKqkg34QEPH0xuhLJ2LjikZYc= github.com/aos-dev/specs/go v0.0.0-20210205073047-af8ef94af73d/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= -github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b h1:qIehSnBbr31ATAckM9u9h6gSz+9PkGqu79vTngx6wPw= -github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= +github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3 h1:LiW0Ki0Gw6opu11JwMhxWw5M+V6I9JypJ5eAIp+Rqt4= +github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -170,6 +170,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= +github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/service.toml b/service.toml index 347488f..fc6ba20 100644 --- a/service.toml +++ b/service.toml @@ -18,7 +18,7 @@ optional = ["offset", "io_callback", "size", "sse_customer_key"] optional = ["content_md5", "content_type", "io_callback", "storage_class", "sse_customer_key"] [pairs.sse_customer_key] -type = "string" +type = "byte_array" description = "is the customer's 32-byte AES-256 key" [pairs.default_service_pairs] diff --git a/storage.go b/storage.go index 7fcf3f7..621611a 100644 --- a/storage.go +++ b/storage.go @@ -139,7 +139,7 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt object := s.bucket.Object(rp) if opt.HasSseCustomerKey { - object = object.Key([]byte(opt.SseCustomerKey)) + object = object.Key(opt.SseCustomerKey) } rc, err = object.NewReader(ctx) if err != nil { @@ -175,7 +175,7 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 object := s.bucket.Object(rp) if opt.HasSseCustomerKey { - object = object.Key([]byte(opt.SseCustomerKey)) + object = object.Key(opt.SseCustomerKey) } w := object.NewWriter(ctx) defer func() { From 3b124f2be480bee8f40cbdee66d0232397c01fd2 Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 22 Apr 2021 15:18:43 +0800 Subject: [PATCH 3/3] rename encryption pairs --- generated.go | 70 ++++++++++++++++++++++++++-------------------------- service.toml | 10 ++++---- storage.go | 8 +++--- utils.go | 2 +- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/generated.go b/generated.go index 59f0abd..64e3812 100644 --- a/generated.go +++ b/generated.go @@ -27,17 +27,17 @@ const ( pairDefaultServicePairs = "gcs_default_service_pairs" // DefaultStoragePairs set default pairs for storager actions pairDefaultStoragePairs = "gcs_default_storage_pairs" + // EncryptionKey is the customer's 32-byte AES-256 key + pairEncryptionKey = "gcs_encryption_key" // ProjectID pairProjectID = "gcs_project_id" - // SseCustomerKey is the customer's 32-byte AES-256 key - pairSseCustomerKey = "gcs_sse_customer_key" // StorageClass pairStorageClass = "gcs_storage_class" ) // Service available metadata. const ( - MetadataSseCustomerKeySha256 = "gcs-sse_customer_key_sha256" + MetadataEncryptionKeySha256 = "gcs-encryption_key_sha256" MetadataStorageClass = "gcs-storage-class" ) @@ -60,20 +60,20 @@ func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { } } -// WithProjectID will apply project_id value to Options -// ProjectID -func WithProjectID(v string) Pair { +// WithEncryptionKey will apply encryption_key value to Options +// EncryptionKey is the customer's 32-byte AES-256 key +func WithEncryptionKey(v []byte) Pair { return Pair{ - Key: pairProjectID, + Key: pairEncryptionKey, Value: v, } } -// WithSseCustomerKey will apply sse_customer_key value to Options -// SseCustomerKey is the customer's 32-byte AES-256 key -func WithSseCustomerKey(v []byte) Pair { +// WithProjectID will apply project_id value to Options +// ProjectID +func WithProjectID(v string) Pair { return Pair{ - Key: pairSseCustomerKey, + Key: pairProjectID, Value: v, } } @@ -591,14 +591,14 @@ type pairStorageRead struct { // Required pairs // Optional pairs - HasIoCallback bool - IoCallback func([]byte) - HasOffset bool - Offset int64 - HasSize bool - Size int64 - HasSseCustomerKey bool - SseCustomerKey []byte + HasEncryptionKey bool + EncryptionKey []byte + HasIoCallback bool + IoCallback func([]byte) + HasOffset bool + Offset int64 + HasSize bool + Size int64 // Generated pairs } @@ -612,6 +612,9 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { switch v.Key { // Required pairs // Optional pairs + case pairEncryptionKey: + result.HasEncryptionKey = true + result.EncryptionKey = v.Value.([]byte) case "io_callback": result.HasIoCallback = true result.IoCallback = v.Value.(func([]byte)) @@ -621,9 +624,6 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { case "size": result.HasSize = true result.Size = v.Value.(int64) - case pairSseCustomerKey: - result.HasSseCustomerKey = true - result.SseCustomerKey = v.Value.([]byte) // Generated pairs default: @@ -675,16 +675,16 @@ type pairStorageWrite struct { // Required pairs // Optional pairs - HasContentMd5 bool - ContentMd5 string - HasContentType bool - ContentType string - HasIoCallback bool - IoCallback func([]byte) - HasSseCustomerKey bool - SseCustomerKey []byte - HasStorageClass bool - StorageClass string + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasEncryptionKey bool + EncryptionKey []byte + HasIoCallback bool + IoCallback func([]byte) + HasStorageClass bool + StorageClass string // Generated pairs } @@ -704,12 +704,12 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { case "content_type": result.HasContentType = true result.ContentType = v.Value.(string) + case pairEncryptionKey: + result.HasEncryptionKey = true + result.EncryptionKey = v.Value.([]byte) case "io_callback": result.HasIoCallback = true result.IoCallback = v.Value.(func([]byte)) - case pairSseCustomerKey: - result.HasSseCustomerKey = true - result.SseCustomerKey = v.Value.([]byte) case pairStorageClass: result.HasStorageClass = true result.StorageClass = v.Value.(string) diff --git a/service.toml b/service.toml index fc6ba20..fefe8c6 100644 --- a/service.toml +++ b/service.toml @@ -12,12 +12,12 @@ optional = ["default_storage_pairs", "pair_policy", "work_dir"] optional = ["list_mode"] [namespace.storage.op.read] -optional = ["offset", "io_callback", "size", "sse_customer_key"] +optional = ["offset", "io_callback", "size", "encryption_key"] [namespace.storage.op.write] -optional = ["content_md5", "content_type", "io_callback", "storage_class", "sse_customer_key"] +optional = ["content_md5", "content_type", "io_callback", "storage_class", "encryption_key"] -[pairs.sse_customer_key] +[pairs.encryption_key] type = "byte_array" description = "is the customer's 32-byte AES-256 key" @@ -38,5 +38,5 @@ type = "string" [infos.object.meta.storage-class] type = "string" -[infos.object.meta.sse_customer_key_sha256] -type = "string" \ No newline at end of file +[infos.object.meta.encryption_key_sha256] +type = "string" diff --git a/storage.go b/storage.go index 621611a..d68f5e3 100644 --- a/storage.go +++ b/storage.go @@ -138,8 +138,8 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt var rc io.ReadCloser object := s.bucket.Object(rp) - if opt.HasSseCustomerKey { - object = object.Key(opt.SseCustomerKey) + if opt.HasEncryptionKey { + object = object.Key(opt.EncryptionKey) } rc, err = object.NewReader(ctx) if err != nil { @@ -174,8 +174,8 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 rp := s.getAbsPath(path) object := s.bucket.Object(rp) - if opt.HasSseCustomerKey { - object = object.Key(opt.SseCustomerKey) + if opt.HasEncryptionKey { + object = object.Key(opt.EncryptionKey) } w := object.NewWriter(ctx) defer func() { diff --git a/utils.go b/utils.go index 9f4567e..8782573 100644 --- a/utils.go +++ b/utils.go @@ -278,7 +278,7 @@ func (s *Storage) formatFileObject(v *gs.ObjectAttrs) (o *typ.Object, err error) sm[MetadataStorageClass] = value } if value := v.CustomerKeySHA256; value != "" { - sm[MetadataSseCustomerKeySha256] = value + sm[MetadataEncryptionKeySha256] = value } o.SetServiceMetadata(sm)