From b7515c26863ec297e460e8ec67f7b224b2906034 Mon Sep 17 00:00:00 2001
From: JinnyYi <jinnyyi@yunify.com>
Date: Mon, 28 Jun 2021 15:50:27 +0800
Subject: [PATCH 1/4] Implement GSP-93 and GSP-109
---
generated.go | 620 ++++++++++++++++++++----------------------
go.mod | 4 +-
go.sum | 26 +-
service.toml | 12 +-
storage.go | 71 ++++-
tests/storage_test.go | 7 +
tests/utils_test.go | 3 +
utils.go | 1 +
8 files changed, 403 insertions(+), 341 deletions(-)
diff --git a/generated.go b/generated.go
index ee035fa..4fda2cd 100644
--- a/generated.go
+++ b/generated.go
@@ -21,35 +21,9 @@ var _ httpclient.Options
// Type is the type for oss
const Type = "oss"
-// Service available pairs.
-const (
- // DefaultServicePairs set default pairs for service actions
- pairDefaultServicePairs = "oss_default_service_pairs"
- // DefaultStoragePairs set default pairs for storager actions
- pairDefaultStoragePairs = "oss_default_storage_pairs"
- // ServerSideDataEncryption specifies the encryption algorithm when server_side_encryption is KMS. Can only be set to SM4. If this is not set, AES256 will be used.
- //
- // For Chinese users, refer to https://help.aliyun.com/document_detail/31871.html for details.
- //
- // For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used.
- pairServerSideDataEncryption = "oss_server_side_data_encryption"
- // ServerSideEncryption specifies the encryption algorithm. Can be AES256, KMS or SM4.
- //
- // For Chinese users, refer to https://help.aliyun.com/document_detail/31871.html for details.
- //
- // For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used.
- pairServerSideEncryption = "oss_server_side_encryption"
- // ServerSideEncryptionKeyID is the KMS-managed user master key. Only valid when server_side_encryption is KMS.
- pairServerSideEncryptionKeyID = "oss_server_side_encryption_key_id"
- // ServiceFeatures set service features
- pairServiceFeatures = "oss_service_features"
- // StorageClass
- pairStorageClass = "oss_storage_class"
- // StorageFeatures set storage features
- pairStorageFeatures = "oss_storage_features"
-)
-
// ObjectMetadata stores service metadata for object.
+//
+// Deprecated: Use ObjectSystemMetadata instead.
type ObjectMetadata struct {
// ServerSideEncryption
ServerSideEncryption string
@@ -63,6 +37,8 @@ type ObjectMetadata struct {
//
// - This function should not be called by service implementer.
// - The returning ObjectMetadata is read only and should not be modified.
+//
+// Deprecated: Use GetObjectSystemMetadata instead.
func GetObjectMetadata(o *Object) ObjectMetadata {
om, ok := o.GetServiceMetadata()
if ok {
@@ -74,16 +50,69 @@ func GetObjectMetadata(o *Object) ObjectMetadata {
// setObjectMetadata will set ObjectMetadata into Object.
//
// - This function should only be called once, please make sure all data has been written before set.
+//
+// Deprecated: Use setObjectSystemMetadata instead.
func setObjectMetadata(o *Object, om ObjectMetadata) {
o.SetServiceMetadata(om)
}
+// ObjectSystemMetadata stores system metadata for object.
+type ObjectSystemMetadata struct {
+ // ServerSideEncryption
+ ServerSideEncryption string
+ // ServerSideEncryptionKeyID
+ ServerSideEncryptionKeyID string
+ // StorageClass
+ StorageClass string
+}
+
+// GetObjectSystemMetadata will get ObjectSystemMetadata from Object.
+//
+// - This function should not be called by service implementer.
+// - The returning ObjectServiceMetadata is read only and should not be modified.
+func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata {
+ sm, ok := o.GetSystemMetadata()
+ if ok {
+ return sm.(ObjectSystemMetadata)
+ }
+ return ObjectSystemMetadata{}
+}
+
+// setObjectSystemMetadata will set ObjectSystemMetadata into Object.
+//
+// - This function should only be called once, please make sure all data has been written before set.
+func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) {
+ o.SetSystemMetadata(sm)
+}
+
+// StorageSystemMetadata stores system metadata for storage meta.
+type StorageSystemMetadata struct {
+}
+
+// GetStorageSystemMetadata will get SystemMetadata from StorageMeta.
+//
+// - The returning StorageSystemMetadata is read only and should not be modified.
+func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata {
+ sm, ok := s.GetSystemMetadata()
+ if ok {
+ return sm.(StorageSystemMetadata)
+ }
+ return StorageSystemMetadata{}
+}
+
+// setStorageSystemMetadata will set SystemMetadata into StorageMeta.
+//
+// - This function should only be called once, please make sure all data has been written before set.
+func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) {
+ s.SetSystemMetadata(sm)
+}
+
// WithDefaultServicePairs will apply default_service_pairs value to Options.
//
// DefaultServicePairs set default pairs for service actions
func WithDefaultServicePairs(v DefaultServicePairs) Pair {
return Pair{
- Key: pairDefaultServicePairs,
+ Key: "default_service_pairs",
Value: v,
}
}
@@ -93,7 +122,7 @@ func WithDefaultServicePairs(v DefaultServicePairs) Pair {
// DefaultStoragePairs set default pairs for storager actions
func WithDefaultStoragePairs(v DefaultStoragePairs) Pair {
return Pair{
- Key: pairDefaultStoragePairs,
+ Key: "default_storage_pairs",
Value: v,
}
}
@@ -107,7 +136,7 @@ func WithDefaultStoragePairs(v DefaultStoragePairs) Pair {
// For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used.
func WithServerSideDataEncryption(v string) Pair {
return Pair{
- Key: pairServerSideDataEncryption,
+ Key: "server_side_data_encryption",
Value: v,
}
}
@@ -121,7 +150,7 @@ func WithServerSideDataEncryption(v string) Pair {
// For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used.
func WithServerSideEncryption(v string) Pair {
return Pair{
- Key: pairServerSideEncryption,
+ Key: "server_side_encryption",
Value: v,
}
}
@@ -131,7 +160,7 @@ func WithServerSideEncryption(v string) Pair {
// ServerSideEncryptionKeyID is the KMS-managed user master key. Only valid when server_side_encryption is KMS.
func WithServerSideEncryptionKeyID(v string) Pair {
return Pair{
- Key: pairServerSideEncryptionKeyID,
+ Key: "server_side_encryption_key_id",
Value: v,
}
}
@@ -141,7 +170,7 @@ func WithServerSideEncryptionKeyID(v string) Pair {
// ServiceFeatures set service features
func WithServiceFeatures(v ServiceFeatures) Pair {
return Pair{
- Key: pairServiceFeatures,
+ Key: "service_features",
Value: v,
}
}
@@ -151,7 +180,7 @@ func WithServiceFeatures(v ServiceFeatures) Pair {
// StorageClass
func WithStorageClass(v string) Pair {
return Pair{
- Key: pairStorageClass,
+ Key: "storage_class",
Value: v,
}
}
@@ -161,24 +190,59 @@ func WithStorageClass(v string) Pair {
// StorageFeatures set storage features
func WithStorageFeatures(v StorageFeatures) Pair {
return Pair{
- Key: pairStorageFeatures,
+ Key: "storage_features",
Value: v,
}
}
+var pairMap = map[string]string{
+ "content_md5": "string",
+ "content_type": "string",
+ "context": "context.Context",
+ "continuation_token": "string",
+ "credential": "string",
+ "default_service_pairs": "DefaultServicePairs",
+ "default_storage_pairs": "DefaultStoragePairs",
+ "endpoint": "string",
+ "expire": "int",
+ "http_client_options": "*httpclient.Options",
+ "interceptor": "Interceptor",
+ "io_callback": "func([]byte)",
+ "list_mode": "ListMode",
+ "location": "string",
+ "multipart_id": "string",
+ "name": "string",
+ "object_mode": "ObjectMode",
+ "offset": "int64",
+ "server_side_data_encryption": "string",
+ "server_side_encryption": "string",
+ "server_side_encryption_key_id": "string",
+ "service_features": "ServiceFeatures",
+ "size": "int64",
+ "storage_class": "string",
+ "storage_features": "StorageFeatures",
+ "work_dir": "string",
+}
var (
_ Servicer = &Service{}
)
type ServiceFeatures struct {
- LooseOperationAll bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationAll bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
LooseOperationCreate bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
LooseOperationDelete bool
- LooseOperationGet bool
- LooseOperationList bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationGet bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationList bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
VirtualOperationAll bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
VirtualPairAll bool
}
@@ -216,7 +280,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) {
result.HasCredential = true
result.Credential = v.Value.(string)
// Optional pairs
- case pairDefaultServicePairs:
+ case "default_service_pairs":
if result.HasDefaultServicePairs {
continue
}
@@ -234,7 +298,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) {
}
result.HasHTTPClientOptions = true
result.HTTPClientOptions = v.Value.(*httpclient.Options)
- case pairServiceFeatures:
+ case "service_features":
if result.HasServiceFeatures {
continue
}
@@ -269,23 +333,10 @@ func (s *Service) parsePairServiceCreate(opts []Pair) (pairServiceCreate, error)
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
+ return pairServiceCreate{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationCreate {
- continue
- }
- return pairServiceCreate{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -305,23 +356,10 @@ func (s *Service) parsePairServiceDelete(opts []Pair) (pairServiceDelete, error)
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
+ return pairServiceDelete{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationDelete {
- continue
- }
- return pairServiceDelete{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -341,23 +379,10 @@ func (s *Service) parsePairServiceGet(opts []Pair) (pairServiceGet, error) {
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
+ return pairServiceGet{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationGet {
- continue
- }
- return pairServiceGet{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -377,23 +402,10 @@ func (s *Service) parsePairServiceList(opts []Pair) (pairServiceList, error) {
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
+ return pairServiceList{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationList {
- continue
- }
- return pairServiceList{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -504,30 +516,57 @@ func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *Stor
var (
_ Appender = &Storage{}
+ _ Direr = &Storage{}
_ Multiparter = &Storage{}
_ Storager = &Storage{}
)
type StorageFeatures struct {
- LooseOperationAll bool
- LooseOperationCommitAppend bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationAll bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationCommitAppend bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
LooseOperationCompleteMultipart bool
- LooseOperationCreate bool
- LooseOperationCreateAppend bool
- LooseOperationCreateMultipart bool
- LooseOperationDelete bool
- LooseOperationList bool
- LooseOperationListMultipart bool
- LooseOperationMetadata bool
- LooseOperationRead bool
- LooseOperationStat bool
- LooseOperationWrite bool
- LooseOperationWriteAppend bool
- LooseOperationWriteMultipart bool
-
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationCreate bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationCreateAppend bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationCreateDir bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationCreateMultipart bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationDelete bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationList bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationListMultipart bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationMetadata bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationRead bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationStat bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationWrite bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationWriteAppend bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
+ LooseOperationWriteMultipart bool
+
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
VirtualOperationAll bool
+ // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0.
VirtualPairAll bool
+ // VirtualDir virtual_dir feature is designed for a service that doesn't have native dir support but wants to provide simulated operations.
+ //
+ // - If this feature is disabled (the default behavior), the service will behave like it doesn't have any dir support.
+ // - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, list, delete, and so on.
+ //
+ // This feature was introduced in GSP-109.
+ VirtualDir bool
}
// pairStorageNew is the parsed struct
@@ -562,13 +601,13 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) {
result.HasName = true
result.Name = v.Value.(string)
// Optional pairs
- case pairDefaultStoragePairs:
+ case "default_storage_pairs":
if result.HasDefaultStoragePairs {
continue
}
result.HasDefaultStoragePairs = true
result.DefaultStoragePairs = v.Value.(DefaultStoragePairs)
- case pairStorageFeatures:
+ case "storage_features":
if result.HasStorageFeatures {
continue
}
@@ -595,6 +634,7 @@ type DefaultStoragePairs struct {
CompleteMultipart []Pair
Create []Pair
CreateAppend []Pair
+ CreateDir []Pair
CreateMultipart []Pair
Delete []Pair
List []Pair
@@ -619,23 +659,10 @@ func (s *Storage) parsePairStorageCommitAppend(opts []Pair) (pairStorageCommitAp
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
- }
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationCommitAppend {
- continue
+ return pairStorageCommitAppend{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageCommitAppend{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -655,23 +682,10 @@ func (s *Storage) parsePairStorageCompleteMultipart(opts []Pair) (pairStorageCom
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
+ return pairStorageCompleteMultipart{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationCompleteMultipart {
- continue
- }
- return pairStorageCompleteMultipart{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -684,6 +698,8 @@ type pairStorageCreate struct {
pairs []Pair
HasMultipartID bool
MultipartID string
+ HasObjectMode bool
+ ObjectMode ObjectMode
}
// parsePairStorageCreate will parse Pair slice into *pairStorageCreate
@@ -693,9 +709,6 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error)
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "multipart_id":
if result.HasMultipartID {
@@ -704,19 +717,16 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error)
result.HasMultipartID = true
result.MultipartID = v.Value.(string)
continue
- default:
- isUnsupportedPair = true
- }
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationCreate {
+ case "object_mode":
+ if result.HasObjectMode {
+ continue
+ }
+ result.HasObjectMode = true
+ result.ObjectMode = v.Value.(ObjectMode)
continue
+ default:
+ return pairStorageCreate{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageCreate{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -746,9 +756,6 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "content_type":
if result.HasContentType {
@@ -757,28 +764,28 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp
result.HasContentType = true
result.ContentType = v.Value.(string)
continue
- case pairServerSideDataEncryption:
+ case "server_side_data_encryption":
if result.HasServerSideDataEncryption {
continue
}
result.HasServerSideDataEncryption = true
result.ServerSideDataEncryption = v.Value.(string)
continue
- case pairServerSideEncryption:
+ case "server_side_encryption":
if result.HasServerSideEncryption {
continue
}
result.HasServerSideEncryption = true
result.ServerSideEncryption = v.Value.(string)
continue
- case pairServerSideEncryptionKeyID:
+ case "server_side_encryption_key_id":
if result.HasServerSideEncryptionKeyID {
continue
}
result.HasServerSideEncryptionKeyID = true
result.ServerSideEncryptionKeyID = v.Value.(string)
continue
- case pairStorageClass:
+ case "storage_class":
if result.HasStorageClass {
continue
}
@@ -786,18 +793,40 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp
result.StorageClass = v.Value.(string)
continue
default:
- isUnsupportedPair = true
+ return pairStorageCreateAppend{}, services.PairUnsupportedError{Pair: v}
}
+ }
- if !isUnsupportedPair {
- continue
- }
+ // Check required pairs.
+
+ return result, nil
+}
+
+// pairStorageCreateDir is the parsed struct
+type pairStorageCreateDir struct {
+ pairs []Pair
+ HasStorageClass bool
+ StorageClass string
+}
+
+// parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir
+func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) {
+ result := pairStorageCreateDir{
+ pairs: opts,
+ }
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationCreateAppend {
+ for _, v := range opts {
+ switch v.Key {
+ case "storage_class":
+ if result.HasStorageClass {
+ continue
+ }
+ result.HasStorageClass = true
+ result.StorageClass = v.Value.(string)
continue
+ default:
+ return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageCreateAppend{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -827,9 +856,6 @@ func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreat
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "content_type":
if result.HasContentType {
@@ -838,28 +864,28 @@ func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreat
result.HasContentType = true
result.ContentType = v.Value.(string)
continue
- case pairServerSideDataEncryption:
+ case "server_side_data_encryption":
if result.HasServerSideDataEncryption {
continue
}
result.HasServerSideDataEncryption = true
result.ServerSideDataEncryption = v.Value.(string)
continue
- case pairServerSideEncryption:
+ case "server_side_encryption":
if result.HasServerSideEncryption {
continue
}
result.HasServerSideEncryption = true
result.ServerSideEncryption = v.Value.(string)
continue
- case pairServerSideEncryptionKeyID:
+ case "server_side_encryption_key_id":
if result.HasServerSideEncryptionKeyID {
continue
}
result.HasServerSideEncryptionKeyID = true
result.ServerSideEncryptionKeyID = v.Value.(string)
continue
- case pairStorageClass:
+ case "storage_class":
if result.HasStorageClass {
continue
}
@@ -867,18 +893,8 @@ func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreat
result.StorageClass = v.Value.(string)
continue
default:
- isUnsupportedPair = true
- }
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationCreateMultipart {
- continue
+ return pairStorageCreateMultipart{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageCreateMultipart{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -891,6 +907,8 @@ type pairStorageDelete struct {
pairs []Pair
HasMultipartID bool
MultipartID string
+ HasObjectMode bool
+ ObjectMode ObjectMode
}
// parsePairStorageDelete will parse Pair slice into *pairStorageDelete
@@ -900,9 +918,6 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error)
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "multipart_id":
if result.HasMultipartID {
@@ -911,19 +926,16 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error)
result.HasMultipartID = true
result.MultipartID = v.Value.(string)
continue
- default:
- isUnsupportedPair = true
- }
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationDelete {
+ case "object_mode":
+ if result.HasObjectMode {
+ continue
+ }
+ result.HasObjectMode = true
+ result.ObjectMode = v.Value.(ObjectMode)
continue
+ default:
+ return pairStorageDelete{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageDelete{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -945,9 +957,6 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) {
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "list_mode":
if result.HasListMode {
@@ -957,18 +966,8 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) {
result.ListMode = v.Value.(ListMode)
continue
default:
- isUnsupportedPair = true
+ return pairStorageList{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationList {
- continue
- }
- return pairStorageList{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -988,23 +987,10 @@ func (s *Storage) parsePairStorageListMultipart(opts []Pair) (pairStorageListMul
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
- }
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationListMultipart {
- continue
+ return pairStorageListMultipart{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageListMultipart{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -1024,23 +1010,10 @@ func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, er
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
default:
- isUnsupportedPair = true
+ return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationMetadata {
- continue
- }
- return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -1066,9 +1039,6 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) {
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "io_callback":
if result.HasIoCallback {
@@ -1092,18 +1062,8 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) {
result.Size = v.Value.(int64)
continue
default:
- isUnsupportedPair = true
+ return pairStorageRead{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationRead {
- continue
- }
- return pairStorageRead{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -1116,6 +1076,8 @@ type pairStorageStat struct {
pairs []Pair
HasMultipartID bool
MultipartID string
+ HasObjectMode bool
+ ObjectMode ObjectMode
}
// parsePairStorageStat will parse Pair slice into *pairStorageStat
@@ -1125,9 +1087,6 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) {
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "multipart_id":
if result.HasMultipartID {
@@ -1136,19 +1095,16 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) {
result.HasMultipartID = true
result.MultipartID = v.Value.(string)
continue
- default:
- isUnsupportedPair = true
- }
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationStat {
+ case "object_mode":
+ if result.HasObjectMode {
+ continue
+ }
+ result.HasObjectMode = true
+ result.ObjectMode = v.Value.(ObjectMode)
continue
+ default:
+ return pairStorageStat{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageStat{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -1182,9 +1138,6 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) {
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "content_md5":
if result.HasContentMd5 {
@@ -1207,28 +1160,28 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) {
result.HasIoCallback = true
result.IoCallback = v.Value.(func([]byte))
continue
- case pairServerSideDataEncryption:
+ case "server_side_data_encryption":
if result.HasServerSideDataEncryption {
continue
}
result.HasServerSideDataEncryption = true
result.ServerSideDataEncryption = v.Value.(string)
continue
- case pairServerSideEncryption:
+ case "server_side_encryption":
if result.HasServerSideEncryption {
continue
}
result.HasServerSideEncryption = true
result.ServerSideEncryption = v.Value.(string)
continue
- case pairServerSideEncryptionKeyID:
+ case "server_side_encryption_key_id":
if result.HasServerSideEncryptionKeyID {
continue
}
result.HasServerSideEncryptionKeyID = true
result.ServerSideEncryptionKeyID = v.Value.(string)
continue
- case pairStorageClass:
+ case "storage_class":
if result.HasStorageClass {
continue
}
@@ -1236,18 +1189,8 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) {
result.StorageClass = v.Value.(string)
continue
default:
- isUnsupportedPair = true
+ return pairStorageWrite{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationWrite {
- continue
- }
- return pairStorageWrite{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -1271,9 +1214,6 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "content_md5":
if result.HasContentMd5 {
@@ -1290,18 +1230,8 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe
result.IoCallback = v.Value.(func([]byte))
continue
default:
- isUnsupportedPair = true
+ return pairStorageWriteAppend{}, services.PairUnsupportedError{Pair: v}
}
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationWriteAppend {
- continue
- }
- return pairStorageWriteAppend{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -1323,9 +1253,6 @@ func (s *Storage) parsePairStorageWriteMultipart(opts []Pair) (pairStorageWriteM
}
for _, v := range opts {
- // isUnsupportedPair records whether current pair is unsupported.
- isUnsupportedPair := false
-
switch v.Key {
case "content_md5":
if result.HasContentMd5 {
@@ -1335,18 +1262,8 @@ func (s *Storage) parsePairStorageWriteMultipart(opts []Pair) (pairStorageWriteM
result.ContentMd5 = v.Value.(string)
continue
default:
- isUnsupportedPair = true
- }
-
- if !isUnsupportedPair {
- continue
- }
-
- // If user enables the loose operation feature, we will ignore PairUnsupportedError.
- if s.features.LooseOperationAll || s.features.LooseOperationWriteMultipart {
- continue
+ return pairStorageWriteMultipart{}, services.PairUnsupportedError{Pair: v}
}
- return pairStorageWriteMultipart{}, services.PairUnsupportedError{Pair: v}
}
// Check required pairs.
@@ -1414,6 +1331,11 @@ func (s *Storage) CompleteMultipartWithContext(ctx context.Context, o *Object, p
// Create will create a new object without any api call.
//
+// ## Behavior
+//
+// - Create SHOULD NOT send any API call.
+// - Create SHOULD accept ObjectMode pair as object mode.
+//
// This function will create a context by default.
func (s *Storage) Create(path string, pairs ...Pair) (o *Object) {
pairs = append(pairs, s.defaultPairs.Create...)
@@ -1450,6 +1372,31 @@ func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pair
return s.createAppend(ctx, path, opt)
}
+// CreateDir will create a new dir object.
+//
+// This function will create a context by default.
+func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) {
+ ctx := context.Background()
+ return s.CreateDirWithContext(ctx, path, pairs...)
+}
+
+// CreateDirWithContext will create a new dir object.
+func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) {
+ defer func() {
+ err = s.formatError("create_dir", err, path)
+ }()
+
+ pairs = append(pairs, s.defaultPairs.CreateDir...)
+ var opt pairStorageCreateDir
+
+ opt, err = s.parsePairStorageCreateDir(pairs)
+ if err != nil {
+ return
+ }
+
+ return s.createDir(ctx, path, opt)
+}
+
// CreateMultipart will create a new multipart.
//
// This function will create a context by default.
@@ -1475,7 +1422,17 @@ func (s *Storage) CreateMultipartWithContext(ctx context.Context, path string, p
return s.createMultipart(ctx, path, opt)
}
-// Delete will delete an Object from service.
+// Delete will delete an object from service.
+//
+// ## Behavior
+//
+// - Delete only delete one and only one object.
+// - Service DON'T NEED to support remove all.
+// - User NEED to implement remove_all by themself.
+// - Delete is idempotent.
+// - Successful delete always return nil error.
+// - Delete SHOULD never return `ObjectNotExist`
+// - Delete DON'T NEED to check the object exist or not.
//
// This function will create a context by default.
func (s *Storage) Delete(path string, pairs ...Pair) (err error) {
@@ -1483,7 +1440,17 @@ func (s *Storage) Delete(path string, pairs ...Pair) (err error) {
return s.DeleteWithContext(ctx, path, pairs...)
}
-// DeleteWithContext will delete an Object from service.
+// DeleteWithContext will delete an object from service.
+//
+// ## Behavior
+//
+// - Delete only delete one and only one object.
+// - Service DON'T NEED to support remove all.
+// - User NEED to implement remove_all by themself.
+// - Delete is idempotent.
+// - Successful delete always return nil error.
+// - Delete SHOULD never return `ObjectNotExist`
+// - Delete DON'T NEED to check the object exist or not.
func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) {
defer func() {
err = s.formatError("delete", err, path)
@@ -1594,6 +1561,12 @@ func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer,
// Stat will stat a path to get info of an object.
//
+// ## Behavior
+//
+// - Stat SHOULD accept ObjectMode pair as hints.
+// - Service COULD have different implementations for different object mode.
+// - Service SHOULD check if returning ObjectMode is match
+//
// This function will create a context by default.
func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) {
ctx := context.Background()
@@ -1601,6 +1574,12 @@ func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) {
}
// StatWithContext will stat a path to get info of an object.
+//
+// ## Behavior
+//
+// - Stat SHOULD accept ObjectMode pair as hints.
+// - Service COULD have different implementations for different object mode.
+// - Service SHOULD check if returning ObjectMode is match
func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) {
defer func() {
err = s.formatError("stat", err, path)
@@ -1703,4 +1682,5 @@ func (s *Storage) WriteMultipartWithContext(ctx context.Context, o *Object, r io
func init() {
services.RegisterServicer(Type, NewServicer)
services.RegisterStorager(Type, NewStorager)
+ services.RegisterSchema(Type, pairMap)
}
diff --git a/go.mod b/go.mod
index 3df7fdf..fb1022f 100644
--- a/go.mod
+++ b/go.mod
@@ -5,8 +5,8 @@ go 1.14
require (
github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
- github.com/beyondstorage/go-integration-test/v4 v4.0.0
- github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d
+ github.com/beyondstorage/go-integration-test/v4 v4.1.1
+ github.com/beyondstorage/go-storage/v4 v4.2.0
github.com/google/uuid v1.2.0
github.com/satori/go.uuid v1.2.0 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
diff --git a/go.sum b/go.sum
index fe9c273..f1a2c8a 100644
--- a/go.sum
+++ b/go.sum
@@ -4,22 +4,21 @@ github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible h1:hLUNPbx10wawWW7DeNExv
github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
-github.com/beyondstorage/go-integration-test/v4 v4.0.0 h1:tdXQV9yxQ3Q6p9xfyQKzK3MEo9r9j9g3uT5+3sbVtnQ=
-github.com/beyondstorage/go-integration-test/v4 v4.0.0/go.mod h1:26/JF4b0XxRN0pL4kihpnVNhbbw+QWvmmvgxfnFJDfA=
-github.com/beyondstorage/go-storage/v4 v4.0.0/go.mod h1:oa2dYco+xplPj99WSBnYVw/xXvRkIKWSSVDQKNZ5Kz8=
-github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d h1:s9t6VNNRDqmg+PXyXtVEsxIM7xZQIJOYlma28IpkbNQ=
-github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d/go.mod h1:kXMu07IDZaKtxbqI1ufuhqo0FjYe0nH7zPCbBanln/Y=
-github.com/beyondstorage/specs/go v0.0.0-20210521044836-3d41c1d9c97f/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ=
-github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9 h1:YSiF27cAHlDZk9q+oaEHQbA8dH8XTvYxeTOoPzNCwOQ=
-github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ=
+github.com/beyondstorage/go-integration-test/v4 v4.1.1 h1:9bSXKbr6hLb4+ZsmAhWE32fvqhyrpub4U4qgBGeth4A=
+github.com/beyondstorage/go-integration-test/v4 v4.1.1/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc=
+github.com/beyondstorage/go-storage/v4 v4.2.0 h1:J0xqqy4qEQRtIS2zUWMA5wRXVHx/cxX5fHsU2ezA3+I=
+github.com/beyondstorage/go-storage/v4 v4.2.0/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU=
+github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259 h1:mW9XpHLc6pdXBRnsha1VlqF0rNsB/Oc+8l+5UYngmRA=
+github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w=
github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY=
github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8=
github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -37,19 +36,22 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/pelletier/go-toml v1.9.1 h1:a6qW1EVNZWH9WGI6CsYdD8WAylkoXBS5yv0XHlh17Tc=
-github.com/pelletier/go-toml v1.9.1/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -68,6 +70,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/service.toml b/service.toml
index 99a737c..6f04c56 100644
--- a/service.toml
+++ b/service.toml
@@ -7,20 +7,24 @@ required = ["credential"]
optional = ["service_features", "default_service_pairs", "endpoint", "http_client_options"]
[namespace.storage]
-implement = ["appender", "multiparter"]
+features = ["virtual_dir"]
+implement = ["appender", "direr", "multiparter"]
[namespace.storage.new]
required = ["name"]
optional = ["storage_features", "default_storage_pairs", "work_dir"]
[namespace.storage.op.create]
-optional = ["multipart_id"]
+optional = ["multipart_id", "object_mode"]
+
+[namespace.storage.op.create_dir]
+optional = ["storage_class"]
[namespace.storage.op.delete]
-optional = ["multipart_id"]
+optional = ["multipart_id", "object_mode"]
[namespace.storage.op.stat]
-optional = ["multipart_id"]
+optional = ["multipart_id", "object_mode"]
[namespace.storage.op.list]
optional = ["list_mode"]
diff --git a/storage.go b/storage.go
index 56cd316..bd82222 100644
--- a/storage.go
+++ b/storage.go
@@ -8,6 +8,7 @@ import (
"github.com/aliyun/aliyun-oss-go-sdk/oss"
+ ps "github.com/beyondstorage/go-storage/v4/pairs"
"github.com/beyondstorage/go-storage/v4/pkg/headers"
"github.com/beyondstorage/go-storage/v4/pkg/iowrap"
"github.com/beyondstorage/go-storage/v4/services"
@@ -46,16 +47,27 @@ func (s *Storage) completeMultipart(ctx context.Context, o *Object, parts []*Par
}
func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) {
+ rp := s.getAbsPath(path)
+
// Handle create multipart object separately.
if opt.HasMultipartID {
o = s.newObject(true)
o.Mode = ModePart
o.SetMultipartID(opt.MultipartID)
} else {
- o = s.newObject(false)
- o.Mode = ModeRead
+ if opt.HasObjectMode && opt.ObjectMode.IsDir() {
+ if !s.features.VirtualDir {
+ return
+ }
+ rp += "/"
+ o = s.newObject(true)
+ o.Mode = ModeDir
+ } else {
+ o = s.newObject(false)
+ o.Mode = ModeRead
+ }
}
- o.ID = s.getAbsPath(path)
+ o.ID = rp
o.Path = path
return o
}
@@ -94,6 +106,35 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage
return o, nil
}
+func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) {
+ if !s.features.VirtualDir {
+ err = NewOperationNotImplementedError("create_dir")
+ return
+ }
+ rp := s.getAbsPath(path)
+
+ // Add `/` at the end of path to simulate directory.
+ // ref: https://help.aliyun.com/document_detail/31978.html#title-gkg-amg-aes
+ rp += "/"
+
+ options := make([]oss.Option, 0)
+ options = append(options, oss.ContentLength(0))
+ if opt.HasStorageClass {
+ options = append(options, oss.StorageClass(oss.StorageClassType(opt.StorageClass)))
+ }
+
+ err = s.bucket.PutObject(rp, nil, options...)
+ if err != nil {
+ return
+ }
+
+ o = s.newObject(true)
+ o.Path = path
+ o.ID = rp
+ o.Mode |= ModeDir
+ return
+}
+
func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStorageCreateMultipart) (o *Object, err error) {
rp := s.getAbsPath(path)
@@ -152,6 +193,15 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete
return
}
+ if opt.HasObjectMode && opt.ObjectMode.IsDir() {
+ if !s.features.VirtualDir {
+ err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)}
+ return
+ }
+
+ rp += "/"
+ }
+
// OSS DeleteObject is idempotent, so we don't need to check NoSuchKey error.
//
// References
@@ -390,6 +440,15 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
return o, nil
}
+ if opt.HasObjectMode && opt.ObjectMode.IsDir() {
+ if !s.features.VirtualDir {
+ err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)}
+ return
+ }
+
+ rp += "/"
+ }
+
output, err := s.bucket.GetObjectMeta(rp)
if err != nil {
return nil, err
@@ -398,7 +457,11 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
o = s.newObject(true)
o.ID = rp
o.Path = path
- o.Mode |= ModeRead
+ if opt.HasObjectMode && opt.ObjectMode.IsDir() {
+ o.Mode |= ModeDir
+ } else {
+ o.Mode |= ModeRead
+ }
if v := output.Get(headers.ContentLength); v != "" {
size, err := strconv.ParseInt(v, 10, 64)
diff --git a/tests/storage_test.go b/tests/storage_test.go
index 4cbd418..0b0776c 100644
--- a/tests/storage_test.go
+++ b/tests/storage_test.go
@@ -27,3 +27,10 @@ func TestMultiparter(t *testing.T) {
}
tests.TestMultiparter(t, setupTest(t))
}
+
+func TestDirer(t *testing.T) {
+ if os.Getenv("STORAGE_OSS_INTEGRATION_TEST") != "on" {
+ t.Skipf("STORAGE_OSS_INTEGRATION_TEST is not 'on', skipped")
+ }
+ tests.TestDirer(t, setupTest(t))
+}
diff --git a/tests/utils_test.go b/tests/utils_test.go
index 6574b98..c803638 100644
--- a/tests/utils_test.go
+++ b/tests/utils_test.go
@@ -18,6 +18,9 @@ func setupTest(t *testing.T) types.Storager {
ps.WithName(os.Getenv("STORAGE_OSS_NAME")),
ps.WithEndpoint(os.Getenv("STORAGE_OSS_ENDPOINT")),
ps.WithWorkDir("/"+uuid.New().String()+"/"),
+ oss.WithStorageFeatures(oss.StorageFeatures{
+ VirtualDir: true,
+ }),
)
if err != nil {
t.Errorf("new storager: %v", err)
diff --git a/utils.go b/utils.go
index 04e1d25..158130a 100644
--- a/utils.go
+++ b/utils.go
@@ -42,6 +42,7 @@ type Storage struct {
typ.UnimplementedStorager
typ.UnimplementedAppender
typ.UnimplementedMultiparter
+ typ.UnimplementedDirer
}
// String implements Storager.String
From 8273b5f3ac2a9f881372c68085400c99a5b41606 Mon Sep 17 00:00:00 2001
From: JinnyYi <jinnyyi@yunify.com>
Date: Mon, 28 Jun 2021 15:58:51 +0800
Subject: [PATCH 2/4] Implement GSP-117 Rename Service to System as the
Opposite to Global
---
storage.go | 4 ++--
utils.go | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/storage.go b/storage.go
index bd82222..fd35b91 100644
--- a/storage.go
+++ b/storage.go
@@ -490,7 +490,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
o.SetContentType(v)
}
- var sm ObjectMetadata
+ var sm ObjectSystemMetadata
if v := output.Get(storageClassHeader); v != "" {
sm.StorageClass = v
}
@@ -500,7 +500,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
if v := output.Get(serverSideEncryptionKeyIdHeader); v != "" {
sm.ServerSideEncryptionKeyID = v
}
- o.SetServiceMetadata(sm)
+ o.SetSystemMetadata(sm)
return o, nil
}
diff --git a/utils.go b/utils.go
index 158130a..48b97fd 100644
--- a/utils.go
+++ b/utils.go
@@ -260,11 +260,11 @@ func (s *Storage) formatFileObject(v oss.ObjectProperties) (o *typ.Object, err e
o.SetEtag(v.ETag)
}
- var sm ObjectMetadata
+ var sm ObjectSystemMetadata
if value := v.Type; value != "" {
sm.StorageClass = value
}
- o.SetServiceMetadata(sm)
+ o.SetSystemMetadata(sm)
return
}
From 5981c270d76f74b7361188ec170559dd3799b6a9 Mon Sep 17 00:00:00 2001
From: JinnyYi <jinnyyi@yunify.com>
Date: Mon, 28 Jun 2021 16:03:25 +0800
Subject: [PATCH 3/4] pkg/endpoint deprecated by go-endpoint
---
go.mod | 1 +
go.sum | 2 ++
utils.go | 14 ++++++++++++--
3 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/go.mod b/go.mod
index fb1022f..99804fc 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.14
require (
github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
+ github.com/beyondstorage/go-endpoint v1.0.1
github.com/beyondstorage/go-integration-test/v4 v4.1.1
github.com/beyondstorage/go-storage/v4 v4.2.0
github.com/google/uuid v1.2.0
diff --git a/go.sum b/go.sum
index f1a2c8a..2839254 100644
--- a/go.sum
+++ b/go.sum
@@ -4,6 +4,8 @@ github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible h1:hLUNPbx10wawWW7DeNExv
github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
+github.com/beyondstorage/go-endpoint v1.0.1 h1:F8x2dGLMu9je6g7zPbKoxCXDlug97K26SeCx7KEHgyg=
+github.com/beyondstorage/go-endpoint v1.0.1/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM=
github.com/beyondstorage/go-integration-test/v4 v4.1.1 h1:9bSXKbr6hLb4+ZsmAhWE32fvqhyrpub4U4qgBGeth4A=
github.com/beyondstorage/go-integration-test/v4 v4.1.1/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc=
github.com/beyondstorage/go-storage/v4 v4.2.0 h1:J0xqqy4qEQRtIS2zUWMA5wRXVHx/cxX5fHsU2ezA3+I=
diff --git a/utils.go b/utils.go
index 48b97fd..e7a9704 100644
--- a/utils.go
+++ b/utils.go
@@ -6,9 +6,9 @@ import (
"github.com/aliyun/aliyun-oss-go-sdk/oss"
+ "github.com/beyondstorage/go-endpoint"
ps "github.com/beyondstorage/go-storage/v4/pairs"
"github.com/beyondstorage/go-storage/v4/pkg/credential"
- "github.com/beyondstorage/go-storage/v4/pkg/endpoint"
"github.com/beyondstorage/go-storage/v4/pkg/httpclient"
"github.com/beyondstorage/go-storage/v4/services"
typ "github.com/beyondstorage/go-storage/v4/types"
@@ -97,12 +97,22 @@ func newServicer(pairs ...typ.Pair) (srv *Service, err error) {
return nil, err
}
+ var url string
+ switch ep.Protocol() {
+ case endpoint.ProtocolHTTP:
+ url, _, _ = ep.HTTP()
+ case endpoint.ProtocolHTTPS:
+ url, _, _ = ep.HTTPS()
+ default:
+ return nil, services.PairUnsupportedError{Pair: ps.WithEndpoint(opt.Endpoint)}
+ }
+
var copts []oss.ClientOption
if opt.HasHTTPClientOptions {
copts = append(copts, oss.HTTPClient(httpclient.New(opt.HTTPClientOptions)))
}
- srv.service, err = oss.New(ep.String(), ak, sk, copts...)
+ srv.service, err = oss.New(url, ak, sk, copts...)
if err != nil {
return nil, err
}
From ed8bc75beeb5200fcdeab9708f5eae2ea1608c55 Mon Sep 17 00:00:00 2001
From: JinnyYi <jinnyyi@yunify.com>
Date: Mon, 28 Jun 2021 16:19:23 +0800
Subject: [PATCH 4/4] Implement GSP-97 Add Restrictions In Storage Metadata
---
storage.go | 27 +++++++++++++++++++++++----
utils.go | 9 +++++++++
2 files changed, 32 insertions(+), 4 deletions(-)
diff --git a/storage.go b/storage.go
index fd35b91..42c6108 100644
--- a/storage.go
+++ b/storage.go
@@ -2,6 +2,7 @@ package oss
import (
"context"
+ "fmt"
"io"
"strconv"
"time"
@@ -165,10 +166,6 @@ func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStor
o.Path = path
o.Mode |= ModePart
o.SetMultipartID(output.UploadID)
- // set multipart restriction
- o.SetMultipartNumberMaximum(multipartNumberMaximum)
- o.SetMultipartNumberMaximum(multipartSizeMaximum)
- o.SetMultipartSizeMinimum(multipartSizeMinimum)
return o, nil
}
@@ -251,6 +248,14 @@ func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) {
meta = NewStorageMeta()
meta.Name = s.bucket.BucketName
meta.WorkDir = s.workDir
+ // set write restriction
+ meta.SetWriteSizeMaximum(writeSizeMaximum)
+ // set append restriction
+ meta.SetAppendTotalSizeMaximum(appendTotalSizeMaximum)
+ // set multipart restrictions
+ meta.SetMultipartNumberMaximum(multipartNumberMaximum)
+ meta.SetMultipartNumberMaximum(multipartSizeMaximum)
+ meta.SetMultipartSizeMinimum(multipartSizeMinimum)
return
}
@@ -506,6 +511,11 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
}
func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) {
+ if size > writeSizeMaximum {
+ err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied)
+ return
+ }
+
if opt.HasIoCallback {
r = iowrap.CallbackReader(r, opt.IoCallback)
}
@@ -563,6 +573,15 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size
}
func (s *Storage) writeMultipart(ctx context.Context, o *Object, r io.Reader, size int64, index int, opt pairStorageWriteMultipart) (n int64, part *Part, err error) {
+ if index < 0 || index >= multipartNumberMaximum {
+ err = fmt.Errorf("multipart number limit exceeded: %w", services.ErrRestrictionDissatisfied)
+ return
+ }
+ if size > multipartSizeMaximum {
+ err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied)
+ return
+ }
+
imur := oss.InitiateMultipartUploadResult{
Bucket: s.bucket.BucketName,
Key: o.ID,
diff --git a/utils.go b/utils.go
index e7a9704..ff70c71 100644
--- a/utils.go
+++ b/utils.go
@@ -322,3 +322,12 @@ const (
// multipartSizeMinimum is the minimum size for each part, 100KB.
multipartSizeMinimum = 100 * 1024
)
+
+const (
+ // writeSizeMaximum is the maximum size for each object with a single PUT operation, 5GB.
+ // ref: https://help.aliyun.com/document_detail/31978.html#title-gkg-amg-aes
+ writeSizeMaximum = 5 * 1024 * 1024 * 1024
+ // appendSizeMaximum is the total maximum size for an append object, 5GB.
+ // ref: https://help.aliyun.com/document_detail/31981.html?spm=a2c4g.11186623.6.1684.479a3ea7S8dRgB#title-22f-5c3-0sv
+ appendTotalSizeMaximum = 5 * 1024 * 1024 * 1024
+)