feat: optimize GetObjectMetadatasInput validate in dfstore (#2686)

Signed-off-by: Gaius <[email protected]>

client/dfstore/dfstore.go

@@ -40,6 +40,14 @@ import (
 	pkgobjectstorage "d7y.io/dragonfly/v2/pkg/objectstorage"
 )
 
+const (
+	// DefaultGetObjectMetadatasLimit is the default limit of get object metadatas.
+	DefaultGetObjectMetadatasLimit = 1000
+
+	// MaxGetObjectMetadatasLimit is the max limit of get object metadatas.
+	MaxGetObjectMetadatasLimit = 1000
+)
+
 // Dfstore is the interface used for object storage.
 type Dfstore interface {
 	// CreateBucketRequestWithContext returns *http.Request of create bucket.
@@ -226,13 +234,20 @@ type GetObjectMetadatasInput struct {
 	Limit int64
 }
 
+// Convert converts GetObjectMetadatasInput fields.
+func (i *GetObjectMetadatasInput) Convert() {
+	if i.Limit == 0 {
+		i.Limit = DefaultGetObjectMetadatasLimit
+	}
+}
+
 // Validate validates GetObjectMetadatasInput fields.
 func (i *GetObjectMetadatasInput) Validate() error {
 	if i.BucketName == "" {
 		return errors.New("invalid BucketName")
 	}
 
-	if i.Limit < 0 || i.Limit > pkgobjectstorage.DefaultGetObjectMetadatasLimit {
+	if i.Limit <= 0 || i.Limit > MaxGetObjectMetadatasLimit {
 		return errors.New("invalid limit")
 	}
 
@@ -241,6 +256,9 @@ func (i *GetObjectMetadatasInput) Validate() error {
 
 // GetObjectMetadatasRequestWithContext returns *http.Request of getting object metadatas.
 func (dfs *dfstore) GetObjectMetadatasRequestWithContext(ctx context.Context, input *GetObjectMetadatasInput) (*http.Request, error) {
+	// Convert input fields.
+	input.Convert()
+
 	if err := input.Validate(); err != nil {
 		return nil, err
 	}

pkg/objectstorage/constants.go

@@ -16,6 +16,8 @@
 
 package objectstorage
 
+import "time"
+
 const (
 	// ServiceNameS3 is name of s3 storage.
 	ServiceNameS3 = "s3"
@@ -55,12 +57,38 @@ const (
 	MethodList Method = "List"
 )
 
+const (
+	// DefaultS3ForcePathStyle is the default force path style of s3.
+	DefaultS3ForcePathStyle = true
+)
+
 const (
 	// OBSStorageClassStandardIA is the standard ia storage class of obs.
 	OBSStorageClassStandardIA = "STANDARD_IA"
 )
 
 const (
-	// DefaultGetObjectMetadatasLimit is the default limit of get object metadatas.
-	DefaultGetObjectMetadatasLimit = 1000
+	// DefaultTLSHandshakeTimeout is the default timeout of tls handshake of http client.
+	DefaultTLSHandshakeTimeout = 30 * time.Second
+
+	// DefaultResponseHeaderTimeout is the default timeout of response header of http client.
+	DefaultResponseHeaderTimeout = 30 * time.Second
+
+	// DefaultIdleConnTimeout is the default timeout of idle connection of http client.
+	DefaultIdleConnTimeout = 5 * time.Minute
+
+	// DefaultMaxIdleConnsPerHost is the default max idle connections per host of http client.
+	DefaultMaxIdleConnsPerHost = 500
+
+	// DefaultReadBufferSize is the default read buffer size of http client.
+	DefaultReadBufferSize = 32 << 10
+
+	// DefaultWriteBufferSize is the default write buffer size of http client.
+	DefaultWriteBufferSize = 32 << 10
+
+	// DefaultDisableCompression is the default disable compression of http client.
+	DefaultDisableCompression = true
+
+	// DefaultTimeout is the default timeout of http client.
+	DefaultTimeout = time.Hour
 )

pkg/objectstorage/objectstorage.go

@@ -20,7 +20,6 @@ package objectstorage
 
 import (
 	"context"
-	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
@@ -169,20 +168,18 @@ func New(name, region, endpoint, accessKey, secretKey string, options ...Option)
 		endpoint:         endpoint,
 		accessKey:        accessKey,
 		secretKey:        secretKey,
-		s3ForcePathStyle: true,
+		s3ForcePathStyle: DefaultS3ForcePathStyle,
 		httpClient: &http.Client{
 			Transport: &http.Transport{
-				Proxy:                 http.ProxyFromEnvironment,
-				TLSHandshakeTimeout:   time.Second * 20,
-				ResponseHeaderTimeout: time.Second * 30,
-				IdleConnTimeout:       time.Second * 300,
-				MaxIdleConnsPerHost:   500,
-				ReadBufferSize:        32 << 10,
-				WriteBufferSize:       32 << 10,
-				DisableCompression:    true,
-				TLSClientConfig:       &tls.Config{},
+				TLSHandshakeTimeout:   DefaultTLSHandshakeTimeout,
+				ResponseHeaderTimeout: DefaultResponseHeaderTimeout,
+				IdleConnTimeout:       DefaultIdleConnTimeout,
+				MaxIdleConnsPerHost:   DefaultMaxIdleConnsPerHost,
+				ReadBufferSize:        DefaultReadBufferSize,
+				WriteBufferSize:       DefaultWriteBufferSize,
+				DisableCompression:    DefaultDisableCompression,
 			},
-			Timeout: time.Hour,
+			Timeout: DefaultTimeout,
 		},
 	}
 

pkg/objectstorage/obs.go

@@ -122,10 +122,6 @@ func (o *obs) GetObjectMetadata(ctx context.Context, bucketName, objectKey strin
 
 // GetObjectMetadatas returns the metadatas of the objects.
 func (o *obs) GetObjectMetadatas(ctx context.Context, bucketName, prefix, marker, delimiter string, limit int64) (*ObjectMetadatas, error) {
-	if limit == 0 {
-		limit = DefaultGetObjectMetadatasLimit
-	}
-
 	resp, err := o.client.ListObjects(&huaweiobs.ListObjectsInput{
 		ListObjsInput: huaweiobs.ListObjsInput{
 			Prefix:    prefix,

pkg/objectstorage/oss.go

@@ -135,10 +135,6 @@ func (o *oss) GetObjectMetadatas(ctx context.Context, bucketName, prefix, marker
 		return nil, err
 	}
 
-	if limit == 0 {
-		limit = DefaultGetObjectMetadatasLimit
-	}
-
 	resp, err := bucket.ListObjects(aliyunoss.Prefix(prefix), aliyunoss.Marker(marker), aliyunoss.Delimiter(delimiter), aliyunoss.MaxKeys(int(limit)))
 	if err != nil {
 		return nil, err

pkg/objectstorage/s3.go

@@ -124,10 +124,6 @@ func (s *s3) GetObjectMetadata(ctx context.Context, bucketName, objectKey string
 
 // GetObjectMetadatas returns the metadatas of the objects.
 func (s *s3) GetObjectMetadatas(ctx context.Context, bucketName, prefix, marker, delimiter string, limit int64) (*ObjectMetadatas, error) {
-	if limit == 0 {
-		limit = DefaultGetObjectMetadatasLimit
-	}
-
 	resp, err := s.client.ListObjectsWithContext(ctx, &awss3.ListObjectsInput{
 		Bucket:    aws.String(bucketName),
 		Prefix:    aws.String(prefix),
@@ -156,6 +152,7 @@ func (s *s3) GetObjectMetadatas(ctx context.Context, bucketName, prefix, marker,
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode commonPrefixes %s, error: %s", *commonPrefix.Prefix, err.Error())
 		}
+
 		commonPrefixes = append(commonPrefixes, prefix)
 	}