Skip to content

Commit

Permalink
support parallel md5, sha1, sha256 for uploading part
Browse files Browse the repository at this point in the history
  • Loading branch information
taowei.wtw authored and kkuai committed Mar 24, 2021
1 parent 8882a33 commit 261dd8b
Show file tree
Hide file tree
Showing 8 changed files with 77 additions and 7 deletions.
5 changes: 5 additions & 0 deletions oss/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ func (client Client) CreateBucket(bucketName string, options ...Option) error {

isStorageSet, valStroage, _ := IsOptionSet(options, storageClass)
isRedundancySet, valRedundancy, _ := IsOptionSet(options, redundancyType)
isObjectHashFuncSet, valHashFunc, _ := IsOptionSet(options, objectHashFunc)
if isStorageSet {
cbConfig.StorageClass = valStroage.(StorageClassType)
}
Expand All @@ -125,6 +126,10 @@ func (client Client) CreateBucket(bucketName string, options ...Option) error {
cbConfig.DataRedundancyType = valRedundancy.(DataRedundancyType)
}

if isObjectHashFuncSet {
cbConfig.ObjectHashFunction = valHashFunc.(ObjecthashFuncType)
}

bs, err := xml.Marshal(cbConfig)
if err != nil {
return err
Expand Down
5 changes: 4 additions & 1 deletion oss/conn.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@ var signKeyList = []string{"acl", "uploads", "location", "cors",
"policy", "stat", "encryption", "versions", "versioning", "versionId", "requestPayment",
"x-oss-request-payer", "sequential",
"inventory", "inventoryId", "continuation-token", "asyncFetch",
"worm", "wormId", "wormExtend"}
"worm", "wormId", "wormExtend", "withHashContext",
"x-oss-enable-md5", "x-oss-enable-sha1", "x-oss-enable-sha256",
"x-oss-hash-ctx", "x-oss-md5-ctx",
}

// init initializes Conn
func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) error {
Expand Down
10 changes: 10 additions & 0 deletions oss/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,14 @@ const (
RedundancyZRS DataRedundancyType = "ZRS"
)

//ObjecthashFuncType
type ObjecthashFuncType string

const (
HashFuncSha1 ObjecthashFuncType = "SHA-1"
HashFuncSha256 ObjecthashFuncType = "SHA-256"
)

// PayerType the type of request payer
type PayerType string

Expand Down Expand Up @@ -193,6 +201,8 @@ const (
HTTPHeaderOssForbidOverWrite = "X-Oss-Forbid-Overwrite"
HTTPHeaderOssRangeBehavior = "X-Oss-Range-Behavior"
HTTPHeaderOssTaskID = "X-Oss-Task-Id"
HTTPHeaderOssHashCtx = "X-Oss-Hash-Ctx"
HTTPHeaderOssMd5Ctx = "X-Oss-Md5-Ctx"
)

// HTTP Param
Expand Down
7 changes: 2 additions & 5 deletions oss/multipart.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,8 @@ func (bucket Bucket) InitiateMultipartUpload(objectKey string, options ...Option
var imur InitiateMultipartUploadResult
opts := AddContentType(options, objectKey)
params, _ := GetRawParams(options)
_, ok := params["sequential"]
if ok {
// convert "" to nil
params["sequential"] = nil
}
paramKeys := []string{"sequential", "withHashContext", "x-oss-enable-md5", "x-oss-enable-sha1", "x-oss-enable-sha256"}
ConvertEmptyValueToNil(params, paramKeys)
params["uploads"] = nil

resp, err := bucket.do("POST", objectKey, params, opts, nil, nil)
Expand Down
42 changes: 42 additions & 0 deletions oss/option.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
storageClass = "storage-class"
responseHeader = "x-response-header"
redundancyType = "redundancy-type"
objectHashFunc = "object-hash-func"
)

type (
Expand Down Expand Up @@ -290,6 +291,22 @@ func RangeBehavior(value string) Option {
return setHeader(HTTPHeaderOssRangeBehavior, value)
}

func PartHashCtxHeader(value string) Option {
return setHeader(HTTPHeaderOssHashCtx, value)
}

func PartMd5CtxHeader(value string) Option {
return setHeader(HTTPHeaderOssMd5Ctx, value)
}

func PartHashCtxParam(value string) Option {
return addParam("x-oss-hash-ctx", value)
}

func PartMd5CtxParam(value string) Option {
return addParam("x-oss-md5-ctx", value)
}

// Delimiter is an option to set delimiler parameter
func Delimiter(value string) Option {
return addParam("delimiter", value)
Expand Down Expand Up @@ -365,6 +382,26 @@ func Sequential() Option {
return addParam("sequential", "")
}

// WithHashContext is an option to set withHashContext parameter for InitiateMultipartUpload
func WithHashContext() Option {
return addParam("withHashContext", "")
}

// EnableMd5 is an option to set x-oss-enable-md5 parameter for InitiateMultipartUpload
func EnableMd5() Option {
return addParam("x-oss-enable-md5", "")
}

// EnableSha1 is an option to set x-oss-enable-sha1 parameter for InitiateMultipartUpload
func EnableSha1() Option {
return addParam("x-oss-enable-sha1", "")
}

// EnableSha256 is an option to set x-oss-enable-sha256 parameter for InitiateMultipartUpload
func EnableSha256() Option {
return addParam("x-oss-enable-sha256", "")
}

// ListType is an option to set List-type parameter for ListObjectsV2
func ListType(value int) Option {
return addParam("list-type", strconv.Itoa(value))
Expand Down Expand Up @@ -406,6 +443,11 @@ func RedundancyType(value DataRedundancyType) Option {
return addArg(redundancyType, value)
}

// RedundancyType bucket data redundancy type
func ObjectHashFunc(value ObjecthashFuncType) Option {
return addArg(objectHashFunc, value)
}

// Checkpoint configuration
type cpConfig struct {
IsEnable bool
Expand Down
4 changes: 3 additions & 1 deletion oss/progress.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package oss

import "io"
import (
"io"
)

// ProgressEventType defines transfer progress event type
type ProgressEventType int
Expand Down
1 change: 1 addition & 0 deletions oss/type.go
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,7 @@ type createBucketConfiguration struct {
XMLName xml.Name `xml:"CreateBucketConfiguration"`
StorageClass StorageClassType `xml:"StorageClass,omitempty"`
DataRedundancyType DataRedundancyType `xml:"DataRedundancyType,omitempty"`
ObjectHashFunction ObjecthashFuncType `xml:"ObjectHashFunction,omitempty"`
}

// LiveChannelConfiguration defines the configuration for live-channel
Expand Down
10 changes: 10 additions & 0 deletions oss/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -510,3 +510,13 @@ func (drc *DiscardReadCloser) Close() error {
}
return nil
}

func ConvertEmptyValueToNil(params map[string]interface{}, keys []string) {
for _, key := range keys {
value, ok := params[key]
if ok && value == "" {
// convert "" to nil
params[key] = nil
}
}
}

0 comments on commit 261dd8b

Please sign in to comment.