Package AWS:

- rework MultipartUpload process & helper
- update test to use lib size
- update object multipart to use new helper
Package IO Utils :
- add truncate & sync to FileProgress

.golangci.yml

@@ -82,20 +82,20 @@ linters:
   disable:
 #    - bodyclose
 #    - contextcheck
-#    - cyclop
+    - cyclop
     - deadcode
 #    - errname
 #    - errorlint
     - exhaustive
     - exhaustivestruct
     - exhaustruct
 #    - forbidigo
-#    - funlen
+    - funlen
 #    - gci
 #    - gochecknoglobals
 #    - gochecknoinits
 #    - gocognit
-#    - gocritic
+    - gocritic
 #    - gocyclo
 #    - godot
 #    - godox
@@ -106,16 +106,16 @@ linters:
     - ifshort
 #    - interfacebloat
     - interfacer
-#    - ireturn
+    - ireturn
 #    - lll
     - maligned
 #    - nakedret
-#    - nestif
+    - nestif
 #    - nilerr
 #    - nlreturn
 #    - noctx
-#    - nolintlint
-#    - nonamedreturns
+    - nolintlint
+    - nonamedreturns
     - nosnakecase
 #    - revive
 #    - rowserrcheck
@@ -129,5 +129,5 @@ linters:
     - varnamelen
 #    - wastedassign
 #    - whitespace
-#    - wrapcheck
-#    - wsl
+    - wrapcheck
+    - wsl

aws/aws_suite_test.go

@@ -31,6 +31,7 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
+	libsiz "github.com/nabbar/golib/size"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -240,8 +241,8 @@ func WaitMinio(host string) bool {
 	return err == nil
 }
 
-func randContent(size int) *bytes.Buffer {
-	p := make([]byte, size)
+func randContent(size libsiz.Size) *bytes.Buffer {
+	p := make([]byte, size.Int64())
 
 	_, err := rand.Read(p)
 

aws/bucket_test.go

@@ -26,6 +26,7 @@
 package aws_test
 
 import (
+	libsiz "github.com/nabbar/golib/size"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -52,7 +53,7 @@ var _ = Describe("Bucket", func() {
 			It("Must succeed", func() {
 				var (
 					err error
-					rnd = randContent(64 * 1024)
+					rnd = randContent(10 * libsiz.SizeMega)
 				)
 
 				err = cli.Object().MultipartPut("object", rnd)

aws/multipart/errors.go

@@ -0,0 +1,36 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2020 Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package multipart
+
+import "fmt"
+
+var (
+	ErrInvalidInstance = fmt.Errorf("invalid instance")
+	ErrInvalidClient   = fmt.Errorf("invalid aws S3 client")
+	ErrInvalidResponse = fmt.Errorf("invalid aws S3 response")
+	ErrInvalidUploadID = fmt.Errorf("invalid aws s3 MPU Upload ID")
+	ErrInvalidTMPFile  = fmt.Errorf("invalid working or temporary file")
+)

aws/multipart/interface.go

@@ -0,0 +1,67 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2020 Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package multipart
+
+import (
+	"io"
+	"sync"
+
+	sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
+	libctx "github.com/nabbar/golib/context"
+	libsiz "github.com/nabbar/golib/size"
+)
+
+const DefaultPartSize = 5 * libsiz.SizeMega
+
+type FuncClientS3 func() *sdksss.Client
+
+type MultiPart interface {
+	io.WriteCloser
+
+	RegisterContext(fct libctx.FuncContext)
+	RegisterClientS3(fct FuncClientS3)
+	RegisterMultipartID(id string)
+	RegisterWorkingFile(file string) error
+
+	StartMPU() error
+	StopMPU(abort bool) error
+
+	AddPart(r io.Reader) (n int64, e error)
+	AddToPart(p []byte) (n int, e error)
+	RegisterPart(etag string)
+}
+
+func New(partSize libsiz.Size, object string, bucket string) MultiPart {
+	return &mpu{
+		m: sync.RWMutex{},
+		c: nil,
+		s: partSize,
+		i: "",
+		o: object,
+		b: bucket,
+		n: 0,
+	}
+}

aws/multipart/io.go

@@ -0,0 +1,34 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2020 Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package multipart
+
+func (m *mpu) Write(p []byte) (n int, err error) {
+	return m.AddToPart(p)
+}
+
+func (m *mpu) Close() error {
+	return m.StopMPU(false)
+}