[communitybridge#4254] Feature/Missing S3 File upload

- Implemented script that handles file upload via signature records or a given file path

Signed-off-by: Harold Wanyama <[email protected]>

cla-backend-go/cmd/s3_upload/main.go

@@ -89,30 +89,111 @@ type APIErrorResponse struct {
 	Message   string `json:"message"`
 }
 
-func main() {
+func main() { // nolint
 	ctx := context.Background()
 	f := logrus.Fields{
 		"functionName": "main",
 	}
 	// var toUpdate []*signatures.ItemSignature
 
 	dryRun := flag.Bool("dry-run", false, "dry run mode")
+	folder := flag.String("folder", "", "folder to upload the s3 documents")
+	meta := flag.String("meta", "", "meta data to upload the s3 documents")
 
 	flag.Parse()
 
 	// Fetch all the signatures from 2024-02-01T00:00:00.000Z
 	startDate := "2024-02-01T00:00:00.000Z"
 
+	if dryRun != nil && *dryRun {
+		log.WithFields(f).Debug("dry-run mode enabled")
+	}
+
+	if folder != nil && *folder != "" && meta != nil && *meta != "" {
+		log.WithFields(f).Debugf("folder: %s, meta: %s", *folder, *meta)
+		// var metaMap map[string]string
+
+		// Read csv file
+		file, err := os.Open(*meta)
+		if err != nil {
+			log.WithFields(f).WithError(err).Warn("problem opening meta file")
+			return
+		}
+
+		reader := csv.NewReader(file)
+		records, err := reader.ReadAll()
+
+		count := len(records)
+		log.WithFields(f).Debugf("processing %d records", count)
+
+		passed := 0
+
+		if err != nil {
+			log.WithFields(f).WithError(err).Warn("problem reading meta file")
+			return
+		}
+
+		var wg sync.WaitGroup
+
+		// Limit the number of concurrent uploads
+		semaphore := make(chan struct{}, 5)
+
+		for _, record := range records {
+			wg.Add(1)
+			semaphore <- struct{}{}
+			go func(record []string) {
+				defer wg.Done()
+				defer func() { <-semaphore }()
+				fileName := record[0]
+				envelopeID := record[1]
+				signatureID := record[2]
+				projectID := record[3]
+				referenceID := record[4]
+				log.WithFields(f).Debugf("uploading file: %s, envelopeID: %s, signatureID: %s, projectID: %s, referenceID: %s", fileName, envelopeID, signatureID, projectID, referenceID)
+				// Upload the file
+				file, err := os.Open(*folder + "/" + fileName) // nolint
+				if err != nil {
+					log.WithFields(f).WithError(err).Warn("problem opening file")
+					failed++
+					return
+				}
+
+				if dryRun != nil && *dryRun {
+					log.WithFields(f).Debugf("dry-run mode enabled, skipping file upload: %s", fileName)
+					return
+				}
+
+				// Upload the document
+				log.WithFields(f).Debugf("uploading document for signature...: %s", signatureID)
+
+				err = utils.UploadFileToS3(file, projectID, utils.ClaTypeICLA, referenceID, signatureID)
+
+				if err != nil {
+					log.WithFields(f).WithError(err).Warn("problem uploading file")
+					failed++
+					return
+				}
+				passed++
+
+				log.WithFields(f).Debugf("document uploaded for signature: %s", signatureID)
+
+			}(record)
+		}
+
+		wg.Wait()
+
+		log.WithFields(f).Debug("completed processing files")
+
+		log.WithFields(f).Debugf("total: %d, passed: %d, failed: %d", count, passed, failed)
+		return
+	}
+
 	iclaSignatures, err := signatureRepo.GetICLAByDate(ctx, startDate)
 	if err != nil {
 		log.WithFields(f).WithError(err).Warn("problem fetching ICLA signatures")
 		return
 	}
 
-	if dryRun != nil && *dryRun {
-		log.WithFields(f).Debug("dry-run mode enabled")
-	}
-
 	log.WithFields(f).Debugf("processing %d ICLA signatures", len(iclaSignatures))
 	toUpload := make([]signatures.ItemSignature, 0)
 
@@ -151,7 +232,7 @@ func main() {
 		semaphore <- struct{}{}
 		go func(sig signatures.ItemSignature) {
 			defer wg.Done()
-			
+
 			var documentID string
 
 			reportData := ReportData{

cla-backend-go/utils/s3.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"errors"
 	"io"
+	"os"
 	"strings"
 	"time"
 
@@ -25,6 +26,7 @@ const PresignedURLValidity = 15 * time.Minute
 // S3Storage provides methods to handle s3 storage
 type S3Storage interface {
 	Upload(fileContent []byte, projectID string, claType string, identifier string, signatureID string) error
+	UploadFile(file *os.File, projectID string, claType string, identifier string, signatureID string) error
 	Download(filename string) ([]byte, error)
 	Delete(filename string) error
 	GetPresignedURL(filename string) (string, error)
@@ -60,6 +62,16 @@ func (s3c *S3Client) Upload(fileContent []byte, projectID string, claType string
 	return err
 }
 
+func (s3c *S3Client) UploadFile(file *os.File, projectID string, claType string, identifier string, signatureID string) error {
+	filename := strings.Join([]string{"contract-group", projectID, claType, identifier, signatureID}, "/") + ".pdf"
+	_, err := s3c.s3.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(s3c.BucketName),
+		Key:    aws.String(filename),
+		Body:   file,
+	})
+	return err
+}
+
 // Download file from s3
 func (s3c *S3Client) Download(filename string) ([]byte, error) {
 	ou, err := s3c.s3.GetObject(&s3.GetObjectInput{
@@ -114,6 +126,17 @@ func UploadToS3(body []byte, projectID string, claType string, identifier string
 	return s3Storage.Upload(body, projectID, claType, identifier, signatureID)
 }
 
+// UploadFileToS3 uploads file to s3 storage at path contract-group/<project-ID>/<claType>/<identifier>/<signatureID>.pdf
+// claType should be cla or ccla
+// identifier can be user-id or company-id
+func UploadFileToS3(file *os.File, projectID string, claType string, identifier string, signatureID string) error {
+	if s3Storage == nil {
+		return errors.New("s3Storage not set")
+	}
+
+	return s3Storage.UploadFile(file, projectID, claType, identifier, signatureID)
+}
+
 func DocumentExists(key string) (bool, error) {
 	if s3Storage == nil {
 		return false, errors.New("s3 storage not set")