Implement data backend for Google Cloud Storage (gcs)

go.mod

@@ -3,7 +3,7 @@ module github.com/root-gg/plik
 go 1.14
 
 require (
-	cloud.google.com/go v0.37.5-0.20190415145036-430062523376 // indirect
+	cloud.google.com/go v0.37.5-0.20190415145036-430062523376
 	github.com/BurntSushi/toml v0.3.1
 	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc
 	github.com/camathieu/pb v1.0.29-0.20190403132434-889de99fc8d5
@@ -30,6 +30,7 @@ require (
 	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d
 	github.com/olekukonko/ts v0.0.0-20171002115256-78ecb04241c0
 	github.com/pilagod/gorm-cursor-paginator v1.2.0
+	github.com/root-gg/juliet v0.0.0-20160211163218-63378c6bd35e
 	github.com/root-gg/logger v0.0.0-20150501173826-5d9a47a35312
 	github.com/root-gg/utils v0.0.0-20201113182430-4cef83d8cdcf
 	github.com/spf13/cobra v0.0.6-0.20200106181057-89c7ffb5129b

go.sum

@@ -81,12 +81,13 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -212,10 +213,10 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/root-gg/juliet v0.0.0-20160211163218-63378c6bd35e h1:9VnDAruVLz9qIZjezsWlgBJdZPdSzNsy5grHGxzcARU=
+github.com/root-gg/juliet v0.0.0-20160211163218-63378c6bd35e/go.mod h1:M7vO2z1mlCBDXKBEg4CvsO6IJfZu0k+3udyUubfCH+c=
 github.com/root-gg/logger v0.0.0-20150501173826-5d9a47a35312 h1:gLYLMWhdoU02+VLbz4TUhsWZCMvNk3k92YZawb2QcxI=
 github.com/root-gg/logger v0.0.0-20150501173826-5d9a47a35312/go.mod h1:SZ5sf0POVtIvp+vNCGYwJp1/ZT6/nShOau6aVMGgtGU=
-github.com/root-gg/utils v0.0.0-20151025161626-38f45ede2ce2 h1:2IC5+HunAKLL0GxcXtqbsQxEOmDcFTf8gn34mxpTs20=
-github.com/root-gg/utils v0.0.0-20151025161626-38f45ede2ce2/go.mod h1:XF39UJcqum+uYclrw+wB0V1qg9vgglJ8gz8AMrVJ/AQ=
 github.com/root-gg/utils v0.0.0-20201113182430-4cef83d8cdcf h1:Qx8jlCUW4X3+W+k14GCr33IBh/jYt+kaV/AzZWrl4vk=
 github.com/root-gg/utils v0.0.0-20201113182430-4cef83d8cdcf/go.mod h1:XF39UJcqum+uYclrw+wB0V1qg9vgglJ8gz8AMrVJ/AQ=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
@@ -259,10 +260,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
 golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -281,9 +280,7 @@ golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -306,10 +303,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201020230747-6e5568b54d1a h1:e3IU37lwO4aq3uoRKINC7JikojFmE5gO7xhfxs8VC34=
@@ -354,7 +349,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gormigrate.v1 v1.6.0 h1:XpYM6RHQPmzwY7Uyu+t+xxMXc86JYFJn4nEc9HzQjsI=
 gopkg.in/gormigrate.v1 v1.6.0/go.mod h1:Lf00lQrHqfSYWiTtPcyQabsDdM6ejZaMgV0OU6JMSlw=
-gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
 gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

server/data/gcs/gcs.go

@@ -0,0 +1,156 @@
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+
+	"cloud.google.com/go/storage"
+	"github.com/root-gg/juliet"
+	"github.com/root-gg/plik/server/common"
+	"github.com/root-gg/plik/server/data"
+	"github.com/root-gg/utils"
+)
+
+// Ensure File Data Backend implements data.Backend interface
+var _ data.Backend = (*Backend)(nil)
+
+// Config describes configuration for Google Cloud Storage data backend
+type Config struct {
+	Bucket string
+	Folder string
+}
+
+// NewConfig instantiate a new default configuration
+// and override it with configuration passed as argument
+func NewConfig(params map[string]interface{}) (config *Config) {
+	config = new(Config)
+	utils.Assign(config, params)
+	return
+}
+
+// Backend object
+type Backend struct {
+	Config *Config
+	client *storage.Client
+	lock   *sync.RWMutex
+}
+
+// NewBackend instantiate a new File Data Backend
+// from configuration passed as argument
+func NewBackend(config *Config) (b *Backend) {
+	b = new(Backend)
+	b.Config = config
+	b.lock = new(sync.RWMutex)
+	return
+}
+
+// GetFile implementation for Google Cloud Storage Data Backend
+func (sb *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) {
+
+	// Get the GCS client
+	err = sb.initClient()
+	if err != nil {
+		err = fmt.Errorf("unable to get gcs client: %s", err)
+		return
+	}
+
+	// Get the object
+	return sb.client.Bucket(sb.Config.Bucket).Object(sb.getFileID(file.UploadID, file.ID)).NewReader(context.Background())
+}
+
+// AddFile implementation for Google Cloud Storage Data Backend
+func (sb *Backend) AddFile(file *common.File, fileReader io.Reader) (err error) {
+
+	// Get the GCS client
+	err = sb.initClient()
+	if err != nil {
+		err = fmt.Errorf("unable to get gcs client: %s", err)
+		return
+	}
+
+	// Get a writer
+	wc := sb.client.Bucket(sb.Config.Bucket).Object(sb.getFileID(file.UploadID, file.ID)).NewWriter(context.Background())
+	defer wc.Close()
+
+	_, err = io.Copy(wc, fileReader)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// RemoveFile implementation for Google Cloud Storage Data Backend
+func (sb *Backend) RemoveFile(file *common.File) (err error) {
+
+	// Get the GCS client
+	err = sb.initClient()
+	if err != nil {
+		err = fmt.Errorf("unable to get gcs client: %s", err)
+		return
+	}
+
+	// Delete the object
+	err = sb.client.Bucket(sb.Config.Bucket).Object(sb.getFileID(file.UploadID, file.ID)).Delete(context.Background())
+	if err != nil {
+		if err == storage.ErrObjectNotExist {
+			err = nil
+		}
+	}
+
+	return
+}
+
+// RemoveUpload implementation for Google Cloud Storage Data Backend
+// Iterates on each upload file and call RemoveFile
+func (sb *Backend) RemoveUpload(ctx *juliet.Context, upload *common.Upload) (err error) {
+
+	// Get the GCS client
+	err = sb.initClient()
+	if err != nil {
+		err = fmt.Errorf("unable to get gcs client: %s", err)
+		return
+	}
+
+	// Remove all files
+	for _, file := range upload.Files {
+		uuid := sb.getFileID(file.UploadID, file.ID)
+		err = sb.client.Bucket(sb.Config.Bucket).Object(uuid).Delete(context.Background())
+		if err != nil {
+			if err == storage.ErrObjectNotExist {
+				err = nil
+			} else {
+				return fmt.Errorf("unable to remove object %s : %s", uuid, err)
+			}
+		}
+	}
+
+	return
+}
+
+func (sb *Backend) getFileID(uploadID string, fileID string) string {
+	if sb.Config.Folder != "" {
+		return fmt.Sprintf("%s/%s.%s", sb.Config.Folder, uploadID, fileID)
+	}
+
+	return fmt.Sprintf("%s.%s", uploadID, fileID)
+}
+
+func (sb *Backend) initClient() (err error) {
+
+	// Lock
+	sb.lock.Lock()
+	defer sb.lock.Unlock()
+
+	// Get the GCS client
+	if sb.client == nil {
+		sb.client, err = storage.NewClient(context.Background())
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}

server/plikd.cfg

@@ -47,6 +47,13 @@ OvhApiEndpoint      = ""            # OVH api endpoint to use. Defaults to https
 #   [DataBackendConfig]
 #       Directory = "files"
 #
+#   Example using Google Cloud Storage :
+#
+#   DataBackend = "gcs"
+#   [DataBackendConfig]
+#       Bucket = "MyAwesomeBucket"
+#       Folder = "plik"
+#
 #   Example using OpenStack Swift :
 #
 #   DataBackend = "swift"

server/server/server.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/root-gg/plik/server/data/gcs"
 	"github.com/root-gg/plik/server/data/s3"
 
 	"github.com/gorilla/mux"
@@ -330,6 +331,8 @@ func NewDataBackend(impl string, params map[string]interface{}) (backend data.Ba
 		}
 	case "swift":
 		backend = swift.NewBackend(swift.NewConfig(params))
+	case "gcs":
+		backend = gcs.NewBackend(gcs.NewConfig(params))
 	case "testing":
 		backend = data_test.NewBackend()
 	default: