diff --git a/generated.go b/generated.go index 0e16e4b..b6d2943 100644 --- a/generated.go +++ b/generated.go @@ -85,6 +85,7 @@ var pairMap = map[string]string{ "work_dir": "string", } var ( + _ Direr = &Storage{} _ Storager = &Storage{} ) @@ -147,13 +148,14 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - Create []Pair - Delete []Pair - List []Pair - Metadata []Pair - Read []Pair - Stat []Pair - Write []Pair + Create []Pair + CreateDir []Pair + Delete []Pair + List []Pair + Metadata []Pair + Read []Pair + Stat []Pair + Write []Pair } // pairStorageCreate is the parsed struct @@ -188,6 +190,29 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) return result, nil } +// pairStorageCreateDir is the parsed struct +type pairStorageCreateDir struct { + pairs []Pair +} + +// parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := pairStorageCreateDir{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} + } + } + + // Check required pairs. + + return result, nil +} + // pairStorageDelete is the parsed struct type pairStorageDelete struct { pairs []Pair @@ -425,6 +450,31 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } +// CreateDir will create a new dir object. +// +// This function will create a context by default. +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} + +// CreateDirWithContext will create a new dir object. +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + + return s.createDir(ctx, path, opt) +} + // Delete will delete an object from service. // // ## Behavior diff --git a/go.sum b/go.sum index 5da2eeb..fd162c6 100644 --- a/go.sum +++ b/go.sum @@ -45,11 +45,9 @@ github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6U github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= github.com/beyondstorage/go-integration-test/v4 v4.1.1 h1:9bSXKbr6hLb4+ZsmAhWE32fvqhyrpub4U4qgBGeth4A= github.com/beyondstorage/go-integration-test/v4 v4.1.1/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc= -github.com/beyondstorage/go-storage/v4 v4.2.0 h1:J0xqqy4qEQRtIS2zUWMA5wRXVHx/cxX5fHsU2ezA3+I= github.com/beyondstorage/go-storage/v4 v4.2.0/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU= github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1 h1:5rloKVOavOAOgE7pfBMk8kxVjyjaedyw3opXyXg4Vqc= github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1/go.mod h1:0fdcRCzLKMQe7Ve4zPlyTGgoPYwuINiV79Gx9tCt9tQ= -github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259 h1:mW9XpHLc6pdXBRnsha1VlqF0rNsB/Oc+8l+5UYngmRA= github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -90,6 +88,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/iterator.go b/iterator.go new file mode 100644 index 0000000..7ae4cdb --- /dev/null +++ b/iterator.go @@ -0,0 +1,11 @@ +package gdrive + +type objectPageStatus struct { + limit uint32 + path string + pageToken string +} + +func (i *objectPageStatus) ContinuationToken() string { + return i.pageToken +} diff --git a/service.toml b/service.toml index 89d198a..4a89373 100644 --- a/service.toml +++ b/service.toml @@ -1,6 +1,7 @@ name = "gdrive" [namespace.storage] +implement = ["direr"] [namespace.storage.new] required = ["name","credential"] diff --git a/storage.go b/storage.go index 4ed14c3..fd0222a 100644 --- a/storage.go +++ b/storage.go @@ -2,35 +2,251 @@ package gdrive import ( "context" + "fmt" "io" + "path/filepath" + "strings" + "google.golang.org/api/drive/v3" + + "github.com/beyondstorage/go-storage/v4/pkg/iowrap" + "github.com/beyondstorage/go-storage/v4/services" . "github.com/beyondstorage/go-storage/v4/types" ) +const directoryMimeType = "application/vnd.google-apps.folder" + func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { - panic("not implemented") + o = s.newObject(false) + o.ID = s.getAbsPath(path) + o.Path = path + return o +} + +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + path = s.getAbsPath(path) + pathUnits := strings.Split(path, "/") + parentsId := "root" + for _, v := range pathUnits { + parentsId, err = s.mkDir(ctx, parentsId, v) + if err != nil { + return nil, err + } + } + + o = s.newObject(true) + o.ID = path + o.Path = path + o.Mode = ModeDir + + return o, nil + } func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { - panic("not implemented") + var fileId string + fileId, err = s.pathToId(ctx, path) + if err != nil { + return err + } + err = s.service.Files.Delete(fileId).Do() + if err != nil { + return err + } + return nil } func (s *Storage) list(ctx context.Context, path string, opt pairStorageList) (oi *ObjectIterator, err error) { - panic("not implemented") + input := &objectPageStatus{ + limit: 200, + path: s.getAbsPath(path), + } + + if !opt.HasListMode || opt.ListMode.IsDir() { + return NewObjectIterator(ctx, s.nextObjectPage, input), nil + } else { + return nil, services.ListModeInvalidError{Actual: opt.ListMode} + } } func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { - panic("not implemented") + meta = NewStorageMeta() + meta.Name = s.name + meta.WorkDir = s.workDir + return meta +} + +// Create a directory by passing it's name and the parents' fileId. +// It will return the fileId of the directory whether it exist or not. +// If error occurs, it will return an empty string and error. +func (s *Storage) mkDir(ctx context.Context, parents string, dirName string) (string, error) { + id, err := s.searchContentInDir(ctx, parents, dirName) + if err != nil { + return "", err + } + // Simply return the fileId if the directory already exist + if id != "" { + return id, nil + } + + // create a directory if not exist + dir := &drive.File{ + Name: dirName, + Parents: []string{parents}, + MimeType: directoryMimeType, + } + f, err := s.service.Files.Create(dir).Context(ctx).Do() + if err != nil { + return "", err + } + return f.Id, nil +} + +func (s *Storage) nextObjectPage(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + dirId, err := s.pathToId(ctx, input.path) + if err != nil { + return err + } + q := s.service.Files.List().Q(fmt.Sprintf("parents=%s", dirId)) + + if input.pageToken != "" { + q = q.PageToken(input.pageToken) + } + r, err := q.Do() + + if err != nil { + return err + } + for _, f := range r.Files { + o := s.newObject(true) + // There is no way to get the path of the file directly, we have to do this + o.Path = input.path + "/" + f.Name + switch f.MimeType { + case directoryMimeType: + o.Mode = ModeDir + default: + o.Mode = ModeRead + } + page.Data = append(page.Data, o) + } + + input.pageToken = r.NextPageToken + return nil +} + +// TODO: add cache support +// pathToId converts path to fileId, as we talked in RFC-14. +// Ref: https://github.com/beyondstorage/go-service-gdrive/blob/master/docs/rfcs/14-gdrive-for-go-storage-design.md +// Behavior: +// err represents the error handled in pathToId +// fileId represents the results: fileId empty means the path is not exist, otherwise it's the fileId of input path +func (s *Storage) pathToId(ctx context.Context, path string) (fileId string, err error) { + absPath := s.getAbsPath(path) + pathUnits := strings.Split(absPath, "/") + fileId = "root" + // Traverse the whole path, break the loop if we fails at one search + for _, v := range pathUnits { + fileId, err = s.searchContentInDir(ctx, fileId, v) + + if fileId == "" || err != nil { + break + } + } + + if err != nil { + return "", err + } + + return fileId, nil } func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairStorageRead) (n int64, err error) { - panic("not implemented") + fileId, err := s.pathToId(ctx, path) + if err != nil { + return 0, err + } + f, err := s.service.Files.Get(fileId).Context(ctx).Download() + if err != nil { + return 0, err + } + + if opt.HasIoCallback { + iowrap.CallbackReadCloser(f.Body, opt.IoCallback) + } + + return io.Copy(w, f.Body) +} + +// Search something in directory by passing it's name and the fileId of the folder. +// It will return the fileId of the content we want, and nil for sure. +// If nothing is found, we will return an empty string and nil. +// We will only return non nil if error occurs. +func (s *Storage) searchContentInDir(ctx context.Context, dirId string, contentName string) (fileId string, err error) { + searchArg := fmt.Sprintf("name = %s and parents = %s", contentName, dirId) + fileList, err := s.service.Files.List().Context(ctx).Q(searchArg).Do() + if err != nil { + return "", err + } + // Because we assume that the path is unique, so there would be only two results: One file matches or none + if len(fileList.Files) == 0 { + return "", nil + } + return fileList.Files[0].Id, nil + } func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { - panic("not implemented") + _, err = s.pathToId(ctx, path) + if err != nil { + return nil, err + } + rp := s.getAbsPath(path) + o = s.newObject(true) + o.ID = rp + o.Path = path + return o, nil } +// First we need make sure this file is not exist. +// If it is, then we upload it, or we will overwrite it. func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { - panic("not implemented") + r = io.LimitReader(r, size) + + if opt.HasIoCallback { + r = iowrap.CallbackReader(r, opt.IoCallback) + } + + fileId, err := s.pathToId(ctx, path) + + if err != nil { + // upload + dirs, fileName := filepath.Split(path) + + if dirs != "" { + _, err = s.createDir(ctx, dirs, pairStorageCreateDir{}) + if err != nil { + return 0, err + } + + } + + file := &drive.File{Name: fileName} + _, err = s.service.Files.Create(file).Context(ctx).Media(r).Do() + + if err != nil { + return 0, err + } + + } else { + // update + newFile := &drive.File{Name: s.getFileName(path)} + _, err = s.service.Files.Update(fileId, newFile).Context(ctx).Media(r).Do() + + if err != nil { + return 0, err + } + } + + return size, nil } diff --git a/utils.go b/utils.go index a942d66..3ad2c23 100644 --- a/utils.go +++ b/utils.go @@ -3,6 +3,7 @@ package gdrive import ( "context" "fmt" + "strings" ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/credential" @@ -22,6 +23,7 @@ type Storage struct { features StorageFeatures types.UnimplementedStorager + types.UnimplementedDirer } // String implements Storager.String @@ -118,3 +120,29 @@ func (s *Storage) formatError(op string, err error, path ...string) error { Path: path, } } + +func (s *Storage) newObject(done bool) *types.Object { + return types.NewObject(s, done) +} + +// getAbsPath will calculate object storage's abs path +func (s *Storage) getAbsPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return prefix + path +} + +// getRelPath will get object storage's rel path. +func (s *Storage) getRelPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return strings.TrimPrefix(path, prefix) +} + +// getFileName will get a file's name without path +func (s *Storage) getFileName(path string) string { + if strings.Contains(path, "/") { + tmp := strings.Split(path, "/") + return tmp[len(tmp)-1] + } else { + return path + } +}