From a57c10de42e680f7c0527d80be2101f444147cd2 Mon Sep 17 00:00:00 2001 From: zu1k <42370281+zu1k@users.noreply.github.com> Date: Thu, 15 Jul 2021 10:36:18 +0800 Subject: [PATCH] Implement GSP-134, GSP-654 & Direr (#7) * fest: Implement GSP-134 & Direr * Implement GSP-654 * tests: Update go-integration-test to latest * deps: Update go-storage to latest * deps: Update go-storage to v4.3.0 * deps: Update go-integration-test to v4.2.0 --- generated.go | 207 ++++++++++++++++++++++++++++++++------------------- go.mod | 6 +- go.sum | 14 ++-- service.toml | 2 +- storage.go | 47 +++++++++++- utils.go | 1 + 6 files changed, 185 insertions(+), 92 deletions(-) diff --git a/generated.go b/generated.go index 8089976..6366ef1 100644 --- a/generated.go +++ b/generated.go @@ -6,14 +6,12 @@ import ( "io" "github.com/beyondstorage/go-storage/v4/pkg/credential" - "github.com/beyondstorage/go-storage/v4/pkg/endpoint" "github.com/beyondstorage/go-storage/v4/pkg/httpclient" "github.com/beyondstorage/go-storage/v4/services" . "github.com/beyondstorage/go-storage/v4/types" ) var _ credential.Provider -var _ endpoint.Value var _ Storager var _ services.ServiceError var _ httpclient.Options @@ -21,47 +19,6 @@ var _ httpclient.Options // Type is the type for ipfs const Type = "ipfs" -// ObjectMetadata stores service metadata for object. -// -// Deprecated: Use ObjectSystemMetadata instead. -type ObjectMetadata struct { - // Blocks the number of files in the directory or the number of blocks that make up the file - Blocks int - // CumulativeSize the size of the DAGNodes making up the file in Bytes, or the sum of the sizes of all files in the directory - CumulativeSize uint64 - // Hash the CID of the file or directory - Hash string - // Local whether the file`s dags is fully present locally - Local bool - // SizeLocal the cumulative size of the data present locally - SizeLocal uint64 - // WithLocality whether the locality information is present - WithLocality bool -} - -// GetObjectMetadata will get ObjectMetadata from Object. -// -// - This function should not be called by service implementer. -// - The returning ObjectMetadata is read only and should not be modified. -// -// Deprecated: Use GetObjectSystemMetadata instead. -func GetObjectMetadata(o *Object) ObjectMetadata { - om, ok := o.GetServiceMetadata() - if ok { - return om.(ObjectMetadata) - } - return ObjectMetadata{} -} - -// setObjectMetadata will set ObjectMetadata into Object. -// -// - This function should only be called once, please make sure all data has been written before set. -// -// Deprecated: Use setObjectSystemMetadata instead. -func setObjectMetadata(o *Object, om ObjectMetadata) { - o.SetServiceMetadata(om) -} - // ObjectSystemMetadata stores system metadata for object. type ObjectSystemMetadata struct { // Blocks the number of files in the directory or the number of blocks that make up the file @@ -163,37 +120,12 @@ var pairMap = map[string]string{ } var ( _ Copier = &Storage{} + _ Direr = &Storage{} _ Mover = &Storage{} _ Storager = &Storage{} ) type StorageFeatures struct { - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationAll bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationCopy bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationCreate bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationDelete bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationList bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationMetadata bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationMove bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationRead bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationStat bool - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - LooseOperationWrite bool - - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - VirtualOperationAll bool - - // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. - VirtualPairAll bool } // pairStorageNew is the parsed struct @@ -257,15 +189,16 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - Copy []Pair - Create []Pair - Delete []Pair - List []Pair - Metadata []Pair - Move []Pair - Read []Pair - Stat []Pair - Write []Pair + Copy []Pair + Create []Pair + CreateDir []Pair + Delete []Pair + List []Pair + Metadata []Pair + Move []Pair + Read []Pair + Stat []Pair + Write []Pair } // pairStorageCopy is the parsed struct @@ -323,6 +256,29 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) return result, nil } +// pairStorageCreateDir is the parsed struct +type pairStorageCreateDir struct { + pairs []Pair +} + +// parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := pairStorageCreateDir{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} + } + } + + // Check required pairs. + + return result, nil +} + // pairStorageDelete is the parsed struct type pairStorageDelete struct { pairs []Pair @@ -567,6 +523,17 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { // Copy will copy an Object or multiple object in the service. // +// ## Behavior +// +// - Copy only copy one and only one object. +// - Service DON'T NEED to support copy a non-empty directory or copy files recursively. +// - User NEED to implement copy a non-empty directory and copy recursively by themself. +// - Copy a file to a directory SHOULD return `ErrObjectModeInvalid`. +// - Copy SHOULD NOT return an error as dst object exists. +// - Service that has native support for `overwrite` doesn't NEED to check the dst object exists or not. +// - Service that doesn't have native support for `overwrite` SHOULD check and delete the dst object if exists. +// - A successful copy opration should be complete, which means the dst object's content and metadata should be the same as src object. +// // This function will create a context by default. func (s *Storage) Copy(src string, dst string, pairs ...Pair) (err error) { ctx := context.Background() @@ -574,6 +541,17 @@ func (s *Storage) Copy(src string, dst string, pairs ...Pair) (err error) { } // CopyWithContext will copy an Object or multiple object in the service. +// +// ## Behavior +// +// - Copy only copy one and only one object. +// - Service DON'T NEED to support copy a non-empty directory or copy files recursively. +// - User NEED to implement copy a non-empty directory and copy recursively by themself. +// - Copy a file to a directory SHOULD return `ErrObjectModeInvalid`. +// - Copy SHOULD NOT return an error as dst object exists. +// - Service that has native support for `overwrite` doesn't NEED to check the dst object exists or not. +// - Service that doesn't have native support for `overwrite` SHOULD check and delete the dst object if exists. +// - A successful copy opration should be complete, which means the dst object's content and metadata should be the same as src object. func (s *Storage) CopyWithContext(ctx context.Context, src string, dst string, pairs ...Pair) (err error) { defer func() { err = s.formatError("copy", err, src, dst) @@ -608,6 +586,31 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } +// CreateDir will create a new dir object. +// +// This function will create a context by default. +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} + +// CreateDirWithContext will create a new dir object. +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + + return s.createDir(ctx, path, opt) +} + // Delete will delete an object from service. // // ## Behavior @@ -655,6 +658,12 @@ func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...P // List will return list a specific path. // +// ## Behavior +// +// - Service SHOULD support default `ListMode`. +// - Service SHOULD implement `ListModeDir` without the check for `VirtualDir`. +// - Service DON'T NEED to `Stat` while in `List`. +// // This function will create a context by default. func (s *Storage) List(path string, pairs ...Pair) (oi *ObjectIterator, err error) { ctx := context.Background() @@ -662,6 +671,12 @@ func (s *Storage) List(path string, pairs ...Pair) (oi *ObjectIterator, err erro } // ListWithContext will return list a specific path. +// +// ## Behavior +// +// - Service SHOULD support default `ListMode`. +// - Service SHOULD implement `ListModeDir` without the check for `VirtualDir`. +// - Service DON'T NEED to `Stat` while in `List`. func (s *Storage) ListWithContext(ctx context.Context, path string, pairs ...Pair) (oi *ObjectIterator, err error) { defer func() { err = s.formatError("list", err, path) @@ -693,6 +708,17 @@ func (s *Storage) Metadata(pairs ...Pair) (meta *StorageMeta) { // Move will move an object in the service. // +// ## Behavior +// +// - Move only move one and only one object. +// - Service DON'T NEED to support move a non-empty directory. +// - User NEED to implement move a non-empty directory by themself. +// - Move a file to a directory SHOULD return `ErrObjectModeInvalid`. +// - Move SHOULD NOT return an error as dst object exists. +// - Service that has native support for `overwrite` doesn't NEED to check the dst object exists or not. +// - Service that doesn't have native support for `overwrite` SHOULD check and delete the dst object if exists. +// - A successful move operation SHOULD be complete, which means the dst object's content and metadata should be the same as src object. +// // This function will create a context by default. func (s *Storage) Move(src string, dst string, pairs ...Pair) (err error) { ctx := context.Background() @@ -700,6 +726,17 @@ func (s *Storage) Move(src string, dst string, pairs ...Pair) (err error) { } // MoveWithContext will move an object in the service. +// +// ## Behavior +// +// - Move only move one and only one object. +// - Service DON'T NEED to support move a non-empty directory. +// - User NEED to implement move a non-empty directory by themself. +// - Move a file to a directory SHOULD return `ErrObjectModeInvalid`. +// - Move SHOULD NOT return an error as dst object exists. +// - Service that has native support for `overwrite` doesn't NEED to check the dst object exists or not. +// - Service that doesn't have native support for `overwrite` SHOULD check and delete the dst object if exists. +// - A successful move operation SHOULD be complete, which means the dst object's content and metadata should be the same as src object. func (s *Storage) MoveWithContext(ctx context.Context, src string, dst string, pairs ...Pair) (err error) { defer func() { err = s.formatError("move", err, src, dst) @@ -780,6 +817,13 @@ func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pai // Write will write data into a file. // +// ## Behavior +// +// - Write SHOULD NOT return an error as the object exist. +// - Service that has native support for `overwrite` doesn't NEED to check the object exists or not. +// - Service that doesn't have native support for `overwrite` SHOULD check and delete the object if exists. +// - A successful write operation SHOULD be complete, which means the object's content and metadata should be the same as specified in write request. +// // This function will create a context by default. func (s *Storage) Write(path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { ctx := context.Background() @@ -787,6 +831,13 @@ func (s *Storage) Write(path string, r io.Reader, size int64, pairs ...Pair) (n } // WriteWithContext will write data into a file. +// +// ## Behavior +// +// - Write SHOULD NOT return an error as the object exist. +// - Service that has native support for `overwrite` doesn't NEED to check the object exists or not. +// - Service that doesn't have native support for `overwrite` SHOULD check and delete the object if exists. +// - A successful write operation SHOULD be complete, which means the object's content and metadata should be the same as specified in write request. func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { defer func() { err = s.formatError("write", err, path) diff --git a/go.mod b/go.mod index 5f1ec68..47654de 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.15 require ( github.com/beyondstorage/go-endpoint v1.0.1 - github.com/beyondstorage/go-integration-test/v4 v4.1.2-0.20210702060531-e08a973380df - github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210705112743-4787bc39d185 - github.com/google/uuid v1.2.0 + github.com/beyondstorage/go-integration-test/v4 v4.2.0 + github.com/beyondstorage/go-storage/v4 v4.3.0 + github.com/google/uuid v1.3.0 github.com/ipfs/go-ipfs-api v0.2.0 github.com/ipfs/go-ipfs-cmds v0.6.0 ) diff --git a/go.sum b/go.sum index 2b5f48c..3342619 100644 --- a/go.sum +++ b/go.sum @@ -7,13 +7,10 @@ github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuH github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/beyondstorage/go-endpoint v1.0.1 h1:F8x2dGLMu9je6g7zPbKoxCXDlug97K26SeCx7KEHgyg= github.com/beyondstorage/go-endpoint v1.0.1/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= -github.com/beyondstorage/go-integration-test/v4 v4.1.2-0.20210702060531-e08a973380df h1:q8hld94x99zAJ+eOhYUtORsJ3hoHnBT1cBzHBSUWBZc= -github.com/beyondstorage/go-integration-test/v4 v4.1.2-0.20210702060531-e08a973380df/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc= -github.com/beyondstorage/go-storage/v4 v4.2.0/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU= -github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210705112743-4787bc39d185 h1:n+A+jshEDlwwKVsyQutcMowfSHjvDdJOtlKjHBUSYIo= -github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210705112743-4787bc39d185/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU= -github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259 h1:mW9XpHLc6pdXBRnsha1VlqF0rNsB/Oc+8l+5UYngmRA= -github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w= +github.com/beyondstorage/go-integration-test/v4 v4.2.0 h1:h2+SLmlDqjfBg+NzVcDr6VCmcD7I2xG+mqMzDlaCG+0= +github.com/beyondstorage/go-integration-test/v4 v4.2.0/go.mod h1:jLyYWSGUjQRH7U1HdaLbXE5sxBgqrtK73q+Q7PGIuSs= +github.com/beyondstorage/go-storage/v4 v4.3.0 h1:4WvcxGcIWBiLFUICec9OxaFtLfQ1akDIRW1h6D6TfUE= +github.com/beyondstorage/go-storage/v4 v4.3.0/go.mod h1:0fdcRCzLKMQe7Ve4zPlyTGgoPYwuINiV79Gx9tCt9tQ= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -54,8 +51,9 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= diff --git a/service.toml b/service.toml index 016e326..0eba409 100644 --- a/service.toml +++ b/service.toml @@ -1,7 +1,7 @@ name = "ipfs" [namespace.storage] -implement = ["copier", "mover"] +implement = ["copier", "mover", "direr"] [namespace.storage.new] required = ["endpoint"] diff --git a/storage.go b/storage.go index e06fec5..94c220d 100644 --- a/storage.go +++ b/storage.go @@ -2,6 +2,7 @@ package ipfs import ( "context" + "errors" "io" ipfs "github.com/ipfs/go-ipfs-api" @@ -16,7 +17,23 @@ import ( // This means that if the `workDir` is `/ipfs/`, there is a high probability that an error will be returned // See https://github.com/beyondstorage/specs/pull/134#discussion_r663594807 for more details func (s *Storage) copy(ctx context.Context, src string, dst string, opt pairStorageCopy) (err error) { - return s.ipfs.FilesCp(ctx, s.getAbsPath(src), s.getAbsPath(dst)) + dst = s.getAbsPath(dst) + + stat, err := s.ipfs.FilesStat(ctx, dst) + if err == nil { + if stat.Type == "directory" { + return services.ErrObjectModeInvalid + } else { + err = s.ipfs.FilesRm(ctx, dst, true) + if err != nil { + return err + } + } + } else if !errors.Is(formatError(err), services.ErrObjectNotExist) { + return err + } + + return s.ipfs.FilesCp(ctx, s.getAbsPath(src), dst) } func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { @@ -33,6 +50,21 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { return o } +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + path = s.getAbsPath(path) + + err = s.ipfs.FilesMkdir(ctx, path, ipfs.FilesMkdir.Parents(true)) + if err != nil { + return nil, err + } + + o = NewObject(s, true) + o.ID = path + o.Path = path + o.Mode = ModeDir + return +} + // GSP-46: Idempotent Storager Delete Operation // ref: https://github.com/beyondstorage/specs/blob/master/rfcs/46-idempotent-delete.md func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { @@ -43,7 +75,7 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete func (s *Storage) list(ctx context.Context, path string, opt pairStorageList) (oi *ObjectIterator, err error) { rp := s.getAbsPath(path) - if opt.ListMode.IsDir() { + if !opt.HasListMode || opt.ListMode.IsDir() { nextFn := func(ctx context.Context, page *ObjectPage) error { dir, err := s.ipfs.FilesLs(ctx, rp, ipfs.FilesLs.Stat(true)) if err != nil { @@ -80,6 +112,17 @@ func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { } func (s *Storage) move(ctx context.Context, src string, dst string, opt pairStorageMove) (err error) { + dst = s.getAbsPath(dst) + + stat, err := s.ipfs.FilesStat(ctx, dst) + if err == nil { + if stat.Type == "directory" { + return services.ErrObjectModeInvalid + } + } else if !errors.Is(formatError(err), services.ErrObjectNotExist) { + return err + } + return s.ipfs.FilesMv(ctx, s.getAbsPath(src), s.getAbsPath(dst)) } diff --git a/utils.go b/utils.go index 39ae089..bfd6e0d 100644 --- a/utils.go +++ b/utils.go @@ -27,6 +27,7 @@ type Storage struct { types.UnimplementedStorager types.UnimplementedCopier types.UnimplementedMover + types.UnimplementedDirer } // String implements Storager.String