From 07b7fc64e5e0272d91fb943692d8a7609e077827 Mon Sep 17 00:00:00 2001 From: Overbool Date: Thu, 13 Dec 2018 23:02:55 +0800 Subject: [PATCH] cmds/pin: use coreapi/pin License: MIT Signed-off-by: Overbool --- core/commands/pin.go | 125 +++++++++++++++++++------- core/coreapi/interface/options/pin.go | 36 +++++++- core/coreapi/interface/pin.go | 2 +- core/coreapi/pin.go | 32 ++++--- core/corerepo/pinning.go | 80 ----------------- 5 files changed, 148 insertions(+), 127 deletions(-) delete mode 100644 core/corerepo/pinning.go diff --git a/core/commands/pin.go b/core/commands/pin.go index 14d2e07838ab..fbe3b0b0b689 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -10,9 +10,9 @@ import ( core "github.com/ipfs/go-ipfs/core" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" e "github.com/ipfs/go-ipfs/core/commands/e" + coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" iface "github.com/ipfs/go-ipfs/core/coreapi/interface" options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - corerepo "github.com/ipfs/go-ipfs/core/corerepo" pin "github.com/ipfs/go-ipfs/pin" cmds "gx/ipfs/QmPdvMtgpnMuU68mWhGtzCxnddXJoV96tT9aPcNbQsqPaM/go-ipfs-cmds" @@ -39,7 +39,14 @@ var PinCmd = &cmds.Command{ } type PinOutput struct { - Pins []string + Hash string + Error string +} + +// PinUpdateOutput represents the pin update output +type PinUpdateOutput struct { + From string + To string } type AddPinOutput struct { @@ -88,24 +95,25 @@ var addPinCmd = &cmds.Command{ } if !showProgress { - added, err := corerepo.Pin(n, api, req.Context, req.Arguments, recursive) + added, err := pinAddMany(req.Context, api, req.Arguments, recursive) if err != nil { return err } - return cmds.EmitOnce(res, &AddPinOutput{Pins: cidsToStrings(added)}) + + return cmds.EmitOnce(res, &AddPinOutput{Pins: added}) } v := new(dag.ProgressTracker) ctx := v.DeriveContext(req.Context) type pinResult struct { - pins []cid.Cid + pins []string err error } ch := make(chan pinResult, 1) go func() { - added, err := corerepo.Pin(n, api, ctx, req.Arguments, recursive) + added, err := pinAddMany(req.Context, api, req.Arguments, recursive) ch <- pinResult{pins: added, err: err} }() @@ -124,7 +132,7 @@ var addPinCmd = &cmds.Command{ return err } } - return res.Emit(&AddPinOutput{Pins: cidsToStrings(val.pins)}) + return res.Emit(&AddPinOutput{Pins: val.pins}) case <-ticker.C: if err := res.Emit(&AddPinOutput{Progress: v.Value()}); err != nil { return err @@ -181,6 +189,28 @@ var addPinCmd = &cmds.Command{ }, } +func pinAddMany(ctx context.Context, api coreiface.CoreAPI, paths []string, recursive bool) ([]string, error) { + added := make([]string, len(paths)) + for i, b := range paths { + p, err := coreiface.ParsePath(b) + if err != nil { + return nil, err + } + + rp, err := api.ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + if err := api.Pin().Add(ctx, p, options.Pin.Recursive(recursive)); err != nil { + return nil, err + } + added[i] = rp.Cid().String() + } + + return added, nil +} + var rmPinCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Remove pinned objects from local storage.", @@ -198,11 +228,6 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) }, Type: PinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - n, err := cmdenv.GetNode(env) - if err != nil { - return err - } - api, err := cmdenv.GetApi(env) if err != nil { return err @@ -215,20 +240,62 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) return err } - removed, err := corerepo.Unpin(n, api, req.Context, req.Arguments, recursive) - if err != nil { - return err + for _, b := range req.Arguments { + p, err := coreiface.ParsePath(b) + if err != nil { + return err + } + + rp, err := api.ResolvePath(req.Context, p) + if err != nil { + return err + } + + if err := api.Pin().Rm(req.Context, rp, options.Pin.RmRecursive(recursive)); err != nil { + if err := res.Emit(&PinOutput{ + Hash: rp.Cid().String(), + Error: err.Error(), + }); err != nil { + return err + } + continue + } + + if err := res.Emit(&PinOutput{ + Hash: rp.Cid().String(), + }); err != nil { + return err + } } - return cmds.EmitOnce(res, &PinOutput{cidsToStrings(removed)}) + return nil }, - Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { - for _, k := range out.Pins { - fmt.Fprintf(w, "unpinned %s\n", k) + PostRun: cmds.PostRunMap{ + cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { + failed := false + for { + out, err := res.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + r := out.(*PinOutput) + if r.Hash == "" && r.Error != "" { + return fmt.Errorf("aborted: %s", r.Error) + } else if r.Error != "" { + failed = true + fmt.Fprintf(os.Stderr, "cannot unpin %s: %s\n", r.Hash, r.Error) + } else { + fmt.Fprintf(os.Stdout, "unpinned %s\n", r.Hash) + } + } + + if failed { + return fmt.Errorf("some hash not unpinned") } return nil - }), + }, }, } @@ -364,7 +431,7 @@ new pin and removing the old one. Options: []cmdkit.Option{ cmdkit.BoolOption(pinUnpinOptionName, "Remove the old pin.").WithDefault(true), }, - Type: PinOutput{}, + Type: PinUpdateOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env) if err != nil { @@ -388,11 +455,11 @@ new pin and removing the old one. return err } - return cmds.EmitOnce(res, &PinOutput{Pins: []string{from.String(), to.String()}}) + return cmds.EmitOnce(res, &PinUpdateOutput{From: from.String(), To: to.String()}) }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { - fmt.Fprintf(w, "updated %s to %s\n", out.Pins[0], out.Pins[1]) + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinUpdateOutput) error { + fmt.Fprintf(w, "updated %s to %s\n", out.From, out.To) return nil }), }, @@ -628,11 +695,3 @@ func (r PinVerifyRes) Format(out io.Writer) { } } } - -func cidsToStrings(cs []cid.Cid) []string { - out := make([]string, 0, len(cs)) - for _, c := range cs { - out = append(out, c.String()) - } - return out -} diff --git a/core/coreapi/interface/options/pin.go b/core/coreapi/interface/options/pin.go index 9d1107f927db..630b561de4c0 100644 --- a/core/coreapi/interface/options/pin.go +++ b/core/coreapi/interface/options/pin.go @@ -8,12 +8,23 @@ type PinLsSettings struct { Type string } +// PinRmSettings represents the settings of pin rm command +type PinRmSettings struct { + Recursive bool + Force bool +} + type PinUpdateSettings struct { Unpin bool } type PinAddOption func(*PinAddSettings) error -type PinLsOption func(settings *PinLsSettings) error + +// PinRmOption pin rm option func +type PinRmOption func(*PinRmSettings) error + +// PinLsOption pin ls option func +type PinLsOption func(*PinLsSettings) error type PinUpdateOption func(*PinUpdateSettings) error func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { @@ -31,6 +42,21 @@ func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { return options, nil } +// PinRmOptions pin rm options +func PinRmOptions(opts ...PinRmOption) (*PinRmSettings, error) { + options := &PinRmSettings{ + Recursive: true, + } + + for _, opt := range opts { + if err := opt(options); err != nil { + return nil, err + } + } + + return options, nil +} + func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { options := &PinLsSettings{ Type: "all", @@ -102,6 +128,14 @@ func (pinOpts) Recursive(recursive bool) PinAddOption { } } +// RmRecursive is an option for Pin.Rm +func (pinOpts) RmRecursive(recursive bool) PinRmOption { + return func(settings *PinRmSettings) error { + settings.Recursive = recursive + return nil + } +} + // Type is an option for Pin.Ls which allows to specify which pin types should // be returned // diff --git a/core/coreapi/interface/pin.go b/core/coreapi/interface/pin.go index 2e119cbeae8b..6e13def8f541 100644 --- a/core/coreapi/interface/pin.go +++ b/core/coreapi/interface/pin.go @@ -43,7 +43,7 @@ type PinAPI interface { Ls(context.Context, ...options.PinLsOption) ([]Pin, error) // Rm removes pin for object specified by the path - Rm(context.Context, Path) error + Rm(context.Context, Path, ...options.PinRmOption) error // Update changes one pin to another, skipping checks for matching paths in // the old tree diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index d5222fdf7af5..fff6388d0b88 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -6,32 +6,30 @@ import ( coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - corerepo "github.com/ipfs/go-ipfs/core/corerepo" bserv "gx/ipfs/QmPoh3SrQzFBWtdGK6qmHDV4EanKR6kYPj4DD3J2NLoEmZ/go-blockservice" - merkledag "gx/ipfs/QmdV35UHnL1FM52baPkeUo6u7Fxm2CRUkPTLRPxeF8a4Ap/go-merkledag" - cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" offline "gx/ipfs/QmYZwey1thDTynSrvd6qQkX24UpTka6TFhQ2v569UpoqxD/go-ipfs-exchange-offline" + merkledag "gx/ipfs/QmdV35UHnL1FM52baPkeUo6u7Fxm2CRUkPTLRPxeF8a4Ap/go-merkledag" ) type PinAPI CoreAPI func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.PinAddOption) error { - settings, err := caopts.PinAddOptions(opts...) + defer api.node.Blockstore.PinLock().Unlock() + + dagNode, err := api.core().ResolveNode(ctx, p) if err != nil { - return err + return fmt.Errorf("pin: %s", err) } - rp, err := api.core().ResolvePath(ctx, p) + settings, err := caopts.PinAddOptions(opts...) if err != nil { return err } - defer api.node.Blockstore.PinLock().Unlock() - - _, err = corerepo.Pin(api.node, api.core(), ctx, []string{rp.Cid().String()}, settings.Recursive) + err = api.node.Pinning.Pin(ctx, dagNode, settings.Recursive) if err != nil { - return err + return fmt.Errorf("pin: %s", err) } return api.node.Pinning.Flush() @@ -52,12 +50,22 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]coreif return api.pinLsAll(settings.Type, ctx) } -func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error { - _, err := corerepo.Unpin(api.node, api.core(), ctx, []string{p.String()}, true) +// Rm pin rm api +func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path, opts ...caopts.PinRmOption) error { + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return err + } + + settings, err := caopts.PinRmOptions(opts...) if err != nil { return err } + if err = api.node.Pinning.Unpin(ctx, rp.Cid(), settings.Recursive); err != nil { + return err + } + return api.node.Pinning.Flush() } diff --git a/core/corerepo/pinning.go b/core/corerepo/pinning.go deleted file mode 100644 index f39d392769be..000000000000 --- a/core/corerepo/pinning.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Package corerepo provides pinning and garbage collection for local -IPFS block services. - -IPFS nodes will keep local copies of any object that have either been -added or requested locally. Not all of these objects are worth -preserving forever though, so the node administrator can pin objects -they want to keep and unpin objects that they don't care about. - -Garbage collection sweeps iterate through the local block store -removing objects that aren't pinned, which frees storage space for new -objects. -*/ -package corerepo - -import ( - "context" - "fmt" - - "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - - "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" -) - -func Pin(n *core.IpfsNode, api iface.CoreAPI, ctx context.Context, paths []string, recursive bool) ([]cid.Cid, error) { - out := make([]cid.Cid, len(paths)) - - for i, fpath := range paths { - p, err := iface.ParsePath(fpath) - if err != nil { - return nil, err - } - - dagnode, err := api.ResolveNode(ctx, p) - if err != nil { - return nil, fmt.Errorf("pin: %s", err) - } - err = n.Pinning.Pin(ctx, dagnode, recursive) - if err != nil { - return nil, fmt.Errorf("pin: %s", err) - } - out[i] = dagnode.Cid() - } - - err := n.Pinning.Flush() - if err != nil { - return nil, err - } - - return out, nil -} - -func Unpin(n *core.IpfsNode, api iface.CoreAPI, ctx context.Context, paths []string, recursive bool) ([]cid.Cid, error) { - unpinned := make([]cid.Cid, len(paths)) - - for i, p := range paths { - p, err := iface.ParsePath(p) - if err != nil { - return nil, err - } - - k, err := api.ResolvePath(ctx, p) - if err != nil { - return nil, err - } - - err = n.Pinning.Unpin(ctx, k.Cid(), recursive) - if err != nil { - return nil, err - } - unpinned[i] = k.Cid() - } - - err := n.Pinning.Flush() - if err != nil { - return nil, err - } - return unpinned, nil -}