diff --git a/Rules.mk b/Rules.mk index b416393e37a2..5af58d8667e7 100644 --- a/Rules.mk +++ b/Rules.mk @@ -48,9 +48,6 @@ ifneq ($(filter coverage% clean distclean test/unit/gotest.junit.xml,$(MAKECMDGO include $(dir)/Rules.mk endif -dir := pin/internal/pb -include $(dir)/Rules.mk - # -------------------- # # universal rules # # -------------------- # diff --git a/blocks/blockstoreutil/remove.go b/blocks/blockstoreutil/remove.go index f4d93ea0de4e..0c036af2d8a3 100644 --- a/blocks/blockstoreutil/remove.go +++ b/blocks/blockstoreutil/remove.go @@ -6,10 +6,9 @@ import ( "fmt" "io" - "github.com/ipfs/go-ipfs/pin" - cid "github.com/ipfs/go-cid" bs "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-ipfs-pinner" ) // RemovedBlock is used to respresent the result of removing a block. diff --git a/core/commands/object/diff.go b/core/commands/object/diff.go index 25b98925d59f..02904476f3c8 100644 --- a/core/commands/object/diff.go +++ b/core/commands/object/diff.go @@ -4,11 +4,11 @@ import ( "fmt" "io" - cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" - "github.com/ipfs/go-ipfs/dagutils" - cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/go-merkledag/dagutils" path "github.com/ipfs/interface-go-ipfs-core/path" + + cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" ) const ( @@ -70,7 +70,7 @@ Example: out := make([]*dagutils.Change, len(changes)) for i, change := range changes { out[i] = &dagutils.Change{ - Type: change.Type, + Type: dagutils.ChangeType(change.Type), Path: change.Path, } diff --git a/core/commands/pin.go b/core/commands/pin.go index c6e5ca6b547a..ae54c79c031e 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -8,21 +8,21 @@ import ( "os" "time" - core "github.com/ipfs/go-ipfs/core" - cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" - e "github.com/ipfs/go-ipfs/core/commands/e" - pin "github.com/ipfs/go-ipfs/pin" - bserv "github.com/ipfs/go-blockservice" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" cmds "github.com/ipfs/go-ipfs-cmds" offline "github.com/ipfs/go-ipfs-exchange-offline" + pin "github.com/ipfs/go-ipfs-pinner" dag "github.com/ipfs/go-merkledag" verifcid "github.com/ipfs/go-verifcid" coreiface "github.com/ipfs/interface-go-ipfs-core" options "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + + core "github.com/ipfs/go-ipfs/core" + cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" + e "github.com/ipfs/go-ipfs/core/commands/e" ) var PinCmd = &cmds.Command{ diff --git a/core/core.go b/core/core.go index c8633f9757a9..1d1424ab4ec1 100644 --- a/core/core.go +++ b/core/core.go @@ -14,15 +14,7 @@ import ( "io" "github.com/ipfs/go-filestore" - "github.com/ipfs/go-ipfs/core/bootstrap" - "github.com/ipfs/go-ipfs/core/node" - "github.com/ipfs/go-ipfs/core/node/libp2p" - "github.com/ipfs/go-ipfs/fuse/mount" - "github.com/ipfs/go-ipfs/namesys" - ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher" - "github.com/ipfs/go-ipfs/p2p" - "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/repo" + "github.com/ipfs/go-ipfs-pinner" bserv "github.com/ipfs/go-blockservice" bstore "github.com/ipfs/go-ipfs-blockstore" @@ -47,6 +39,15 @@ import ( record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/p2p/discovery" p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic" + + "github.com/ipfs/go-ipfs/core/bootstrap" + "github.com/ipfs/go-ipfs/core/node" + "github.com/ipfs/go-ipfs/core/node/libp2p" + "github.com/ipfs/go-ipfs/fuse/mount" + "github.com/ipfs/go-ipfs/namesys" + ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher" + "github.com/ipfs/go-ipfs/p2p" + "github.com/ipfs/go-ipfs/repo" ) var log = logging.Logger("core") diff --git a/core/coreapi/block.go b/core/coreapi/block.go index f9f43bfc6f26..18d9fb763d6b 100644 --- a/core/coreapi/block.go +++ b/core/coreapi/block.go @@ -7,14 +7,14 @@ import ( "io" "io/ioutil" - util "github.com/ipfs/go-ipfs/blocks/blockstoreutil" - pin "github.com/ipfs/go-ipfs/pin" - blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" + pin "github.com/ipfs/go-ipfs-pinner" coreiface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" path "github.com/ipfs/interface-go-ipfs-core/path" + + util "github.com/ipfs/go-ipfs/blocks/blockstoreutil" ) type BlockAPI CoreAPI diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index 5f2201df624c..3235b7d39661 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -18,16 +18,11 @@ import ( "errors" "fmt" - "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/core/node" - "github.com/ipfs/go-ipfs/namesys" - "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/repo" - bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-exchange-interface" offlinexch "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipfs-provider" offlineroute "github.com/ipfs/go-ipfs-routing/offline" ipld "github.com/ipfs/go-ipld-format" @@ -42,6 +37,11 @@ import ( routing "github.com/libp2p/go-libp2p-core/routing" pubsub "github.com/libp2p/go-libp2p-pubsub" record "github.com/libp2p/go-libp2p-record" + + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/core/node" + "github.com/ipfs/go-ipfs/namesys" + "github.com/ipfs/go-ipfs/repo" ) var log = logging.Logger("core/coreapi") diff --git a/core/coreapi/dag.go b/core/coreapi/dag.go index d2ec4505cce0..ecc4a7408bb1 100644 --- a/core/coreapi/dag.go +++ b/core/coreapi/dag.go @@ -3,9 +3,8 @@ package coreapi import ( "context" - "github.com/ipfs/go-ipfs/pin" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-pinner" ipld "github.com/ipfs/go-ipld-format" ) diff --git a/core/coreapi/object.go b/core/coreapi/object.go index fd1e824cc59f..9369ffeeea2c 100644 --- a/core/coreapi/object.go +++ b/core/coreapi/object.go @@ -11,12 +11,11 @@ import ( "io" "io/ioutil" - "github.com/ipfs/go-ipfs/dagutils" - "github.com/ipfs/go-ipfs/pin" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-pinner" ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" ft "github.com/ipfs/go-unixfs" coreiface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -307,7 +306,7 @@ func (api *ObjectAPI) Diff(ctx context.Context, before ipath.Path, after ipath.P out := make([]coreiface.ObjectChange, len(changes)) for i, change := range changes { out[i] = coreiface.ObjectChange{ - Type: change.Type, + Type: coreiface.ChangeType(change.Type), Path: change.Path, } diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go index 46e3f2f4b691..e62d490c4b17 100644 --- a/core/corerepo/gc.go +++ b/core/corerepo/gc.go @@ -7,13 +7,13 @@ import ( "time" "github.com/ipfs/go-ipfs/core" - gc "github.com/ipfs/go-ipfs/pin/gc" - repo "github.com/ipfs/go-ipfs/repo" + "github.com/ipfs/go-ipfs/gc" + "github.com/ipfs/go-ipfs/repo" - humanize "github.com/dustin/go-humanize" - cid "github.com/ipfs/go-cid" + "github.com/dustin/go-humanize" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" - mfs "github.com/ipfs/go-mfs" + "github.com/ipfs/go-mfs" ) var log = logging.Logger("corerepo") diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 2bbec78647ae..6b44226cecd7 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -8,12 +8,11 @@ import ( gopath "path" "strconv" - "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-cid" bstore "github.com/ipfs/go-ipfs-blockstore" chunker "github.com/ipfs/go-ipfs-chunker" "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipfs-posinfo" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 296056410a7e..4d4fa0e94e06 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -12,13 +12,13 @@ import ( "time" "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/pin/gc" + "github.com/ipfs/go-ipfs/gc" "github.com/ipfs/go-ipfs/repo" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" - cid "github.com/ipfs/go-cid" - datastore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" config "github.com/ipfs/go-ipfs-config" diff --git a/core/node/core.go b/core/node/core.go index ddb42aff5c45..b0b2d0ab1784 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -4,10 +4,6 @@ import ( "context" "fmt" - "github.com/ipfs/go-ipfs/core/node/helpers" - "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/repo" - "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" "github.com/ipfs/go-blockservice" @@ -16,6 +12,7 @@ import ( "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-exchange-interface" "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" "github.com/ipfs/go-mfs" @@ -23,6 +20,9 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" "go.uber.org/fx" + + "github.com/ipfs/go-ipfs/core/node/helpers" + "github.com/ipfs/go-ipfs/repo" ) // BlockService creates new blockservice which provides an interface to fetch content-addressable blocks diff --git a/core/node/provider.go b/core/node/provider.go index 5390c3a9b70c..52d48036efc0 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -5,16 +5,16 @@ import ( "fmt" "time" - "github.com/ipfs/go-ipfs/core/node/helpers" - "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/repo" - + "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipfs-provider" q "github.com/ipfs/go-ipfs-provider/queue" "github.com/ipfs/go-ipfs-provider/simple" ipld "github.com/ipfs/go-ipld-format" "github.com/libp2p/go-libp2p-core/routing" "go.uber.org/fx" + + "github.com/ipfs/go-ipfs/core/node/helpers" + "github.com/ipfs/go-ipfs/repo" ) const kReprovideFrequency = time.Hour * 12 diff --git a/dagutils/diff.go b/dagutils/diff.go deleted file mode 100644 index a43756ffd9b6..000000000000 --- a/dagutils/diff.go +++ /dev/null @@ -1,211 +0,0 @@ -package dagutils - -import ( - "context" - "fmt" - "path" - - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - dag "github.com/ipfs/go-merkledag" - coreiface "github.com/ipfs/interface-go-ipfs-core" -) - -// These constants define the changes that can be applied to a DAG. -const ( - Add = iota - Remove - Mod -) - -// Change represents a change to a DAG and contains a reference to the old and -// new CIDs. -type Change struct { - Type coreiface.ChangeType - Path string - Before cid.Cid - After cid.Cid -} - -// String prints a human-friendly line about a change. -func (c *Change) String() string { - switch c.Type { - case Add: - return fmt.Sprintf("Added %s at %s", c.After.String(), c.Path) - case Remove: - return fmt.Sprintf("Removed %s from %s", c.Before.String(), c.Path) - case Mod: - return fmt.Sprintf("Changed %s to %s at %s", c.Before.String(), c.After.String(), c.Path) - default: - panic("nope") - } -} - -// ApplyChange applies the requested changes to the given node in the given dag. -func ApplyChange(ctx context.Context, ds ipld.DAGService, nd *dag.ProtoNode, cs []*Change) (*dag.ProtoNode, error) { - e := NewDagEditor(nd, ds) - for _, c := range cs { - switch c.Type { - case Add: - child, err := ds.Get(ctx, c.After) - if err != nil { - return nil, err - } - - childpb, ok := child.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil) - if err != nil { - return nil, err - } - - case Remove: - err := e.RmLink(ctx, c.Path) - if err != nil { - return nil, err - } - - case Mod: - err := e.RmLink(ctx, c.Path) - if err != nil { - return nil, err - } - child, err := ds.Get(ctx, c.After) - if err != nil { - return nil, err - } - - childpb, ok := child.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil) - if err != nil { - return nil, err - } - } - } - - return e.Finalize(ctx, ds) -} - -// Diff returns a set of changes that transform node 'a' into node 'b'. -// It only traverses links in the following cases: -// 1. two node's links number are greater than 0. -// 2. both of two nodes are ProtoNode. -// Otherwise, it compares the cid and emits a Mod change object. -func Diff(ctx context.Context, ds ipld.DAGService, a, b ipld.Node) ([]*Change, error) { - // Base case where both nodes are leaves, just compare - // their CIDs. - if len(a.Links()) == 0 && len(b.Links()) == 0 { - return getChange(a, b) - } - - var out []*Change - cleanA, okA := a.Copy().(*dag.ProtoNode) - cleanB, okB := b.Copy().(*dag.ProtoNode) - if !okA || !okB { - return getChange(a, b) - } - - // strip out unchanged stuff - for _, lnk := range a.Links() { - l, _, err := b.ResolveLink([]string{lnk.Name}) - if err == nil { - if l.Cid.Equals(lnk.Cid) { - // no change... ignore it - } else { - anode, err := lnk.GetNode(ctx, ds) - if err != nil { - return nil, err - } - - bnode, err := l.GetNode(ctx, ds) - if err != nil { - return nil, err - } - - sub, err := Diff(ctx, ds, anode, bnode) - if err != nil { - return nil, err - } - - for _, subc := range sub { - subc.Path = path.Join(lnk.Name, subc.Path) - out = append(out, subc) - } - } - _ = cleanA.RemoveNodeLink(l.Name) - _ = cleanB.RemoveNodeLink(l.Name) - } - } - - for _, lnk := range cleanA.Links() { - out = append(out, &Change{ - Type: Remove, - Path: lnk.Name, - Before: lnk.Cid, - }) - } - for _, lnk := range cleanB.Links() { - out = append(out, &Change{ - Type: Add, - Path: lnk.Name, - After: lnk.Cid, - }) - } - - return out, nil -} - -// Conflict represents two incompatible changes and is returned by MergeDiffs(). -type Conflict struct { - A *Change - B *Change -} - -// MergeDiffs takes two slice of changes and adds them to a single slice. -// When a Change from b happens to the same path of an existing change in a, -// a conflict is created and b is not added to the merged slice. -// A slice of Conflicts is returned and contains pointers to the -// Changes involved (which share the same path). -func MergeDiffs(a, b []*Change) ([]*Change, []Conflict) { - var out []*Change - var conflicts []Conflict - paths := make(map[string]*Change) - for _, c := range a { - paths[c.Path] = c - } - - for _, c := range b { - if ca, ok := paths[c.Path]; ok { - conflicts = append(conflicts, Conflict{ - A: ca, - B: c, - }) - } else { - out = append(out, c) - } - } - for _, c := range paths { - out = append(out, c) - } - return out, conflicts -} - -func getChange(a, b ipld.Node) ([]*Change, error) { - if a.Cid().Equals(b.Cid()) { - return []*Change{}, nil - } - return []*Change{ - { - Type: Mod, - Before: a.Cid(), - After: b.Cid(), - }, - }, nil -} diff --git a/dagutils/diffenum.go b/dagutils/diffenum.go deleted file mode 100644 index fdab772c8750..000000000000 --- a/dagutils/diffenum.go +++ /dev/null @@ -1,99 +0,0 @@ -package dagutils - -import ( - "context" - "fmt" - - mdag "github.com/ipfs/go-merkledag" - - cid "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" -) - -// DiffEnumerate fetches every object in the graph pointed to by 'to' that is -// not in 'from'. This can be used to more efficiently fetch a graph if you can -// guarantee you already have the entirety of 'from' -func DiffEnumerate(ctx context.Context, dserv ipld.NodeGetter, from, to cid.Cid) error { - fnd, err := dserv.Get(ctx, from) - if err != nil { - return fmt.Errorf("get %s: %s", from, err) - } - - tnd, err := dserv.Get(ctx, to) - if err != nil { - return fmt.Errorf("get %s: %s", to, err) - } - - diff := getLinkDiff(fnd, tnd) - - sset := cid.NewSet() - for _, c := range diff { - // Since we're already assuming we have everything in the 'from' graph, - // add all those cids to our 'already seen' set to avoid potentially - // enumerating them later - if c.bef.Defined() { - sset.Add(c.bef) - } - } - for _, c := range diff { - if !c.bef.Defined() { - if sset.Has(c.aft) { - continue - } - err := mdag.Walk(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit, mdag.Concurrent()) - if err != nil { - return err - } - } else { - err := DiffEnumerate(ctx, dserv, c.bef, c.aft) - if err != nil { - return err - } - } - } - - return nil -} - -// if both bef and aft are not nil, then that signifies bef was replaces with aft. -// if bef is nil and aft is not, that means aft was newly added -// if aft is nil and bef is not, that means bef was deleted -type diffpair struct { - bef, aft cid.Cid -} - -// getLinkDiff returns a changeset between nodes 'a' and 'b'. Currently does -// not log deletions as our usecase doesnt call for this. -func getLinkDiff(a, b ipld.Node) []diffpair { - ina := make(map[string]*ipld.Link) - inb := make(map[string]*ipld.Link) - var aonly []cid.Cid - for _, l := range b.Links() { - inb[l.Cid.KeyString()] = l - } - for _, l := range a.Links() { - var key = l.Cid.KeyString() - ina[key] = l - if inb[key] == nil { - aonly = append(aonly, l.Cid) - } - } - - var out []diffpair - var aindex int - - for _, l := range b.Links() { - if ina[l.Cid.KeyString()] != nil { - continue - } - - if aindex < len(aonly) { - out = append(out, diffpair{bef: aonly[aindex], aft: l.Cid}) - aindex++ - } else { - out = append(out, diffpair{aft: l.Cid}) - continue - } - } - return out -} diff --git a/dagutils/diffenum_test.go b/dagutils/diffenum_test.go deleted file mode 100644 index e8db27817a97..000000000000 --- a/dagutils/diffenum_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package dagutils - -import ( - "context" - "fmt" - "testing" - - dag "github.com/ipfs/go-merkledag" - mdtest "github.com/ipfs/go-merkledag/test" - - cid "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" -) - -func buildNode(name string, desc map[string]ndesc, out map[string]ipld.Node) ipld.Node { - this := desc[name] - nd := new(dag.ProtoNode) - nd.SetData([]byte(name)) - for k, v := range this { - child, ok := out[v] - if !ok { - child = buildNode(v, desc, out) - out[v] = child - } - - if err := nd.AddNodeLink(k, child); err != nil { - panic(err) - } - } - - return nd -} - -type ndesc map[string]string - -func mkGraph(desc map[string]ndesc) map[string]ipld.Node { - out := make(map[string]ipld.Node) - for name := range desc { - if _, ok := out[name]; ok { - continue - } - - out[name] = buildNode(name, desc, out) - } - return out -} - -var tg1 = map[string]ndesc{ - "a1": ndesc{ - "foo": "b", - }, - "b": ndesc{}, - "a2": ndesc{ - "foo": "b", - "bar": "c", - }, - "c": ndesc{}, -} - -var tg2 = map[string]ndesc{ - "a1": ndesc{ - "foo": "b", - }, - "b": ndesc{}, - "a2": ndesc{ - "foo": "b", - "bar": "c", - }, - "c": ndesc{"baz": "d"}, - "d": ndesc{}, -} - -var tg3 = map[string]ndesc{ - "a1": ndesc{ - "foo": "b", - "bar": "c", - }, - "b": ndesc{}, - "a2": ndesc{ - "foo": "b", - "bar": "d", - }, - "c": ndesc{}, - "d": ndesc{}, -} - -var tg4 = map[string]ndesc{ - "a1": ndesc{ - "key1": "b", - "key2": "c", - }, - "a2": ndesc{ - "key1": "b", - "key2": "d", - }, -} - -var tg5 = map[string]ndesc{ - "a1": ndesc{ - "key1": "a", - "key2": "b", - }, - "a2": ndesc{ - "key1": "c", - "key2": "d", - }, -} - -func TestNameMatching(t *testing.T) { - nds := mkGraph(tg4) - - diff := getLinkDiff(nds["a1"], nds["a2"]) - if len(diff) != 1 { - t.Fatal(fmt.Errorf("node diff didn't match by name")) - } -} - -func TestNameMatching2(t *testing.T) { - nds := mkGraph(tg5) - - diff := getLinkDiff(nds["a1"], nds["a2"]) - if len(diff) != 2 { - t.Fatal(fmt.Errorf("incorrect number of link diff elements")) - } - if !(diff[0].bef.Equals(nds["a1"].Links()[0].Cid) && diff[0].aft.Equals(nds["a2"].Links()[0].Cid)) { - t.Fatal(fmt.Errorf("node diff didn't match by name")) - } -} - -func TestDiffEnumBasic(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - nds := mkGraph(tg1) - - ds := mdtest.Mock() - lgds := &getLogger{ds: ds} - - for _, nd := range nds { - err := ds.Add(ctx, nd) - if err != nil { - t.Fatal(err) - } - } - - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) - if err != nil { - t.Fatal(err) - } - - err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()}) - if err != nil { - t.Fatal(err) - } -} - -type getLogger struct { - ds ipld.NodeGetter - log []cid.Cid -} - -func (gl *getLogger) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { - nd, err := gl.ds.Get(ctx, c) - if err != nil { - return nil, err - } - gl.log = append(gl.log, c) - return nd, nil -} - -func (gl *getLogger) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption { - outCh := make(chan *ipld.NodeOption, len(cids)) - nds := gl.ds.GetMany(ctx, cids) - for no := range nds { - if no.Err == nil { - gl.log = append(gl.log, no.Node.Cid()) - } - select { - case outCh <- no: - default: - panic("too many responses") - } - } - return nds -} - -func assertCidList(a, b []cid.Cid) error { - if len(a) != len(b) { - return fmt.Errorf("got different number of cids than expected") - } - for i, c := range a { - if !c.Equals(b[i]) { - return fmt.Errorf("expected %s, got %s", c, b[i]) - } - } - return nil -} - -func TestDiffEnumFail(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - nds := mkGraph(tg2) - - ds := mdtest.Mock() - lgds := &getLogger{ds: ds} - - for _, s := range []string{"a1", "a2", "b", "c"} { - err := ds.Add(ctx, nds[s]) - if err != nil { - t.Fatal(err) - } - } - - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) - if err != ipld.ErrNotFound { - t.Fatal("expected err not found") - } - - err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()}) - if err != nil { - t.Fatal(err) - } - -} - -func TestDiffEnumRecurse(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - nds := mkGraph(tg3) - - ds := mdtest.Mock() - lgds := &getLogger{ds: ds} - - for _, s := range []string{"a1", "a2", "b", "c", "d"} { - err := ds.Add(ctx, nds[s]) - if err != nil { - t.Fatal(err) - } - } - - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) - if err != nil { - t.Fatal(err) - } - - err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid(), nds["d"].Cid()}) - if err != nil { - t.Fatal(err) - } -} diff --git a/dagutils/utils.go b/dagutils/utils.go deleted file mode 100644 index 3a796a9c5986..000000000000 --- a/dagutils/utils.go +++ /dev/null @@ -1,234 +0,0 @@ -package dagutils - -import ( - "context" - "errors" - - bserv "github.com/ipfs/go-blockservice" - dag "github.com/ipfs/go-merkledag" - path "github.com/ipfs/go-path" - - ds "github.com/ipfs/go-datastore" - syncds "github.com/ipfs/go-datastore/sync" - bstore "github.com/ipfs/go-ipfs-blockstore" - offline "github.com/ipfs/go-ipfs-exchange-offline" - ipld "github.com/ipfs/go-ipld-format" -) - -// Editor represents a ProtoNode tree editor and provides methods to -// modify it. -type Editor struct { - root *dag.ProtoNode - - // tmp is a temporary in memory (for now) dagstore for all of the - // intermediary nodes to be stored in - tmp ipld.DAGService - - // src is the dagstore with *all* of the data on it, it is used to pull - // nodes from for modification (nil is a valid value) - src ipld.DAGService -} - -// NewMemoryDagService returns a new, thread-safe in-memory DAGService. -func NewMemoryDagService() ipld.DAGService { - // build mem-datastore for editor's intermediary nodes - bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) - bsrv := bserv.New(bs, offline.Exchange(bs)) - return dag.NewDAGService(bsrv) -} - -// NewDagEditor returns an ProtoNode editor. -// -// * root is the node to be modified -// * source is the dagstore to pull nodes from (optional) -func NewDagEditor(root *dag.ProtoNode, source ipld.DAGService) *Editor { - return &Editor{ - root: root, - tmp: NewMemoryDagService(), - src: source, - } -} - -// GetNode returns the a copy of the root node being edited. -func (e *Editor) GetNode() *dag.ProtoNode { - return e.root.Copy().(*dag.ProtoNode) -} - -// GetDagService returns the DAGService used by this editor. -func (e *Editor) GetDagService() ipld.DAGService { - return e.tmp -} - -func addLink(ctx context.Context, ds ipld.DAGService, root *dag.ProtoNode, childname string, childnd ipld.Node) (*dag.ProtoNode, error) { - if childname == "" { - return nil, errors.New("cannot create link with no name") - } - - // ensure that the node we are adding is in the dagservice - err := ds.Add(ctx, childnd) - if err != nil { - return nil, err - } - - _ = ds.Remove(ctx, root.Cid()) - - // ensure no link with that name already exists - _ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound - - if err := root.AddNodeLink(childname, childnd); err != nil { - return nil, err - } - - if err := ds.Add(ctx, root); err != nil { - return nil, err - } - return root, nil -} - -// InsertNodeAtPath inserts a new node in the tree and replaces the current root with the new one. -func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert ipld.Node, create func() *dag.ProtoNode) error { - splpath := path.SplitList(pth) - nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create) - if err != nil { - return err - } - e.root = nd - return nil -} - -func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert ipld.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) { - if len(path) == 1 { - return addLink(ctx, e.tmp, root, path[0], toinsert) - } - - nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0]) - if err != nil { - // if 'create' is true, we create directories on the way down as needed - if err == dag.ErrLinkNotFound && create != nil { - nd = create() - err = nil // no longer an error case - } else if err == ipld.ErrNotFound { - // try finding it in our source dagstore - nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0]) - } - - // if we receive an ErrNotFound, then our second 'GetLinkedNode' call - // also fails, we want to error out - if err != nil { - return nil, err - } - } - - ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create) - if err != nil { - return nil, err - } - - _ = e.tmp.Remove(ctx, root.Cid()) - - _ = root.RemoveNodeLink(path[0]) - err = root.AddNodeLink(path[0], ndprime) - if err != nil { - return nil, err - } - - err = e.tmp.Add(ctx, root) - if err != nil { - return nil, err - } - - return root, nil -} - -// RmLink removes the link with the given name and updates the root node of -// the editor. -func (e *Editor) RmLink(ctx context.Context, pth string) error { - splpath := path.SplitList(pth) - nd, err := e.rmLink(ctx, e.root, splpath) - if err != nil { - return err - } - e.root = nd - return nil -} - -func (e *Editor) rmLink(ctx context.Context, root *dag.ProtoNode, path []string) (*dag.ProtoNode, error) { - if len(path) == 1 { - // base case, remove node in question - err := root.RemoveNodeLink(path[0]) - if err != nil { - return nil, err - } - - err = e.tmp.Add(ctx, root) - if err != nil { - return nil, err - } - - return root, nil - } - - // search for node in both tmp dagstore and source dagstore - nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0]) - if err == ipld.ErrNotFound { - nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0]) - } - - if err != nil { - return nil, err - } - - nnode, err := e.rmLink(ctx, nd, path[1:]) - if err != nil { - return nil, err - } - - _ = e.tmp.Remove(ctx, root.Cid()) - - _ = root.RemoveNodeLink(path[0]) - err = root.AddNodeLink(path[0], nnode) - if err != nil { - return nil, err - } - - err = e.tmp.Add(ctx, root) - if err != nil { - return nil, err - } - - return root, nil -} - -// Finalize writes the new DAG to the given DAGService and returns the modified -// root node. -func (e *Editor) Finalize(ctx context.Context, ds ipld.DAGService) (*dag.ProtoNode, error) { - nd := e.GetNode() - err := copyDag(ctx, nd, e.tmp, ds) - return nd, err -} - -func copyDag(ctx context.Context, nd ipld.Node, from, to ipld.DAGService) error { - // TODO(#4609): make this batch. - err := to.Add(ctx, nd) - if err != nil { - return err - } - - for _, lnk := range nd.Links() { - child, err := lnk.GetNode(ctx, from) - if err != nil { - if err == ipld.ErrNotFound { - // not found means we didnt modify it, and it should - // already be in the target datastore - continue - } - return err - } - - err = copyDag(ctx, child, from, to) - if err != nil { - return err - } - } - return nil -} diff --git a/dagutils/utils_test.go b/dagutils/utils_test.go deleted file mode 100644 index c9c55286dfdd..000000000000 --- a/dagutils/utils_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package dagutils - -import ( - "context" - "testing" - - dag "github.com/ipfs/go-merkledag" - mdtest "github.com/ipfs/go-merkledag/test" - path "github.com/ipfs/go-path" - - cid "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" -) - -func TestAddLink(t *testing.T) { - ctx, context := context.WithCancel(context.Background()) - defer context() - - ds := mdtest.Mock() - fishnode := dag.NodeWithData([]byte("fishcakes!")) - - err := ds.Add(ctx, fishnode) - if err != nil { - t.Fatal(err) - } - - nd := new(dag.ProtoNode) - nnode, err := addLink(ctx, ds, nd, "fish", fishnode) - if err != nil { - t.Fatal(err) - } - - fnprime, err := nnode.GetLinkedNode(ctx, ds, "fish") - if err != nil { - t.Fatal(err) - } - - fnpkey := fnprime.Cid() - if !fnpkey.Equals(fishnode.Cid()) { - t.Fatal("wrong child node found!") - } -} - -func assertNodeAtPath(t *testing.T, ds ipld.DAGService, root *dag.ProtoNode, pth string, exp cid.Cid) { - parts := path.SplitList(pth) - cur := root - for _, e := range parts { - nxt, err := cur.GetLinkedProtoNode(context.Background(), ds, e) - if err != nil { - t.Fatal(err) - } - - cur = nxt - } - - curc := cur.Cid() - if !curc.Equals(exp) { - t.Fatal("node not as expected at end of path") - } -} - -func TestInsertNode(t *testing.T) { - root := new(dag.ProtoNode) - e := NewDagEditor(root, nil) - - testInsert(t, e, "a", "anodefortesting", false, "") - testInsert(t, e, "a/b", "data", false, "") - testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name") - testInsert(t, e, "a/b/c/d/e", "foo", true, "") - testInsert(t, e, "a/b/c/d/f", "baz", true, "") - testInsert(t, e, "a/b/c/d/f", "bar", true, "") - - testInsert(t, e, "", "bar", true, "cannot create link with no name") - testInsert(t, e, "////", "slashes", true, "cannot create link with no name") - - c := e.GetNode().Cid() - - if c.String() != "QmZ8yeT9uD6ouJPNAYt62XffYuXBT6b4mP4obRSE9cJrSt" { - t.Fatal("output was different than expected: ", c) - } -} - -func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) { - child := dag.NodeWithData([]byte(data)) - err := e.tmp.Add(context.Background(), child) - if err != nil { - t.Fatal(err) - } - - var c func() *dag.ProtoNode - if create { - c = func() *dag.ProtoNode { - return &dag.ProtoNode{} - } - } - - err = e.InsertNodeAtPath(context.Background(), path, child, c) - if experr != "" { - var got string - if err != nil { - got = err.Error() - } - if got != experr { - t.Fatalf("expected '%s' but got '%s'", experr, got) - } - return - } - - if err != nil { - t.Fatal(err, path, data, create, experr) - } - - assertNodeAtPath(t, e.tmp, e.root, path, child.Cid()) -} diff --git a/pin/gc/gc.go b/gc/gc.go similarity index 99% rename from pin/gc/gc.go rename to gc/gc.go index a8309aeac476..6b44771d4e96 100644 --- a/pin/gc/gc.go +++ b/gc/gc.go @@ -8,15 +8,14 @@ import ( "strings" bserv "github.com/ipfs/go-blockservice" - pin "github.com/ipfs/go-ipfs/pin" - dag "github.com/ipfs/go-merkledag" - cid "github.com/ipfs/go-cid" dstore "github.com/ipfs/go-datastore" bstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" + pin "github.com/ipfs/go-ipfs-pinner" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" + dag "github.com/ipfs/go-merkledag" "github.com/ipfs/go-verifcid" ) diff --git a/go.mod b/go.mod index c7ba2ed6d760..9e66a0dc76e6 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.4 + github.com/ipfs/go-ipfs-pinner v0.0.0 github.com/ipfs/go-ipfs-posinfo v0.0.1 github.com/ipfs/go-ipfs-provider v0.3.0 github.com/ipfs/go-ipfs-routing v0.1.0 @@ -44,7 +45,7 @@ require ( github.com/ipfs/go-ipld-git v0.0.2 github.com/ipfs/go-ipns v0.0.1 github.com/ipfs/go-log v0.0.1 - github.com/ipfs/go-merkledag v0.2.3 + github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.1 @@ -107,4 +108,8 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.28 ) +replace github.com/ipfs/go-merkledag => github.com/MichaelMure/go-merkledag v0.2.1-0.20191119160700-c20b9a52f504 + +replace github.com/ipfs/go-ipfs-pinner => github.com/MichaelMure/go-ipfs-pinner v0.0.0-20191120152055-f93a308647da + go 1.13 diff --git a/go.sum b/go.sum index 9620ee35623b..926b8fb08de2 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,10 @@ github.com/AndreasBriese/bbloom v0.0.0-20190823232136-616930265c33/go.mod h1:bOv github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/MichaelMure/go-ipfs-pinner v0.0.0-20191120152055-f93a308647da h1:VujEWo1PUA+EIMvYU26fNvqykLMBuyXkILKbue86bzg= +github.com/MichaelMure/go-ipfs-pinner v0.0.0-20191120152055-f93a308647da/go.mod h1:0G0RKJr9eZwNvcwdLNVymjCcFhWtX9aXVGOxCZglwMg= +github.com/MichaelMure/go-merkledag v0.2.1-0.20191119160700-c20b9a52f504 h1:2qPGrw2YNfWNaOTPHY/WiSQZUKgiGQWh34my2zSppzo= +github.com/MichaelMure/go-merkledag v0.2.1-0.20191119160700-c20b9a52f504/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= @@ -46,7 +50,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= @@ -74,7 +77,6 @@ github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= @@ -118,7 +120,6 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -130,7 +131,6 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -138,8 +138,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitswap v0.0.3/go.mod h1:jadAZYsP/tcRMl47ZhFxhaNuDQoXawT8iHMg+iFoQbg= -github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= @@ -148,8 +146,6 @@ github.com/ipfs/go-bitswap v0.1.9/go.mod h1:1RtsyyeRe5nZgsitHPzVhBgEzaeemy8s0Arb github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRMtYNGrlxZ8KuI= -github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc= @@ -208,14 +204,12 @@ github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjN github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE= github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1 h1:zgUotX8dcAB/w/HidJh1zzc1yFq6Vm8J7T2F4itj/RU= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-provider v0.3.0 h1:W3AO8YQVPK/9NFu1HRJxuMZftw6K+rWv1j3y5K5e06A= github.com/ipfs/go-ipfs-provider v0.3.0/go.mod h1:rcQBVqfblDQRk5LaCtf2uxuKxMJxvKmF5pLS0pO4au4= -github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs= github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRDI/HQQ= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= @@ -233,11 +227,6 @@ github.com/ipfs/go-ipns v0.0.1 h1:5vX0+ehF55YWxE8Pmf4eB8szcP+fh24AXnvCkOmSLCc= github.com/ipfs/go-ipns v0.0.1/go.mod h1:HOiAXgGiH0wCSwsFM1IKdOy6YGT4iZafcsUKni703/g= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= -github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= -github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= -github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= -github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= -github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s= @@ -247,7 +236,6 @@ github.com/ipfs/go-mfs v0.1.1/go.mod h1:nk17h6kCOrfaNqXvx2VJ3SqkJ0VYUzJFAsc+BA0s github.com/ipfs/go-path v0.0.3/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo= github.com/ipfs/go-path v0.0.7 h1:H06hKMquQ0aYtHiHryOMLpQC1qC3QwXwkahcEVD51Ho= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= -github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.1.1 h1:+gPjbI+V3NktXZOqJA1kzbms2pYmhjgQQal0MzZrOAY= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -261,7 +249,6 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/interface-go-ipfs-core v0.2.3 h1:E6uQ+1fJjkxJWlL9lAE72a5FWeyeeNL3GitLy8+jq3Y= github.com/ipfs/interface-go-ipfs-core v0.2.3/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= -github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -306,9 +293,6 @@ github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpz github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= -github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= -github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= -github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= @@ -318,26 +302,19 @@ github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atq github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= github.com/libp2p/go-flow-metrics v0.0.2 h1:U5TvqfoyR6GVRM+bC15Ux1ltar1kbj6Zw6xOVR02CZs= github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-libp2p v0.0.2/go.mod h1:Qu8bWqFXiocPloabFGUcVG4kk94fLvfC8mWTDdFC9wE= -github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= github.com/libp2p/go-libp2p v0.3.1/go.mod h1:e6bwxbdYH1HqWTz8faTChKGR0BjPc8p+6SyP8GTTR7Y= github.com/libp2p/go-libp2p v0.4.0 h1:nV2q3fdhL80OWtPyBrsoWKcw32qC4TbbR+iGjEOMRaU= github.com/libp2p/go-libp2p v0.4.0/go.mod h1:9EsEIf9p2UDuwtPd0DwJsAl0qXVxgAnuDGRvHbfATfI= -github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= -github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= github.com/libp2p/go-libp2p-autonat-svc v0.1.0 h1:28IM7iWMDclZeVkpiFQaWVANwXwE7zLlpbnS7yXxrfs= github.com/libp2p/go-libp2p-autonat-svc v0.1.0/go.mod h1:fqi8Obl/z3R4PFVLm8xFtZ6PBL9MlV/xumymRFkKq5A= -github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= -github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE= -github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.3 h1:WsMYYaA0PwdpgJSQu12EzPYf5ypkLSTgcOsWr7DYrgI= @@ -356,56 +333,36 @@ github.com/libp2p/go-libp2p-core v0.2.3/go.mod h1:GqhyQqyIAPsxFYXHMjfXgMv03lxsvM github.com/libp2p/go-libp2p-core v0.2.5 h1:iP1PIiIrlRrGbE1fYq2918yBc5NlCH3pFuIPSWU9hds= github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= -github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI= -github.com/libp2p/go-libp2p-discovery v0.0.5/go.mod h1:YtF20GUxjgoKZ4zmXj8j3Nb2TUSBHFlOCetzYdbZL5I= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= github.com/libp2p/go-libp2p-gostream v0.2.0 h1:YGLebbo8KfylSkuralCCyas/hVrgWjc+cfnLMCZWvEs= github.com/libp2p/go-libp2p-gostream v0.2.0/go.mod h1:nN/Aw00orrADXaXgNCeYjCtQrk6eT20PX/G8F12NW/s= -github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= -github.com/libp2p/go-libp2p-host v0.0.3/go.mod h1:Y/qPyA6C8j2coYyos1dfRm0I8+nvd4TGrDGt4tA7JR8= github.com/libp2p/go-libp2p-http v0.1.4 h1:eN7nWHBIk55nbYalryW9s2Q3WYbvVdluFp6+T6fLM5o= github.com/libp2p/go-libp2p-http v0.1.4/go.mod h1:Y6svvGxdx9lZCCoOiz6L3bg+sIdkkcQtsZsNn8xjCTk= -github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= -github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= -github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= -github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= github.com/libp2p/go-libp2p-kad-dht v0.2.1 h1:+pb1DCkV/6oNQjTZVXl+Y++eV0rnelx/L8y1t4J+Rnw= github.com/libp2p/go-libp2p-kad-dht v0.2.1/go.mod h1:k7ONOlup7HKzQ68dE6lSnp07cdxdkmnRa+6B4Fh9/w0= github.com/libp2p/go-libp2p-kbucket v0.2.1 h1:q9Jfwww9XnXc1K9dyYuARJxJvIvhgYVaQCuziO/dF3c= github.com/libp2p/go-libp2p-kbucket v0.2.1/go.mod h1:/Rtu8tqbJ4WQ2KTCOMJhggMukOLNLNPY1EtEWWLxUvc= -github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= -github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08= -github.com/libp2p/go-libp2p-mplex v0.1.1/go.mod h1:KUQWpGkCzfV7UIpi8SKsAVxyBgz1c9R5EvxgnwLsb/I= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= -github.com/libp2p/go-libp2p-nat v0.0.2/go.mod h1:QrjXQSD5Dj4IJOdEcjHRkWTSomyxRo6HnUkf/TfQpLQ= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= -github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= -github.com/libp2p/go-libp2p-net v0.0.2/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= -github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q= github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= -github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= -github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= -github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= -github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-pubsub v0.1.1 h1:phDnQvO3H3hAgaEEQi6yt3LILqIYVXaw05bxzezrEwQ= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= @@ -417,23 +374,17 @@ github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFA github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1 h1:ZJK2bHXYUBqObHX+rHLSNrM3M8fmJUlUHrodDPPATmY= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= -github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU= github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8= github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= -github.com/libp2p/go-libp2p-secio v0.0.1/go.mod h1:IdG6iQybdcYmbTzxp4J5dwtUEDTOvZrT0opIDVNPrJs= -github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= -github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs= -github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= -github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -441,28 +392,17 @@ github.com/libp2p/go-libp2p-testing v0.1.0 h1:WaFRj/t3HdMZGNZqnU2pS7pDRBmMeoDx7/ github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-tls v0.1.1 h1:tjW7njTM8JX8FbEvqr8/VSKBdZYZ7CtGtv3i6NiFf10= github.com/libp2p/go-libp2p-tls v0.1.1/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= -github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= -github.com/libp2p/go-libp2p-transport v0.0.4/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= -github.com/libp2p/go-libp2p-transport v0.0.5/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= -github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= -github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2kn/m1w6YXxcIAYJYeI90h6BGgUc= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= -github.com/libp2p/go-libp2p-yamux v0.1.2/go.mod h1:xUoV/RmYkg6BW/qGxA9XJyg+HzXFYkeXbnhjmnYzKp8= -github.com/libp2p/go-libp2p-yamux v0.1.3/go.mod h1:VGSQVrqkh6y4nm0189qqxMtvyBft44MOYYPpYKXiVt4= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= github.com/libp2p/go-libp2p-yamux v0.2.1 h1:Q3XYNiKCC2vIxrvUJL+Jg1kiyeEaIDNKLjgEjo3VQdI= github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= -github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= -github.com/libp2p/go-mplex v0.0.1/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= -github.com/libp2p/go-mplex v0.0.4/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= -github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= @@ -475,29 +415,20 @@ github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIk github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= -github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-socket-activation v0.0.1 h1:X6cbwufPOVAO4RufEDz6rEl9AG1ZKf+bKpyDCUIHdU4= github.com/libp2p/go-socket-activation v0.0.1/go.mod h1:oLhRjibR6AmNIazrF5xRMJTi/Fv1UEcbHCqiWqxBcUs= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= -github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ= -github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw= github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= -github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6mK0en8rlpA6BBTs= -github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19K427vCzQ+xHKH/o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= github.com/libp2p/go-tcp-transport v0.1.1 h1:yGlqURmqgNA2fvzjSgZNlHcsd/IulAnKM8Ncu+vlqnw= github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= -github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= -github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= -github.com/libp2p/go-ws-transport v0.0.5/go.mod h1:Qbl4BxPfXXhhd/o0wcrgoaItHqA9tnZjoFZnxykuaXU= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-ws-transport v0.1.2 h1:VnxQcLfSGtqupqPpBNu8fUiCv+IN1RJ2BcVqQEM+z8E= github.com/libp2p/go-ws-transport v0.1.2/go.mod h1:dsh2Ld8F+XNmzpkaAijmg5Is+e9l6/1tK/6VFOdN69Y= -github.com/libp2p/go-yamux v1.2.1/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -524,7 +455,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -576,8 +506,6 @@ github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcM github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= -github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-varint v0.0.1 h1:TR/0rdQtnNxuN2IhiB639xC3tWM4IUi7DkTBVTdGW/M= @@ -670,10 +598,6 @@ github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= -github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= -github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= -github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= -github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1 h1:ctS9Anw/KozviCCtK6VWMz5kPL9nbQzbQY4yfqlIV4M= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= @@ -687,7 +611,6 @@ github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0d github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -711,7 +634,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -726,7 +648,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -763,7 +684,6 @@ golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/namesys/publisher.go b/namesys/publisher.go index 17c4e1b3fc2d..f5e335cd3163 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -6,11 +6,10 @@ import ( "sync" "time" - pin "github.com/ipfs/go-ipfs/pin" - proto "github.com/gogo/protobuf/proto" ds "github.com/ipfs/go-datastore" dsquery "github.com/ipfs/go-datastore/query" + pin "github.com/ipfs/go-ipfs-pinner" ipns "github.com/ipfs/go-ipns" pb "github.com/ipfs/go-ipns/pb" path "github.com/ipfs/go-path" diff --git a/pin/internal/pb/Rules.mk b/pin/internal/pb/Rules.mk deleted file mode 100644 index 505f70e7541a..000000000000 --- a/pin/internal/pb/Rules.mk +++ /dev/null @@ -1,8 +0,0 @@ -include mk/header.mk - -PB_$(d) = $(wildcard $(d)/*.proto) -TGTS_$(d) = $(PB_$(d):.proto=.pb.go) - -#DEPS_GO += $(TGTS_$(d)) - -include mk/footer.mk diff --git a/pin/internal/pb/doc.go b/pin/internal/pb/doc.go deleted file mode 100644 index 95d4afe67183..000000000000 --- a/pin/internal/pb/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -package pb - -//go:generate protoc --gogo_out=. header.proto diff --git a/pin/internal/pb/header.pb.go b/pin/internal/pb/header.pb.go deleted file mode 100644 index 71196b263101..000000000000 --- a/pin/internal/pb/header.pb.go +++ /dev/null @@ -1,381 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: pin/internal/pb/header.proto - -package pb - -import ( - encoding_binary "encoding/binary" - fmt "fmt" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package - -type Set struct { - // 1 for now, library will refuse to handle entries with an unrecognized version. - Version uint32 `protobuf:"varint,1,opt,name=version" json:"version"` - // how many of the links are subtrees - Fanout uint32 `protobuf:"varint,2,opt,name=fanout" json:"fanout"` - // hash seed for subtree selection, a random number - Seed uint32 `protobuf:"fixed32,3,opt,name=seed" json:"seed"` -} - -func (m *Set) Reset() { *m = Set{} } -func (m *Set) String() string { return proto.CompactTextString(m) } -func (*Set) ProtoMessage() {} -func (*Set) Descriptor() ([]byte, []int) { - return fileDescriptor_cda303a5a3ed87e7, []int{0} -} -func (m *Set) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Set) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Set.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Set) XXX_Merge(src proto.Message) { - xxx_messageInfo_Set.Merge(m, src) -} -func (m *Set) XXX_Size() int { - return m.Size() -} -func (m *Set) XXX_DiscardUnknown() { - xxx_messageInfo_Set.DiscardUnknown(m) -} - -var xxx_messageInfo_Set proto.InternalMessageInfo - -func (m *Set) GetVersion() uint32 { - if m != nil { - return m.Version - } - return 0 -} - -func (m *Set) GetFanout() uint32 { - if m != nil { - return m.Fanout - } - return 0 -} - -func (m *Set) GetSeed() uint32 { - if m != nil { - return m.Seed - } - return 0 -} - -func init() { - proto.RegisterType((*Set)(nil), "ipfs.pin.Set") -} - -func init() { proto.RegisterFile("pin/internal/pb/header.proto", fileDescriptor_cda303a5a3ed87e7) } - -var fileDescriptor_cda303a5a3ed87e7 = []byte{ - // 162 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x29, 0xc8, 0xcc, 0xd3, - 0xcf, 0xcc, 0x2b, 0x49, 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x2f, 0x48, 0xd2, 0xcf, 0x48, 0x4d, 0x4c, - 0x49, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xc8, 0x2c, 0x48, 0x2b, 0xd6, 0x2b, - 0xc8, 0xcc, 0x53, 0x8a, 0xe5, 0x62, 0x0e, 0x4e, 0x2d, 0x11, 0x92, 0xe3, 0x62, 0x2f, 0x4b, 0x2d, - 0x2a, 0xce, 0xcc, 0xcf, 0x93, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x75, 0x62, 0x39, 0x71, 0x4f, 0x9e, - 0x21, 0x08, 0x26, 0x28, 0x24, 0xc3, 0xc5, 0x96, 0x96, 0x98, 0x97, 0x5f, 0x5a, 0x22, 0xc1, 0x84, - 0x24, 0x0d, 0x15, 0x13, 0x92, 0xe0, 0x62, 0x29, 0x4e, 0x4d, 0x4d, 0x91, 0x60, 0x56, 0x60, 0xd4, - 0x60, 0x87, 0xca, 0x81, 0x45, 0x9c, 0x64, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, - 0xc1, 0x23, 0x39, 0xc6, 0x09, 0x8f, 0xe5, 0x18, 0x2e, 0x3c, 0x96, 0x63, 0xb8, 0xf1, 0x58, 0x8e, - 0x21, 0x8a, 0xa9, 0x20, 0x09, 0x10, 0x00, 0x00, 0xff, 0xff, 0x20, 0x85, 0x2f, 0x24, 0xa5, 0x00, - 0x00, 0x00, -} - -func (m *Set) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Set) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - dAtA[i] = 0x8 - i++ - i = encodeVarintHeader(dAtA, i, uint64(m.Version)) - dAtA[i] = 0x10 - i++ - i = encodeVarintHeader(dAtA, i, uint64(m.Fanout)) - dAtA[i] = 0x1d - i++ - encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Seed)) - i += 4 - return i, nil -} - -func encodeVarintHeader(dAtA []byte, offset int, v uint64) int { - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return offset + 1 -} -func (m *Set) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - n += 1 + sovHeader(uint64(m.Version)) - n += 1 + sovHeader(uint64(m.Fanout)) - n += 5 - return n -} - -func sovHeader(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n -} -func sozHeader(x uint64) (n int) { - return sovHeader(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *Set) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowHeader - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Set: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Set: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) - } - m.Version = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowHeader - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Version |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Fanout", wireType) - } - m.Fanout = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowHeader - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Fanout |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 5 { - return fmt.Errorf("proto: wrong wireType = %d for field Seed", wireType) - } - m.Seed = 0 - if (iNdEx + 4) > l { - return io.ErrUnexpectedEOF - } - m.Seed = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:])) - iNdEx += 4 - default: - iNdEx = preIndex - skippy, err := skipHeader(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthHeader - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthHeader - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipHeader(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowHeader - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowHeader - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowHeader - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthHeader - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthHeader - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowHeader - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipHeader(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthHeader - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthHeader = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowHeader = fmt.Errorf("proto: integer overflow") -) diff --git a/pin/internal/pb/header.proto b/pin/internal/pb/header.proto deleted file mode 100644 index 36b32b36dd17..000000000000 --- a/pin/internal/pb/header.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto2"; - -package ipfs.pin; - -option go_package = "pb"; - -message Set { - // 1 for now, library will refuse to handle entries with an unrecognized version. - optional uint32 version = 1; - // how many of the links are subtrees - optional uint32 fanout = 2; - // hash seed for subtree selection, a random number - optional fixed32 seed = 3; -} diff --git a/pin/pin.go b/pin/pin.go deleted file mode 100644 index 15b2396b5529..000000000000 --- a/pin/pin.go +++ /dev/null @@ -1,634 +0,0 @@ -// Package pin implements structures and methods to keep track of -// which objects a user wants to keep stored locally. -package pin - -import ( - "context" - "fmt" - "os" - "sync" - "time" - - "github.com/ipfs/go-ipfs/dagutils" - mdag "github.com/ipfs/go-merkledag" - - cid "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - ipld "github.com/ipfs/go-ipld-format" - logging "github.com/ipfs/go-log" -) - -var log = logging.Logger("pin") - -var pinDatastoreKey = ds.NewKey("/local/pins") - -var emptyKey cid.Cid - -func init() { - e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") - if err != nil { - log.Error("failed to decode empty key constant") - os.Exit(1) - } - emptyKey = e -} - -const ( - linkRecursive = "recursive" - linkDirect = "direct" - linkIndirect = "indirect" - linkInternal = "internal" - linkNotPinned = "not pinned" - linkAny = "any" - linkAll = "all" -) - -// Mode allows to specify different types of pin (recursive, direct etc.). -// See the Pin Modes constants for a full list. -type Mode int - -// Pin Modes -const ( - // Recursive pins pin the target cids along with any reachable children. - Recursive Mode = iota - - // Direct pins pin just the target cid. - Direct - - // Indirect pins are cids who have some ancestor pinned recursively. - Indirect - - // Internal pins are cids used to keep the internal state of the pinner. - Internal - - // NotPinned - NotPinned - - // Any refers to any pinned cid - Any -) - -// ModeToString returns a human-readable name for the Mode. -func ModeToString(mode Mode) (string, bool) { - m := map[Mode]string{ - Recursive: linkRecursive, - Direct: linkDirect, - Indirect: linkIndirect, - Internal: linkInternal, - NotPinned: linkNotPinned, - Any: linkAny, - } - s, ok := m[mode] - return s, ok -} - -// StringToMode parses the result of ModeToString() back to a Mode. -// It returns a boolean which is set to false if the mode is unknown. -func StringToMode(s string) (Mode, bool) { - m := map[string]Mode{ - linkRecursive: Recursive, - linkDirect: Direct, - linkIndirect: Indirect, - linkInternal: Internal, - linkNotPinned: NotPinned, - linkAny: Any, - linkAll: Any, // "all" and "any" means the same thing - } - mode, ok := m[s] - return mode, ok -} - -// A Pinner provides the necessary methods to keep track of Nodes which are -// to be kept locally, according to a pin mode. In practice, a Pinner is in -// in charge of keeping the list of items from the local storage that should -// not be garbage-collected. -type Pinner interface { - // IsPinned returns whether or not the given cid is pinned - // and an explanation of why its pinned - IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) - - // IsPinnedWithType returns whether or not the given cid is pinned with the - // given pin type, as well as returning the type of pin its pinned with. - IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) - - // Pin the given node, optionally recursively. - Pin(ctx context.Context, node ipld.Node, recursive bool) error - - // Unpin the given cid. If recursive is true, removes either a recursive or - // a direct pin. If recursive is false, only removes a direct pin. - Unpin(ctx context.Context, cid cid.Cid, recursive bool) error - - // Update updates a recursive pin from one cid to another - // this is more efficient than simply pinning the new one and unpinning the - // old one - Update(ctx context.Context, from, to cid.Cid, unpin bool) error - - // Check if a set of keys are pinned, more efficient than - // calling IsPinned for each key - CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error) - - // PinWithMode is for manually editing the pin structure. Use with - // care! If used improperly, garbage collection may not be - // successful. - PinWithMode(cid.Cid, Mode) - - // RemovePinWithMode is for manually editing the pin structure. - // Use with care! If used improperly, garbage collection may not - // be successful. - RemovePinWithMode(cid.Cid, Mode) - - // Flush writes the pin state to the backing datastore - Flush(ctx context.Context) error - - // DirectKeys returns all directly pinned cids - DirectKeys(ctx context.Context) ([]cid.Cid, error) - - // DirectKeys returns all recursively pinned cids - RecursiveKeys(ctx context.Context) ([]cid.Cid, error) - - // InternalPins returns all cids kept pinned for the internal state of the - // pinner - InternalPins(ctx context.Context) ([]cid.Cid, error) -} - -// Pinned represents CID which has been pinned with a pinning strategy. -// The Via field allows to identify the pinning parent of this CID, in the -// case that the item is not pinned directly (but rather pinned recursively -// by some ascendant). -type Pinned struct { - Key cid.Cid - Mode Mode - Via cid.Cid -} - -// Pinned returns whether or not the given cid is pinned -func (p Pinned) Pinned() bool { - return p.Mode != NotPinned -} - -// String Returns pin status as string -func (p Pinned) String() string { - switch p.Mode { - case NotPinned: - return "not pinned" - case Indirect: - return fmt.Sprintf("pinned via %s", p.Via) - default: - modeStr, _ := ModeToString(p.Mode) - return fmt.Sprintf("pinned: %s", modeStr) - } -} - -// pinner implements the Pinner interface -type pinner struct { - lock sync.RWMutex - recursePin *cid.Set - directPin *cid.Set - - // Track the keys used for storing the pinning state, so gc does - // not delete them. - internalPin *cid.Set - dserv ipld.DAGService - internal ipld.DAGService // dagservice used to store internal objects - dstore ds.Datastore -} - -// NewPinner creates a new pinner using the given datastore as a backend -func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner { - - rcset := cid.NewSet() - dirset := cid.NewSet() - - return &pinner{ - recursePin: rcset, - directPin: dirset, - dserv: serv, - dstore: dstore, - internal: internal, - internalPin: cid.NewSet(), - } -} - -// Pin the given node, optionally recursive -func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { - err := p.dserv.Add(ctx, node) - if err != nil { - return err - } - - c := node.Cid() - - p.lock.Lock() - defer p.lock.Unlock() - - if recurse { - if p.recursePin.Has(c) { - return nil - } - - p.lock.Unlock() - // temporary unlock to fetch the entire graph - err := mdag.FetchGraph(ctx, c, p.dserv) - p.lock.Lock() - if err != nil { - return err - } - - if p.recursePin.Has(c) { - return nil - } - - if p.directPin.Has(c) { - p.directPin.Remove(c) - } - - p.recursePin.Add(c) - } else { - if p.recursePin.Has(c) { - return fmt.Errorf("%s already pinned recursively", c.String()) - } - - p.directPin.Add(c) - } - return nil -} - -// ErrNotPinned is returned when trying to unpin items which are not pinned. -var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") - -// Unpin a given key -func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { - p.lock.Lock() - defer p.lock.Unlock() - if p.recursePin.Has(c) { - if !recursive { - return fmt.Errorf("%s is pinned recursively", c) - } - p.recursePin.Remove(c) - return nil - } - if p.directPin.Has(c) { - p.directPin.Remove(c) - return nil - } - return ErrNotPinned -} - -func (p *pinner) isInternalPin(c cid.Cid) bool { - return p.internalPin.Has(c) -} - -// IsPinned returns whether or not the given key is pinned -// and an explanation of why its pinned -func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(ctx, c, Any) -} - -// IsPinnedWithType returns whether or not the given cid is pinned with the -// given pin type, as well as returning the type of pin its pinned with. -func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(ctx, c, mode) -} - -// isPinnedWithType is the implementation of IsPinnedWithType that does not lock. -// intended for use by other pinned methods that already take locks -func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) { - switch mode { - case Any, Direct, Indirect, Recursive, Internal: - default: - err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", - mode, Direct, Indirect, Recursive, Internal, Any) - return "", false, err - } - if (mode == Recursive || mode == Any) && p.recursePin.Has(c) { - return linkRecursive, true, nil - } - if mode == Recursive { - return "", false, nil - } - - if (mode == Direct || mode == Any) && p.directPin.Has(c) { - return linkDirect, true, nil - } - if mode == Direct { - return "", false, nil - } - - if (mode == Internal || mode == Any) && p.isInternalPin(c) { - return linkInternal, true, nil - } - if mode == Internal { - return "", false, nil - } - - // Default is Indirect - visitedSet := cid.NewSet() - for _, rc := range p.recursePin.Keys() { - has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) - if err != nil { - return "", false, err - } - if has { - return rc.String(), true, nil - } - } - return "", false, nil -} - -// CheckIfPinned Checks if a set of keys are pinned, more efficient than -// calling IsPinned for each key, returns the pinned status of cid(s) -func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error) { - p.lock.RLock() - defer p.lock.RUnlock() - pinned := make([]Pinned, 0, len(cids)) - toCheck := cid.NewSet() - - // First check for non-Indirect pins directly - for _, c := range cids { - if p.recursePin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Recursive}) - } else if p.directPin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Direct}) - } else if p.isInternalPin(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Internal}) - } else { - toCheck.Add(c) - } - } - - // Now walk all recursive pins to check for indirect pins - var checkChildren func(cid.Cid, cid.Cid) error - checkChildren = func(rk, parentKey cid.Cid) error { - links, err := ipld.GetLinks(ctx, p.dserv, parentKey) - if err != nil { - return err - } - for _, lnk := range links { - c := lnk.Cid - - if toCheck.Has(c) { - pinned = append(pinned, - Pinned{Key: c, Mode: Indirect, Via: rk}) - toCheck.Remove(c) - } - - err := checkChildren(rk, c) - if err != nil { - return err - } - - if toCheck.Len() == 0 { - return nil - } - } - return nil - } - - for _, rk := range p.recursePin.Keys() { - err := checkChildren(rk, rk) - if err != nil { - return nil, err - } - if toCheck.Len() == 0 { - break - } - } - - // Anything left in toCheck is not pinned - for _, k := range toCheck.Keys() { - pinned = append(pinned, Pinned{Key: k, Mode: NotPinned}) - } - - return pinned, nil -} - -// RemovePinWithMode is for manually editing the pin structure. -// Use with care! If used improperly, garbage collection may not -// be successful. -func (p *pinner) RemovePinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Direct: - p.directPin.Remove(c) - case Recursive: - p.recursePin.Remove(c) - default: - // programmer error, panic OK - panic("unrecognized pin type") - } -} - -func cidSetWithValues(cids []cid.Cid) *cid.Set { - out := cid.NewSet() - for _, c := range cids { - out.Add(c) - } - return out -} - -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error) { - p := new(pinner) - - rootKey, err := d.Get(pinDatastoreKey) - if err != nil { - return nil, fmt.Errorf("cannot load pin state: %v", err) - } - rootCid, err := cid.Cast(rootKey) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) - defer cancel() - - root, err := internal.Get(ctx, rootCid) - if err != nil { - return nil, fmt.Errorf("cannot find pinning root object: %v", err) - } - - rootpb, ok := root.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf - } - - internalset := cid.NewSet() - internalset.Add(rootCid) - recordInternal := internalset.Add - - { // load recursive set - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) - } - p.recursePin = cidSetWithValues(recurseKeys) - } - - { // load direct set - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) - } - p.directPin = cidSetWithValues(directKeys) - } - - p.internalPin = internalset - - // assign services - p.dserv = dserv - p.dstore = d - p.internal = internal - - return p, nil -} - -// DirectKeys returns a slice containing the directly pinned keys -func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.directPin.Keys(), nil -} - -// RecursiveKeys returns a slice containing the recursively pinned keys -func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.recursePin.Keys(), nil -} - -// Update updates a recursive pin from one cid to another -// this is more efficient than simply pinning the new one and unpinning the -// old one -func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { - if from == to { - // Nothing to do. Don't remove this check or we'll end up - // _removing_ the pin. - // - // See #6648 - return nil - } - - p.lock.Lock() - defer p.lock.Unlock() - - if !p.recursePin.Has(from) { - return fmt.Errorf("'from' cid was not recursively pinned already") - } - - err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) - if err != nil { - return err - } - - p.recursePin.Add(to) - if unpin { - p.recursePin.Remove(from) - } - return nil -} - -// Flush encodes and writes pinner keysets to the datastore -func (p *pinner) Flush(ctx context.Context) error { - p.lock.Lock() - defer p.lock.Unlock() - - internalset := cid.NewSet() - recordInternal := internalset.Add - - root := &mdag.ProtoNode{} - { - n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkDirect, n); err != nil { - return err - } - } - - { - n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkRecursive, n); err != nil { - return err - } - } - - // add the empty node, its referenced by the pin sets but never created - err := p.internal.Add(ctx, new(mdag.ProtoNode)) - if err != nil { - return err - } - - err = p.internal.Add(ctx, root) - if err != nil { - return err - } - - k := root.Cid() - - internalset.Add(k) - if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { - return fmt.Errorf("cannot store pin state: %v", err) - } - p.internalPin = internalset - return nil -} - -// InternalPins returns all cids kept pinned for the internal state of the -// pinner -func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { - p.lock.Lock() - defer p.lock.Unlock() - var out []cid.Cid - out = append(out, p.internalPin.Keys()...) - return out, nil -} - -// PinWithMode allows the user to have fine grained control over pin -// counts -func (p *pinner) PinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Recursive: - p.recursePin.Add(c) - case Direct: - p.directPin.Add(c) - } -} - -// hasChild recursively looks for a Cid among the children of a root Cid. -// The visit function can be used to shortcut already-visited branches. -func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { - links, err := ipld.GetLinks(ctx, ng, root) - if err != nil { - return false, err - } - for _, lnk := range links { - c := lnk.Cid - if lnk.Cid.Equals(child) { - return true, nil - } - if visit(c) { - has, err := hasChild(ctx, ng, c, child, visit) - if err != nil { - return false, err - } - - if has { - return has, nil - } - } - } - return false, nil -} diff --git a/pin/pin_test.go b/pin/pin_test.go deleted file mode 100644 index e477ac07fe02..000000000000 --- a/pin/pin_test.go +++ /dev/null @@ -1,416 +0,0 @@ -package pin - -import ( - "context" - "io" - "testing" - "time" - - bs "github.com/ipfs/go-blockservice" - mdag "github.com/ipfs/go-merkledag" - - cid "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" - offline "github.com/ipfs/go-ipfs-exchange-offline" - util "github.com/ipfs/go-ipfs-util" -) - -var rand = util.NewTimeSeededRand() - -func randNode() (*mdag.ProtoNode, cid.Cid) { - nd := new(mdag.ProtoNode) - nd.SetData(make([]byte, 32)) - _, err := io.ReadFull(rand, nd.Data()) - if err != nil { - panic(err) - } - k := nd.Cid() - return nd, k -} - -func assertPinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) { - _, pinned, err := p.IsPinned(context.Background(), c) - if err != nil { - t.Fatal(err) - } - - if !pinned { - t.Fatal(failmsg) - } -} - -func assertUnpinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) { - _, pinned, err := p.IsPinned(context.Background(), c) - if err != nil { - t.Fatal(err) - } - - if pinned { - t.Fatal(failmsg) - } -} - -func TestPinnerBasic(t *testing.T) { - ctx := context.Background() - - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - - // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) - - a, ak := randNode() - err := dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - - // Pin A{} - err = p.Pin(ctx, a, false) - if err != nil { - t.Fatal(err) - } - - assertPinned(t, p, ak, "Failed to find key") - - // create new node c, to be indirectly pinned through b - c, _ := randNode() - err = dserv.Add(ctx, c) - if err != nil { - t.Fatal(err) - } - ck := c.Cid() - - // Create new node b, to be parent to a and c - b, _ := randNode() - err = b.AddNodeLink("child", a) - if err != nil { - t.Fatal(err) - } - - err = b.AddNodeLink("otherchild", c) - if err != nil { - t.Fatal(err) - } - - err = dserv.Add(ctx, b) - if err != nil { - t.Fatal(err) - } - bk := b.Cid() - - // recursively pin B{A,C} - err = p.Pin(ctx, b, true) - if err != nil { - t.Fatal(err) - } - - assertPinned(t, p, ck, "child of recursively pinned node not found") - - assertPinned(t, p, bk, "Recursively pinned node not found..") - - d, _ := randNode() - _ = d.AddNodeLink("a", a) - _ = d.AddNodeLink("c", c) - - e, _ := randNode() - _ = d.AddNodeLink("e", e) - - // Must be in dagserv for unpin to work - err = dserv.Add(ctx, e) - if err != nil { - t.Fatal(err) - } - err = dserv.Add(ctx, d) - if err != nil { - t.Fatal(err) - } - - // Add D{A,C,E} - err = p.Pin(ctx, d, true) - if err != nil { - t.Fatal(err) - } - - dk := d.Cid() - assertPinned(t, p, dk, "pinned node not found.") - - // Test recursive unpin - err = p.Unpin(ctx, dk, true) - if err != nil { - t.Fatal(err) - } - - err = p.Flush(ctx) - if err != nil { - t.Fatal(err) - } - - np, err := LoadPinner(dstore, dserv, dserv) - if err != nil { - t.Fatal(err) - } - - // Test directly pinned - assertPinned(t, np, ak, "Could not find pinned node!") - - // Test recursively pinned - assertPinned(t, np, bk, "could not find recursively pinned node") -} - -func TestIsPinnedLookup(t *testing.T) { - // We are going to test that lookups work in pins which share - // the same branches. For that we will construct this tree: - // - // A5->A4->A3->A2->A1->A0 - // / / - // B------- / - // \ / - // C--------------- - // - // We will ensure that IsPinned works for all objects both when they - // are pinned and once they have been unpinned. - aBranchLen := 6 - if aBranchLen < 3 { - t.Fatal("set aBranchLen to at least 3") - } - - ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - - // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) - - aNodes := make([]*mdag.ProtoNode, aBranchLen) - aKeys := make([]cid.Cid, aBranchLen) - for i := 0; i < aBranchLen; i++ { - a, _ := randNode() - if i >= 1 { - err := a.AddNodeLink("child", aNodes[i-1]) - if err != nil { - t.Fatal(err) - } - } - - err := dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - //t.Logf("a[%d] is %s", i, ak) - aNodes[i] = a - aKeys[i] = a.Cid() - } - - // Pin A5 recursively - if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { - t.Fatal(err) - } - - // Create node B and add A3 as child - b, _ := randNode() - if err := b.AddNodeLink("mychild", aNodes[3]); err != nil { - t.Fatal(err) - } - - // Create C node - c, _ := randNode() - // Add A0 as child of C - if err := c.AddNodeLink("child", aNodes[0]); err != nil { - t.Fatal(err) - } - - // Add C - err := dserv.Add(ctx, c) - if err != nil { - t.Fatal(err) - } - ck := c.Cid() - //t.Logf("C is %s", ck) - - // Add C to B and Add B - if err := b.AddNodeLink("myotherchild", c); err != nil { - t.Fatal(err) - } - err = dserv.Add(ctx, b) - if err != nil { - t.Fatal(err) - } - bk := b.Cid() - //t.Logf("B is %s", bk) - - // Pin C recursively - - if err := p.Pin(ctx, c, true); err != nil { - t.Fatal(err) - } - - // Pin B recursively - - if err := p.Pin(ctx, b, true); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, aKeys[0], "A0 should be pinned") - assertPinned(t, p, aKeys[1], "A1 should be pinned") - assertPinned(t, p, ck, "C should be pinned") - assertPinned(t, p, bk, "B should be pinned") - - // Unpin A5 recursively - if err := p.Unpin(ctx, aKeys[5], true); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, aKeys[0], "A0 should still be pinned through B") - assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") - - // Unpin B recursively - if err := p.Unpin(ctx, bk, true); err != nil { - t.Fatal(err) - } - assertUnpinned(t, p, bk, "B should be unpinned") - assertUnpinned(t, p, aKeys[1], "A1 should be unpinned") - assertPinned(t, p, aKeys[0], "A0 should still be pinned through C") -} - -func TestDuplicateSemantics(t *testing.T) { - ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - - // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) - - a, _ := randNode() - err := dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - - // pin is recursively - err = p.Pin(ctx, a, true) - if err != nil { - t.Fatal(err) - } - - // pinning directly should fail - err = p.Pin(ctx, a, false) - if err == nil { - t.Fatal("expected direct pin to fail") - } - - // pinning recursively again should succeed - err = p.Pin(ctx, a, true) - if err != nil { - t.Fatal(err) - } -} - -func TestFlush(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) - _, k := randNode() - - p.PinWithMode(k, Recursive) - if err := p.Flush(context.Background()); err != nil { - t.Fatal(err) - } - assertPinned(t, p, k, "expected key to still be pinned") -} - -func TestPinRecursiveFail(t *testing.T) { - ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - dserv := mdag.NewDAGService(bserv) - - p := NewPinner(dstore, dserv, dserv) - - a, _ := randNode() - b, _ := randNode() - err := a.AddNodeLink("child", b) - if err != nil { - t.Fatal(err) - } - - // NOTE: This isnt a time based test, we expect the pin to fail - mctx, cancel := context.WithTimeout(ctx, time.Millisecond) - defer cancel() - - err = p.Pin(mctx, a, true) - if err == nil { - t.Fatal("should have failed to pin here") - } - - err = dserv.Add(ctx, b) - if err != nil { - t.Fatal(err) - } - - err = dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - - // this one is time based... but shouldnt cause any issues - mctx, cancel = context.WithTimeout(ctx, time.Second) - defer cancel() - err = p.Pin(mctx, a, true) - if err != nil { - t.Fatal(err) - } -} - -func TestPinUpdate(t *testing.T) { - ctx := context.Background() - - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) - n1, c1 := randNode() - n2, c2 := randNode() - - if err := dserv.Add(ctx, n1); err != nil { - t.Fatal(err) - } - if err := dserv.Add(ctx, n2); err != nil { - t.Fatal(err) - } - - if err := p.Pin(ctx, n1, true); err != nil { - t.Fatal(err) - } - - if err := p.Update(ctx, c1, c2, true); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, c2, "c2 should be pinned now") - assertUnpinned(t, p, c1, "c1 should no longer be pinned") - - if err := p.Update(ctx, c2, c1, false); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, c2, "c2 should be pinned still") - assertPinned(t, p, c1, "c1 should be pinned now") -} diff --git a/pin/set.go b/pin/set.go deleted file mode 100644 index b050c31c43bd..000000000000 --- a/pin/set.go +++ /dev/null @@ -1,297 +0,0 @@ -package pin - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "hash/fnv" - "sort" - - "github.com/ipfs/go-ipfs/pin/internal/pb" - "github.com/ipfs/go-merkledag" - - "github.com/gogo/protobuf/proto" - cid "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" -) - -const ( - // defaultFanout specifies the default number of fan-out links per layer - defaultFanout = 256 - - // maxItems is the maximum number of items that will fit in a single bucket - maxItems = 8192 -) - -func hash(seed uint32, c cid.Cid) uint32 { - var buf [4]byte - binary.LittleEndian.PutUint32(buf[:], seed) - h := fnv.New32a() - _, _ = h.Write(buf[:]) - _, _ = h.Write(c.Bytes()) - return h.Sum32() -} - -type itemIterator func() (c cid.Cid, ok bool) - -type keyObserver func(cid.Cid) - -type sortByHash struct { - links []*ipld.Link -} - -func (s sortByHash) Len() int { - return len(s.links) -} - -func (s sortByHash) Less(a, b int) bool { - return bytes.Compare(s.links[a].Cid.Bytes(), s.links[b].Cid.Bytes()) == -1 -} - -func (s sortByHash) Swap(a, b int) { - s.links[a], s.links[b] = s.links[b], s.links[a] -} - -func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, depth uint32, iter itemIterator, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - links := make([]*ipld.Link, 0, defaultFanout+maxItems) - for i := 0; i < defaultFanout; i++ { - links = append(links, &ipld.Link{Cid: emptyKey}) - } - - // add emptyKey to our set of internal pinset objects - n := &merkledag.ProtoNode{} - n.SetLinks(links) - - internalKeys(emptyKey) - - hdr := &pb.Set{ - Version: 1, - Fanout: defaultFanout, - Seed: depth, - } - if err := writeHdr(n, hdr); err != nil { - return nil, err - } - - if estimatedLen < maxItems { - // it'll probably fit - links := n.Links() - for i := 0; i < maxItems; i++ { - k, ok := iter() - if !ok { - // all done - break - } - - links = append(links, &ipld.Link{Cid: k}) - } - - n.SetLinks(links) - - // sort by hash, also swap item Data - s := sortByHash{ - links: n.Links()[defaultFanout:], - } - sort.Stable(s) - } - - hashed := make([][]cid.Cid, defaultFanout) - for { - // This loop essentially enumerates every single item in the set - // and maps them all into a set of buckets. Each bucket will be recursively - // turned into its own sub-set, and so on down the chain. Each sub-set - // gets added to the dagservice, and put into its place in a set nodes - // links array. - // - // Previously, the bucket was selected by taking an int32 from the hash of - // the input key + seed. This was erroneous as we would later be assigning - // the created sub-sets into an array of length 256 by the modulus of the - // int32 hash value with 256. This resulted in overwriting existing sub-sets - // and losing pins. The fix (a few lines down from this comment), is to - // map the hash value down to the 8 bit keyspace here while creating the - // buckets. This way, we avoid any overlapping later on. - k, ok := iter() - if !ok { - break - } - h := hash(depth, k) % defaultFanout - hashed[h] = append(hashed[h], k) - } - - for h, items := range hashed { - if len(items) == 0 { - // recursion base case - continue - } - - childIter := getCidListIterator(items) - - // recursively create a pinset from the items for this bucket index - child, err := storeItems(ctx, dag, uint64(len(items)), depth+1, childIter, internalKeys) - if err != nil { - return nil, err - } - - size, err := child.Size() - if err != nil { - return nil, err - } - - err = dag.Add(ctx, child) - if err != nil { - return nil, err - } - childKey := child.Cid() - - internalKeys(childKey) - - // overwrite the 'empty key' in the existing links array - n.Links()[h] = &ipld.Link{ - Cid: childKey, - Size: size, - } - } - return n, nil -} - -func readHdr(n *merkledag.ProtoNode) (*pb.Set, error) { - hdrLenRaw, consumed := binary.Uvarint(n.Data()) - if consumed <= 0 { - return nil, errors.New("invalid Set header length") - } - - pbdata := n.Data()[consumed:] - if hdrLenRaw > uint64(len(pbdata)) { - return nil, errors.New("impossibly large Set header length") - } - // as hdrLenRaw was <= an int, we now know it fits in an int - hdrLen := int(hdrLenRaw) - var hdr pb.Set - if err := proto.Unmarshal(pbdata[:hdrLen], &hdr); err != nil { - return nil, err - } - - if v := hdr.GetVersion(); v != 1 { - return nil, fmt.Errorf("unsupported Set version: %d", v) - } - if uint64(hdr.GetFanout()) > uint64(len(n.Links())) { - return nil, errors.New("impossibly large Fanout") - } - return &hdr, nil -} - -func writeHdr(n *merkledag.ProtoNode, hdr *pb.Set) error { - hdrData, err := proto.Marshal(hdr) - if err != nil { - return err - } - - // make enough space for the length prefix and the marshaled header data - data := make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)) - - // write the uvarint length of the header data - uvarlen := binary.PutUvarint(data, uint64(len(hdrData))) - - // append the actual protobuf data *after* the length value we wrote - data = append(data[:uvarlen], hdrData...) - - n.SetData(data) - return nil -} - -type walkerFunc func(idx int, link *ipld.Link) error - -func walkItems(ctx context.Context, dag ipld.DAGService, n *merkledag.ProtoNode, fn walkerFunc, children keyObserver) error { - hdr, err := readHdr(n) - if err != nil { - return err - } - // readHdr guarantees fanout is a safe value - fanout := hdr.GetFanout() - for i, l := range n.Links()[fanout:] { - if err := fn(i, l); err != nil { - return err - } - } - for _, l := range n.Links()[:fanout] { - c := l.Cid - children(c) - if c.Equals(emptyKey) { - continue - } - subtree, err := l.GetNode(ctx, dag) - if err != nil { - return err - } - - stpb, ok := subtree.(*merkledag.ProtoNode) - if !ok { - return merkledag.ErrNotProtobuf - } - - if err := walkItems(ctx, dag, stpb, fn, children); err != nil { - return err - } - } - return nil -} - -func loadSet(ctx context.Context, dag ipld.DAGService, root *merkledag.ProtoNode, name string, internalKeys keyObserver) ([]cid.Cid, error) { - l, err := root.GetNodeLink(name) - if err != nil { - return nil, err - } - - lnkc := l.Cid - internalKeys(lnkc) - - n, err := l.GetNode(ctx, dag) - if err != nil { - return nil, err - } - - pbn, ok := n.(*merkledag.ProtoNode) - if !ok { - return nil, merkledag.ErrNotProtobuf - } - - var res []cid.Cid - walk := func(idx int, link *ipld.Link) error { - res = append(res, link.Cid) - return nil - } - - if err := walkItems(ctx, dag, pbn, walk, internalKeys); err != nil { - return nil, err - } - return res, nil -} - -func getCidListIterator(cids []cid.Cid) itemIterator { - return func() (c cid.Cid, ok bool) { - if len(cids) == 0 { - return cid.Cid{}, false - } - - first := cids[0] - cids = cids[1:] - return first, true - } -} - -func storeSet(ctx context.Context, dag ipld.DAGService, cids []cid.Cid, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - iter := getCidListIterator(cids) - - n, err := storeItems(ctx, dag, uint64(len(cids)), 0, iter, internalKeys) - if err != nil { - return nil, err - } - err = dag.Add(ctx, n) - if err != nil { - return nil, err - } - internalKeys(n.Cid()) - return n, nil -} diff --git a/pin/set_test.go b/pin/set_test.go deleted file mode 100644 index d9a573c5fa21..000000000000 --- a/pin/set_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package pin - -import ( - "context" - "encoding/binary" - "testing" - - bserv "github.com/ipfs/go-blockservice" - dag "github.com/ipfs/go-merkledag" - - cid "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - dsq "github.com/ipfs/go-datastore/query" - blockstore "github.com/ipfs/go-ipfs-blockstore" - offline "github.com/ipfs/go-ipfs-exchange-offline" -) - -func ignoreCids(_ cid.Cid) {} - -func objCount(d ds.Datastore) int { - q := dsq.Query{KeysOnly: true} - res, err := d.Query(q) - if err != nil { - panic(err) - } - - var count int - for { - _, ok := res.NextSync() - if !ok { - break - } - - count++ - } - return count -} - -func TestSet(t *testing.T) { - dst := ds.NewMapDatastore() - bstore := blockstore.NewBlockstore(dst) - ds := dag.NewDAGService(bserv.New(bstore, offline.Exchange(bstore))) - - // this value triggers the creation of a recursive shard. - // If the recursive sharding is done improperly, this will result in - // an infinite recursion and crash (OOM) - limit := uint32((defaultFanout * maxItems) + 1) - - var inputs []cid.Cid - buf := make([]byte, 4) - for i := uint32(0); i < limit; i++ { - binary.BigEndian.PutUint32(buf, i) - c := dag.NewRawNode(buf).Cid() - inputs = append(inputs, c) - } - - _, err := storeSet(context.Background(), ds, inputs[:len(inputs)-1], ignoreCids) - if err != nil { - t.Fatal(err) - } - - objs1 := objCount(dst) - - out, err := storeSet(context.Background(), ds, inputs, ignoreCids) - if err != nil { - t.Fatal(err) - } - - objs2 := objCount(dst) - if objs2-objs1 > 2 { - t.Fatal("set sharding does not appear to be deterministic") - } - - // weird wrapper node because loadSet expects us to pass an - // object pointing to multiple named sets - setroot := &dag.ProtoNode{} - err = setroot.AddNodeLink("foo", out) - if err != nil { - t.Fatal(err) - } - - outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreCids) - if err != nil { - t.Fatal(err) - } - - if uint32(len(outset)) != limit { - t.Fatal("got wrong number", len(outset), limit) - } - - seen := cid.NewSet() - for _, c := range outset { - seen.Add(c) - } - - for _, c := range inputs { - if !seen.Has(c) { - t.Fatalf("expected to have '%s', didnt find it", c) - } - } -} diff --git a/tar/format.go b/tar/format.go index 02fb4b2cd33e..2bbf7298fd1a 100644 --- a/tar/format.go +++ b/tar/format.go @@ -8,8 +8,8 @@ import ( "io" "strings" - "github.com/ipfs/go-ipfs/dagutils" dag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" path "github.com/ipfs/go-path" importer "github.com/ipfs/go-unixfs/importer" uio "github.com/ipfs/go-unixfs/io"