From 02af9c9ec4110c7e9f9ace40ba9423cc4162598c Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 28 Sep 2021 22:50:19 -0700 Subject: [PATCH 01/20] Add builder for unixfs chunked files --- data/builder/file.go | 243 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 4 + go.sum | 5 + 3 files changed, 252 insertions(+) create mode 100644 data/builder/file.go diff --git a/data/builder/file.go b/data/builder/file.go new file mode 100644 index 0000000..8fdcc12 --- /dev/null +++ b/data/builder/file.go @@ -0,0 +1,243 @@ +package builder + +import ( + "fmt" + "io" + + "github.com/ipfs/go-cid" + chunk "github.com/ipfs/go-ipfs-chunker" + "github.com/ipfs/go-unixfsnode/data" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/multiformats/go-multicodec" + multihash "github.com/multiformats/go-multihash/core" +) + +// BuildUnixFSFile creates a dag of ipld Nodes represeting file data. +// This recreates the functionality previously found in +// github.com/ipfs/go-unixfs/importer/balanced, but tailored to the +// go-unixfsnode & ipld-prime data layout of nodes. +// We make some assumptions in building files with this builder to reduce +// complexity, namely: +// * we assume we are using CIDv1, which has implied that the leaf +// data nodes are stored as raw bytes. +// ref: https://github.com/ipfs/go-mfs/blob/1b1fd06cff048caabeddb02d4dbf22d2274c7971/file.go#L50 +func BuildUnixFSFile(r io.Reader, chunker string, ls *ipld.LinkSystem) (ipld.Link, error) { + s, err := chunk.FromString(r, chunker) + if err != nil { + return nil, err + } + + var prev []ipld.Link + var prevLen []uint64 + depth := 1 + for { + root, size, err := treeRecursive(depth, prev[:], prevLen[:], s, ls) + if err != nil { + return nil, err + } + + if prev != nil && prev[0] == root { + return root, nil + } + + prev = []ipld.Link{root} + prevLen = []uint64{size} + } +} + +var fileLinkProto = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagPb), + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} + +var leafLinkProto = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.Raw), + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} + +func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk.Splitter, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { + if depth == 1 && len(children) > 0 { + return nil, 0, fmt.Errorf("leaf nodes cannot have children") + } else if depth == 1 { + leaf, err := src.NextBytes() + if err == io.EOF { + return nil, 0, nil + } else if err != nil { + return nil, 0, err + } + node := basicnode.NewBytes(leaf) + link, err := ls.Store(ipld.LinkContext{}, leafLinkProto, node) + return link, uint64(len(leaf)), err + } + // depth > 1. + totalSize := uint64(0) + blksizes := make([]uint64, 0, DefaultLinksPerBlock) + if children == nil { + children = make([]ipld.Link, 0) + } else { + for i := range children { + blksizes = append(blksizes, childLen[i]) + totalSize += childLen[i] + } + } + for len(children) < DefaultLinksPerBlock { + nxt, sz, err := treeRecursive(depth-1, nil, nil, src, ls) + if err != nil { + return nil, 0, err + } else if nxt == nil { + // eof + break + } + totalSize += sz + children = append(children, nxt) + blksizes = append(blksizes, sz) + } + if len(children) == 0 { + // empty case. + return nil, 0, nil + } else if len(children) == 1 { + // degenerate case + return children[0], childLen[0], nil + } + + // make the unixfs node. + node, err := BuildUnixFS(func(b *Builder) { + FileSize(b, totalSize) + BlockSizes(b, blksizes) + }) + if err != nil { + return nil, 0, err + } + + // Pack into the dagpb node. + dpbb := dagpb.Type.PBNode.NewBuilder() + pbm, _ := dpbb.BeginMap(2) + pblb, _ := pbm.AssembleEntry("Links") + pbl, _ := pblb.BeginList(int64(len(children))) + for i, c := range children { + pbln, err := mkLink("", int64(blksizes[i]), c) + if err != nil { + return nil, 0, err + } + _ = pbl.AssembleValue().AssignNode(pbln) + } + _ = pbl.Finish() + _ = pbm.AssembleKey().AssignString("Data") + _ = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)) + _ = pbm.Finish() + pbn := dpbb.Build() + + link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn) + return link, totalSize, err +} + +func sizeOf(n ipld.Node) (uint64, error) { + if n.Kind() == ipld.Kind_Bytes { + b, err := n.AsBytes() + return uint64(len(b)), err + } else if n.Kind() == ipld.Kind_Map { + // dagpb node. + var li ipld.ListIterator + if dpb, ok := n.(dagpb.PBNode); ok { + // see if unixfs. + if ufs, err := data.DecodeUnixFSData(dpb.Data.Must().Bytes()); err == nil { + if ufs.FileSize.Exists() { + return uint64(ufs.FileSize.Must().Int()), nil + } + } + // add up link sizes + li = dpb.Links.ListIterator() + } else { + ln, err := n.LookupByString("Links") + if err != nil { + return 0, err + } + li = ln.ListIterator() + } + + s := uint64(0) + for !li.Done() { + _, l, err := li.Next() + if err != nil { + return 0, err + } + ls, err := l.LookupByString("Tsize") + if err != nil { + return 0, err + } + i, err := ls.AsInt() + if err == nil { + s += uint64(i) + } + } + return s, nil + } else { + return 0, fmt.Errorf("Unexpected node kind %s", n.Kind().String()) + } +} + +func mkLink(name string, size int64, hash ipld.Link) (ipld.Node, error) { + dpbl := dagpb.Type.PBLink.NewBuilder() + lma, err := dpbl.BeginMap(3) + if err != nil { + return nil, err + } + if err = lma.AssembleKey().AssignString("Hash"); err != nil { + return nil, err + } + if err = lma.AssembleValue().AssignLink(hash); err != nil { + return nil, err + } + if err = lma.AssembleKey().AssignString("Name"); err != nil { + return nil, err + } + if err = lma.AssembleValue().AssignString(name); err != nil { + return nil, err + } + if err = lma.AssembleKey().AssignString("Size"); err != nil { + return nil, err + } + if err = lma.AssembleValue().AssignInt(size); err != nil { + return nil, err + } + if err = lma.Finish(); err != nil { + return nil, err + } + return dpbl.Build(), nil +} + +// Constants below are from +// https://github.com/ipfs/go-unixfs/blob/ec6bb5a4c5efdc3a5bce99151b294f663ee9c08d/importer/helpers/helpers.go + +// BlockSizeLimit specifies the maximum size an imported block can have. +var BlockSizeLimit = 1048576 // 1 MB + +// rough estimates on expected sizes +var roughLinkBlockSize = 1 << 13 // 8KB +var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf framing + +// DefaultLinksPerBlock governs how the importer decides how many links there +// will be per block. This calculation is based on expected distributions of: +// * the expected distribution of block sizes +// * the expected distribution of link sizes +// * desired access speed +// For now, we use: +// +// var roughLinkBlockSize = 1 << 13 // 8KB +// var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name +// // + protobuf framing +// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize) +// = ( 8192 / 47 ) +// = (approximately) 174 +var DefaultLinksPerBlock = roughLinkBlockSize / roughLinkSize diff --git a/go.mod b/go.mod index 20afc50..715f571 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,15 @@ go 1.16 require ( github.com/Stebalien/go-bitfield v0.0.1 + github.com/ipfs/go-cid v0.0.7 + github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-merkledag v0.3.2 github.com/ipfs/go-unixfs v0.2.4 github.com/ipld/go-codec-dagpb v1.3.0 github.com/ipld/go-ipld-prime v0.11.0 + github.com/multiformats/go-multicodec v0.3.0 + github.com/multiformats/go-multihash v0.0.15 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index 9fec454..bfce411 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,7 @@ github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= @@ -159,6 +160,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= +github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= @@ -252,6 +254,8 @@ github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.3.0 h1:tstDwfIjiHbnIjeM5Lp+pMrSeN+LCMsEwOrkPmWm03A= +github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -300,6 +304,7 @@ github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvS github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= From 4e7521b6267f9acd195b815b354b3a5b732c5ded Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 28 Sep 2021 22:51:49 -0700 Subject: [PATCH 02/20] minor fixup --- data/builder/file.go | 47 +------------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/data/builder/file.go b/data/builder/file.go index 8fdcc12..44a467b 100644 --- a/data/builder/file.go +++ b/data/builder/file.go @@ -15,7 +15,7 @@ import ( multihash "github.com/multiformats/go-multihash/core" ) -// BuildUnixFSFile creates a dag of ipld Nodes represeting file data. +// BuildUnixFSFile creates a dag of ipld Nodes representing file data. // This recreates the functionality previously found in // github.com/ipfs/go-unixfs/importer/balanced, but tailored to the // go-unixfsnode & ipld-prime data layout of nodes. @@ -142,51 +142,6 @@ func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk return link, totalSize, err } -func sizeOf(n ipld.Node) (uint64, error) { - if n.Kind() == ipld.Kind_Bytes { - b, err := n.AsBytes() - return uint64(len(b)), err - } else if n.Kind() == ipld.Kind_Map { - // dagpb node. - var li ipld.ListIterator - if dpb, ok := n.(dagpb.PBNode); ok { - // see if unixfs. - if ufs, err := data.DecodeUnixFSData(dpb.Data.Must().Bytes()); err == nil { - if ufs.FileSize.Exists() { - return uint64(ufs.FileSize.Must().Int()), nil - } - } - // add up link sizes - li = dpb.Links.ListIterator() - } else { - ln, err := n.LookupByString("Links") - if err != nil { - return 0, err - } - li = ln.ListIterator() - } - - s := uint64(0) - for !li.Done() { - _, l, err := li.Next() - if err != nil { - return 0, err - } - ls, err := l.LookupByString("Tsize") - if err != nil { - return 0, err - } - i, err := ls.AsInt() - if err == nil { - s += uint64(i) - } - } - return s, nil - } else { - return 0, fmt.Errorf("Unexpected node kind %s", n.Kind().String()) - } -} - func mkLink(name string, size int64, hash ipld.Link) (ipld.Node, error) { dpbl := dagpb.Type.PBLink.NewBuilder() lma, err := dpbl.BeginMap(3) From 35e5ff9df7fdc57a02616d21de2acb3dcb8e5f0f Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 29 Sep 2021 12:54:19 -0700 Subject: [PATCH 03/20] add basic directory support --- data/builder/directory.go | 51 +++++++++++++++++++++++++++++++++++++++ data/builder/file.go | 34 +++++++++++++++++++------- 2 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 data/builder/directory.go diff --git a/data/builder/directory.go b/data/builder/directory.go new file mode 100644 index 0000000..f6e06db --- /dev/null +++ b/data/builder/directory.go @@ -0,0 +1,51 @@ +package builder + +import ( + "fmt" + + "github.com/ipfs/go-unixfsnode/data" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" +) + +// BuildUnixFSDirectory creates a directory link over a collection of entries. +// TODO: support sharded directories. +func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { + if len(entries) > DefaultLinksPerBlock { + return nil, fmt.Errorf("this builder does support sharded directories") + } + ufd, err := BuildUnixFS(func(b *Builder) { + DataType(b, data.Data_Directory) + }) + if err != nil { + return nil, err + } + pbb := dagpb.Type.PBNode.NewBuilder() + pbm, err := pbb.BeginMap(2) + if err != nil { + return nil, err + } + pbm.AssembleKey().AssignString("Data") + pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)) + pbm.AssembleKey().AssignString("Links") + lnkBuilder := dagpb.Type.PBLinks.NewBuilder() + lnks, err := lnkBuilder.BeginList(int64(len(entries))) + if err != nil { + return nil, err + } + // TODO: enforce sort order for entries. + for _, e := range entries { + if err := lnks.AssembleValue().AssignNode(e); err != nil { + return nil, err + } + } + if err := lnks.Finish(); err != nil { + return nil, err + } + pbm.AssembleValue().AssignNode(lnkBuilder.Build()) + if err := pbm.Finish(); err != nil { + return nil, err + } + node := pbb.Build() + return ls.Store(ipld.LinkContext{}, fileLinkProto, node) +} diff --git a/data/builder/file.go b/data/builder/file.go index 44a467b..7713505 100644 --- a/data/builder/file.go +++ b/data/builder/file.go @@ -24,10 +24,10 @@ import ( // * we assume we are using CIDv1, which has implied that the leaf // data nodes are stored as raw bytes. // ref: https://github.com/ipfs/go-mfs/blob/1b1fd06cff048caabeddb02d4dbf22d2274c7971/file.go#L50 -func BuildUnixFSFile(r io.Reader, chunker string, ls *ipld.LinkSystem) (ipld.Link, error) { +func BuildUnixFSFile(r io.Reader, chunker string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { s, err := chunk.FromString(r, chunker) if err != nil { - return nil, err + return nil, 0, err } var prev []ipld.Link @@ -36,15 +36,16 @@ func BuildUnixFSFile(r io.Reader, chunker string, ls *ipld.LinkSystem) (ipld.Lin for { root, size, err := treeRecursive(depth, prev[:], prevLen[:], s, ls) if err != nil { - return nil, err + return nil, 0, err } if prev != nil && prev[0] == root { - return root, nil + return root, size, nil } prev = []ipld.Link{root} prevLen = []uint64{size} + depth++ } } @@ -126,7 +127,7 @@ func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk pblb, _ := pbm.AssembleEntry("Links") pbl, _ := pblb.BeginList(int64(len(children))) for i, c := range children { - pbln, err := mkLink("", int64(blksizes[i]), c) + pbln, err := BuildUnixFSDirectoryEntry("", int64(blksizes[i]), c) if err != nil { return nil, 0, err } @@ -139,10 +140,25 @@ func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk pbn := dpbb.Build() link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn) - return link, totalSize, err + // calculate the dagpb node's size and add as overhead. + cl, ok := link.(cidlink.Link) + if !ok { + return nil, 0, fmt.Errorf("unexpected non-cid linksystem") + } + rawlnk := cid.NewCidV1(uint64(multicodec.Raw), cl.Cid.Hash()) + rn, err := ls.Load(ipld.LinkContext{}, cidlink.Link{Cid: rawlnk}, basicnode.Prototype__Bytes{}) + if err != nil { + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + } + rnb, err := rn.AsBytes() + if err != nil { + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + } + return link, totalSize + uint64(len(rnb)), err } -func mkLink(name string, size int64, hash ipld.Link) (ipld.Node, error) { +// BuildUnixFSDirectoryEntry creates the link to a file or directory as it appears within a unixfs directory. +func BuildUnixFSDirectoryEntry(name string, size int64, hash ipld.Link) (dagpb.PBLink, error) { dpbl := dagpb.Type.PBLink.NewBuilder() lma, err := dpbl.BeginMap(3) if err != nil { @@ -160,7 +176,7 @@ func mkLink(name string, size int64, hash ipld.Link) (ipld.Node, error) { if err = lma.AssembleValue().AssignString(name); err != nil { return nil, err } - if err = lma.AssembleKey().AssignString("Size"); err != nil { + if err = lma.AssembleKey().AssignString("Tsize"); err != nil { return nil, err } if err = lma.AssembleValue().AssignInt(size); err != nil { @@ -169,7 +185,7 @@ func mkLink(name string, size int64, hash ipld.Link) (ipld.Node, error) { if err = lma.Finish(); err != nil { return nil, err } - return dpbl.Build(), nil + return dpbl.Build().(dagpb.PBLink), nil } // Constants below are from From 390c8c8f11d68f6a5ed59223a4689265b908a8bf Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 29 Sep 2021 12:57:14 -0700 Subject: [PATCH 04/20] fix static check --- data/builder/file.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/data/builder/file.go b/data/builder/file.go index 7713505..e8f049b 100644 --- a/data/builder/file.go +++ b/data/builder/file.go @@ -140,6 +140,9 @@ func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk pbn := dpbb.Build() link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn) + if err != nil { + return nil, 0, err + } // calculate the dagpb node's size and add as overhead. cl, ok := link.(cidlink.Link) if !ok { @@ -154,7 +157,7 @@ func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk if err != nil { return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") } - return link, totalSize + uint64(len(rnb)), err + return link, totalSize + uint64(len(rnb)), nil } // BuildUnixFSDirectoryEntry creates the link to a file or directory as it appears within a unixfs directory. From 9912974d2c944de8d341b92b77b6c63b6300e210 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 29 Sep 2021 16:43:32 -0700 Subject: [PATCH 05/20] stable-sort of links --- data/builder/directory.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index f6e06db..44656b7 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -2,6 +2,7 @@ package builder import ( "fmt" + "sort" "github.com/ipfs/go-unixfsnode/data" dagpb "github.com/ipld/go-codec-dagpb" @@ -33,7 +34,7 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin if err != nil { return nil, err } - // TODO: enforce sort order for entries. + sort.Stable(LinkSlice(entries)) for _, e := range entries { if err := lnks.AssembleValue().AssignNode(e); err != nil { return nil, err @@ -49,3 +50,19 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin node := pbb.Build() return ls.Store(ipld.LinkContext{}, fileLinkProto, node) } + +type LinkSlice []dagpb.PBLink + +func (ls LinkSlice) Len() int { return len(ls) } +func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } +func (ls LinkSlice) Less(a, b int) bool { + na := "" + if ls[a].Name.Exists() { + na, _ = ls[a].Name.Must().AsString() + } + nb := "" + if ls[b].Name.Exists() { + nb, _ = ls[b].Name.Must().AsString() + } + return na < nb +} From dc5a1347f564295cc6827121e1ca79a3b12b7a1f Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 30 Sep 2021 08:52:20 -0700 Subject: [PATCH 06/20] remove sorting. it happens a layer down --- data/builder/directory.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index 44656b7..277dadd 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -2,7 +2,6 @@ package builder import ( "fmt" - "sort" "github.com/ipfs/go-unixfsnode/data" dagpb "github.com/ipld/go-codec-dagpb" @@ -34,7 +33,7 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin if err != nil { return nil, err } - sort.Stable(LinkSlice(entries)) + // sorting happens in codec-dagpb for _, e := range entries { if err := lnks.AssembleValue().AssignNode(e); err != nil { return nil, err @@ -50,19 +49,3 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin node := pbb.Build() return ls.Store(ipld.LinkContext{}, fileLinkProto, node) } - -type LinkSlice []dagpb.PBLink - -func (ls LinkSlice) Len() int { return len(ls) } -func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } -func (ls LinkSlice) Less(a, b int) bool { - na := "" - if ls[a].Name.Exists() { - na, _ = ls[a].Name.Must().AsString() - } - nb := "" - if ls[b].Name.Exists() { - nb, _ = ls[b].Name.Must().AsString() - } - return na < nb -} From 816434438e29a2c96c7ad7b51cbcb2d869dc43bb Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 1 Oct 2021 17:43:23 -0700 Subject: [PATCH 07/20] add general recursive serialization --- data/builder/directory.go | 40 +++++++++++++++++++++++++++++++++++++++ data/builder/file.go | 6 +++--- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index 277dadd..3eb1822 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -2,12 +2,52 @@ package builder import ( "fmt" + "os" + "path" "github.com/ipfs/go-unixfsnode/data" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" ) +// BuildUnixFSRecursive returns a link pointing to the UnixFS node representing +// the file or directory tree pointed to by `root` +// TODO: support symlinks +func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { + info, err := os.Stat(root) + if err != nil { + return nil, 0, err + } + + if info.IsDir() { + entries, err := os.ReadDir(root) + if err != nil { + return nil, 0, err + } + lnks := make([]dagpb.PBLink, 0, len(entries)) + for _, e := range entries { + lnk, sz, err := BuildUnixFSRecursive(path.Join(root, e.Name()), ls) + if err != nil { + return nil, 0, err + } + entry, err := BuildUnixFSDirectoryEntry(e.Name(), int64(sz), lnk) + if err != nil { + return nil, 0, err + } + lnks = append(lnks, entry) + } + outLnk, err := BuildUnixFSDirectory(lnks, ls) + return outLnk, 0, err + } + // else: file + fp, err := os.Open(root) + if err != nil { + return nil, 0, err + } + defer fp.Close() + return BuildUnixFSFile(fp, "", ls) +} + // BuildUnixFSDirectory creates a directory link over a collection of entries. // TODO: support sharded directories. func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { diff --git a/data/builder/file.go b/data/builder/file.go index e8f049b..2e07085 100644 --- a/data/builder/file.go +++ b/data/builder/file.go @@ -34,7 +34,7 @@ func BuildUnixFSFile(r io.Reader, chunker string, ls *ipld.LinkSystem) (ipld.Lin var prevLen []uint64 depth := 1 for { - root, size, err := treeRecursive(depth, prev[:], prevLen[:], s, ls) + root, size, err := fileTreeRecursive(depth, prev[:], prevLen[:], s, ls) if err != nil { return nil, 0, err } @@ -67,7 +67,7 @@ var leafLinkProto = cidlink.LinkPrototype{ }, } -func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk.Splitter, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { +func fileTreeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk.Splitter, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { if depth == 1 && len(children) > 0 { return nil, 0, fmt.Errorf("leaf nodes cannot have children") } else if depth == 1 { @@ -93,7 +93,7 @@ func treeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk } } for len(children) < DefaultLinksPerBlock { - nxt, sz, err := treeRecursive(depth-1, nil, nil, src, ls) + nxt, sz, err := fileTreeRecursive(depth-1, nil, nil, src, ls) if err != nil { return nil, 0, err } else if nxt == nil { From 2b1f66b6afcaafe5a15ca95afdf1ec23319d446d Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 10 Oct 2021 14:17:46 -0700 Subject: [PATCH 08/20] support for sharded directories --- data/builder/directory.go | 12 ++- data/builder/dirshard.go | 173 ++++++++++++++++++++++++++++++++++++++ data/builder/util.go | 44 ++++++++++ go.mod | 1 + go.sum | 2 + 5 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 data/builder/dirshard.go create mode 100644 data/builder/util.go diff --git a/data/builder/directory.go b/data/builder/directory.go index 3eb1822..c820004 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -1,15 +1,21 @@ package builder import ( - "fmt" "os" "path" "github.com/ipfs/go-unixfsnode/data" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" + "github.com/multiformats/go-multihash" ) +// https://github.com/ipfs/go-ipfs/pull/8114/files#diff-eec963b47a6e1080d9d8023b4e438e6e3591b4154f7379a7e728401d2055374aR319 +const shardSplitThreshold = 262144 + +// https://github.com/ipfs/go-unixfs/blob/ec6bb5a4c5efdc3a5bce99151b294f663ee9c08d/io/directory.go#L29 +const defaultShardWidth = 256 + // BuildUnixFSRecursive returns a link pointing to the UnixFS node representing // the file or directory tree pointed to by `root` // TODO: support symlinks @@ -49,10 +55,10 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, } // BuildUnixFSDirectory creates a directory link over a collection of entries. -// TODO: support sharded directories. func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { + // TODO: flip to shards based on resulting node size estimate rather than link count. if len(entries) > DefaultLinksPerBlock { - return nil, fmt.Errorf("this builder does support sharded directories") + return BuildUnixFSShardedDirectory(defaultShardWidth, multihash.MURMUR3_128, entries, ls) } ufd, err := BuildUnixFS(func(b *Builder) { DataType(b, data.Data_Directory) diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go new file mode 100644 index 0000000..36f41e7 --- /dev/null +++ b/data/builder/dirshard.go @@ -0,0 +1,173 @@ +package builder + +import ( + "fmt" + + bitfield "github.com/ipfs/go-bitfield" + "github.com/ipfs/go-unixfsnode/data" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/multiformats/go-multihash" +) + +type shard struct { + // metadata about the shard + hasher uint64 + size int + width int + depth int + + children map[int]entry +} + +// a shard entry is either another shard, or a direct link. +type entry struct { + *shard + *hamtLink +} + +// a hamtLink is a member of the hamt - the file/directory pointed to, but +// stored with it's hashed key used for addressing. +type hamtLink struct { + hash hashBits + dagpb.PBLink +} + +// BuildUnixFSShardedDirectory will build a hamt of unixfs hamt shards encoing a directory with more entries +// than is typically allowed to fit in a standard IPFS single-block unixFS directory. +func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { + // hash the entries + h, err := multihash.GetHasher(hasher) + if err != nil { + return nil, err + } + he := make([]hamtLink, 0, len(entries)) + for _, e := range entries { + name := e.Name.Must().String() + sum := h.Sum([]byte(name)) + he = append(he, hamtLink{ + sum, + e, + }) + } + + sharder := shard{ + hasher: hasher, + size: size, + width: len(fmt.Sprintf("%X", size-1)), + depth: 0, + + children: make(map[int]entry), + } + + for _, e := range he { + sharder.add(e) + } + + return sharder.serialize(ls) +} + +func (s *shard) add(lnk hamtLink) error { + // get the bucket for lnk + bucket, err := lnk.hash.Slice(s.depth*s.size, s.size) + if err != nil { + return err + } + + current, ok := s.children[bucket] + if !ok { + s.children[bucket] = entry{nil, &lnk} + return nil + } else if current.shard != nil { + return current.shard.add(lnk) + } + // make a shard for current and lnk + newShard := entry{ + &shard{ + hasher: s.hasher, + size: s.size, + width: s.width, + depth: s.depth + 1, + children: make(map[int]entry), + }, + nil, + } + if err := newShard.add(*current.hamtLink); err != nil { + return err + } + s.children[bucket] = newShard + return newShard.add(lnk) +} + +func (s *shard) formatLinkName(name string, idx int) string { + return fmt.Sprintf("%*X%s", s.width, idx, name) +} + +// bitmap calculates the bitmap of which links in the shard are set. +func (s *shard) bitmap() []byte { + bm := bitfield.NewBitfield(s.size) + for i := 0; i < s.size; i++ { + if _, ok := s.children[i]; ok { + bm.SetBit(i) + } + } + return bm.Bytes() +} + +// serialize stores the concrete representation of this shard in the link system and +// returns a link to it. +func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, error) { + ufd, err := BuildUnixFS(func(b *Builder) { + DataType(b, data.Data_HAMTShard) + HashType(b, s.hasher) + Data(b, s.bitmap()) + Fanout(b, uint64(s.size)) + }) + if err != nil { + return nil, err + } + pbb := dagpb.Type.PBNode.NewBuilder() + pbm, err := pbb.BeginMap(2) + if err != nil { + return nil, err + } + pbm.AssembleKey().AssignString("Data") + pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)) + pbm.AssembleKey().AssignString("Links") + + lnkBuilder := dagpb.Type.PBLinks.NewBuilder() + lnks, err := lnkBuilder.BeginList(int64(len(s.children))) + if err != nil { + return nil, err + } + // sorting happens in codec-dagpb + for idx, e := range s.children { + var lnk dagpb.PBLink + if e.shard != nil { + ipldLnk, err := e.shard.serialize(ls) + if err != nil { + return nil, err + } + fullName := s.formatLinkName("", idx) + lnk, err = BuildUnixFSDirectoryEntry(fullName, 0, ipldLnk) + } else { + fullName := s.formatLinkName(e.Name.Must().String(), idx) + lnk, err = BuildUnixFSDirectoryEntry(fullName, e.Tsize.Must().Int(), e.Hash.Link()) + } + if err != nil { + return nil, err + } + if err := lnks.AssembleValue().AssignNode(lnk); err != nil { + return nil, err + } + } + if err := lnks.Finish(); err != nil { + return nil, err + } + pbm.AssembleValue().AssignNode(lnkBuilder.Build()) + if err := pbm.Finish(); err != nil { + return nil, err + } + node := pbb.Build() + return ls.Store(ipld.LinkContext{}, fileLinkProto, node) +} diff --git a/data/builder/util.go b/data/builder/util.go new file mode 100644 index 0000000..4b9decb --- /dev/null +++ b/data/builder/util.go @@ -0,0 +1,44 @@ +package builder + +import ( + "fmt" +) + +// Common code from go-unixfs/hamt/util.go + +// hashBits is a helper for pulling out sections of a hash +type hashBits []byte + +func mkmask(n int) byte { + return (1 << uint(n)) - 1 +} + +// Slice returns the 'width' bits of the hashBits value as an integer, or an +// error if there aren't enough bits. +func (hb hashBits) Slice(offset, width int) (int, error) { + if offset+width > len(hb)*8 { + return 0, fmt.Errorf("sharded directory too deep") + } + return hb.slice(offset, width), nil +} + +func (hb hashBits) slice(offset, width int) int { + curbi := offset / 8 + leftb := 8 - (offset % 8) + + curb := hb[curbi] + if width == leftb { + out := int(mkmask(width) & curb) + return out + } else if width < leftb { + a := curb & mkmask(leftb) // mask out the high bits we don't want + b := a & ^mkmask(leftb-width) // mask out the low bits we don't want + c := b >> uint(leftb-width) // shift whats left down + return int(c) + } else { + out := int(mkmask(leftb) & curb) + out <<= uint(width - leftb) + out += hb.slice(offset+leftb, width-leftb) + return out + } +} diff --git a/go.mod b/go.mod index 715f571..7813833 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/Stebalien/go-bitfield v0.0.1 + github.com/ipfs/go-bitfield v1.0.0 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipld-format v0.2.0 diff --git a/go.sum b/go.sum index bfce411..3d0b921 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3 github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= +github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2 h1:IkhOPiifc6b2LWTi/vp8TXwNT0eGCsizI1JFbZ08IQQ= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= From 40f88ce37a5a222a8af390cfc09d1107ea83b9d7 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 10 Oct 2021 14:40:09 -0700 Subject: [PATCH 09/20] size-based sharding. symlinks --- data/builder/directory.go | 32 +++++++++++++++++++++++++--- data/builder/dirshard.go | 3 +++ data/builder/file.go | 45 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index c820004..136ad07 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -1,12 +1,15 @@ package builder import ( + "fmt" + "io/fs" "os" "path" "github.com/ipfs/go-unixfsnode/data" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/multiformats/go-multihash" ) @@ -20,7 +23,7 @@ const defaultShardWidth = 256 // the file or directory tree pointed to by `root` // TODO: support symlinks func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { - info, err := os.Stat(root) + info, err := os.Lstat(root) if err != nil { return nil, 0, err } @@ -45,6 +48,15 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, outLnk, err := BuildUnixFSDirectory(lnks, ls) return outLnk, 0, err } + if info.Mode().Type() == fs.ModeSymlink { + content, err := os.Readlink(root) + if err != nil { + return nil, 0, err + } + return BuildUnixFSSymlink(content, ls) + } else if !info.Mode().IsRegular() { + return nil, 0, fmt.Errorf("cannot encode non regular file: %s", root) + } // else: file fp, err := os.Open(root) if err != nil { @@ -54,10 +66,24 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, return BuildUnixFSFile(fp, "", ls) } +// estimateDirSize estimates if a directory is big enough that it warrents sharding +func estimateDirSize(entries []dagpb.PBLink) int { + s := 0 + for _, e := range entries { + lnk := e.Hash.Link() + cl, ok := lnk.(cidlink.Link) + if ok { + s += len(e.Name.Must().String()) + cl.ByteLen() + } else { + s += len(e.Name.Must().String()) + len(lnk.String()) + } + } + return s +} + // BuildUnixFSDirectory creates a directory link over a collection of entries. func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { - // TODO: flip to shards based on resulting node size estimate rather than link count. - if len(entries) > DefaultLinksPerBlock { + if estimateDirSize(entries) > shardSplitThreshold { return BuildUnixFSShardedDirectory(defaultShardWidth, multihash.MURMUR3_128, entries, ls) } ufd, err := BuildUnixFS(func(b *Builder) { diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go index 36f41e7..c01566a 100644 --- a/data/builder/dirshard.go +++ b/data/builder/dirshard.go @@ -150,6 +150,9 @@ func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, error) { } fullName := s.formatLinkName("", idx) lnk, err = BuildUnixFSDirectoryEntry(fullName, 0, ipldLnk) + if err != nil { + return nil, err + } } else { fullName := s.formatLinkName(e.Name.Must().String(), idx) lnk, err = BuildUnixFSDirectoryEntry(fullName, e.Tsize.Must().Int(), e.Hash.Link()) diff --git a/data/builder/file.go b/data/builder/file.go index 2e07085..705fe70 100644 --- a/data/builder/file.go +++ b/data/builder/file.go @@ -191,6 +191,51 @@ func BuildUnixFSDirectoryEntry(name string, size int64, hash ipld.Link) (dagpb.P return dpbl.Build().(dagpb.PBLink), nil } +// BuildUnixFSSymlink builds a symlink entry in a unixfs tree +func BuildUnixFSSymlink(content string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { + // make the unixfs node. + node, err := BuildUnixFS(func(b *Builder) { + DataType(b, data.Data_Symlink) + Data(b, []byte(content)) + }) + if err != nil { + return nil, 0, err + } + + dpbb := dagpb.Type.PBNode.NewBuilder() + pbm, err := dpbb.BeginMap(2) + if err != nil { + return nil, 0, err + } + pblb, _ := pbm.AssembleEntry("Links") + pbl, _ := pblb.BeginList(0) + _ = pbl.Finish() + _ = pbm.AssembleKey().AssignString("Data") + _ = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)) + _ = pbm.Finish() + pbn := dpbb.Build() + + link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn) + if err != nil { + return nil, 0, err + } + // calculate the size and add as overhead. + cl, ok := link.(cidlink.Link) + if !ok { + return nil, 0, fmt.Errorf("unexpected non-cid linksystem") + } + rawlnk := cid.NewCidV1(uint64(multicodec.Raw), cl.Cid.Hash()) + rn, err := ls.Load(ipld.LinkContext{}, cidlink.Link{Cid: rawlnk}, basicnode.Prototype__Bytes{}) + if err != nil { + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + } + rnb, err := rn.AsBytes() + if err != nil { + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + } + return link, uint64(len(rnb)), nil +} + // Constants below are from // https://github.com/ipfs/go-unixfs/blob/ec6bb5a4c5efdc3a5bce99151b294f663ee9c08d/importer/helpers/helpers.go From 4acade0cb9a2b3de1698cbdf8d9f7ad6c3e8f224 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 12 Oct 2021 09:11:29 -0700 Subject: [PATCH 10/20] code review --- data/builder/directory.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index 136ad07..0b10230 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -28,7 +28,9 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, return nil, 0, err } - if info.IsDir() { + m := info.Mode() + switch { + case m.IsDir(): entries, err := os.ReadDir(root) if err != nil { return nil, 0, err @@ -47,23 +49,22 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, } outLnk, err := BuildUnixFSDirectory(lnks, ls) return outLnk, 0, err - } - if info.Mode().Type() == fs.ModeSymlink { + case m.Type() == fs.ModeSymlink: content, err := os.Readlink(root) if err != nil { return nil, 0, err } return BuildUnixFSSymlink(content, ls) - } else if !info.Mode().IsRegular() { + case m.IsRegular(): + fp, err := os.Open(root) + if err != nil { + return nil, 0, err + } + defer fp.Close() + return BuildUnixFSFile(fp, "", ls) + default: return nil, 0, fmt.Errorf("cannot encode non regular file: %s", root) } - // else: file - fp, err := os.Open(root) - if err != nil { - return nil, 0, err - } - defer fp.Close() - return BuildUnixFSFile(fp, "", ls) } // estimateDirSize estimates if a directory is big enough that it warrents sharding @@ -100,8 +101,7 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin pbm.AssembleKey().AssignString("Data") pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)) pbm.AssembleKey().AssignString("Links") - lnkBuilder := dagpb.Type.PBLinks.NewBuilder() - lnks, err := lnkBuilder.BeginList(int64(len(entries))) + lnks, err := pbm.AssembleValue().BeginList(int64(len(entries))) if err != nil { return nil, err } @@ -114,7 +114,6 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin if err := lnks.Finish(); err != nil { return nil, err } - pbm.AssembleValue().AssignNode(lnkBuilder.Build()) if err := pbm.Finish(); err != nil { return nil, err } From 8148a62a088739babd4ec8c5d5b5572661c58625 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 12 Oct 2021 11:35:56 -0700 Subject: [PATCH 11/20] remove todo --- data/builder/directory.go | 1 - 1 file changed, 1 deletion(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index 0b10230..90be58b 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -21,7 +21,6 @@ const defaultShardWidth = 256 // BuildUnixFSRecursive returns a link pointing to the UnixFS node representing // the file or directory tree pointed to by `root` -// TODO: support symlinks func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { info, err := os.Lstat(root) if err != nil { From 319d445359412d9e7148e567fc7c9bcf3f829574 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 12 Oct 2021 12:38:26 -0700 Subject: [PATCH 12/20] additional comment --- data/builder/directory.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index 90be58b..635ae74 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -66,7 +66,9 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, } } -// estimateDirSize estimates if a directory is big enough that it warrents sharding +// estimateDirSize estimates if a directory is big enough that it warrents sharding. +// The estimate is the sum over the len(linkName) + bytelen(linkHash) +// https://github.com/ipfs/go-unixfs/blob/master/io/directory.go#L152-L162 func estimateDirSize(entries []dagpb.PBLink) int { s := 0 for _, e := range entries { From ee45835f1076aa21d3f47da7a111c9cb1ad64caf Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 13 Oct 2021 13:28:25 -0700 Subject: [PATCH 13/20] better name --- data/builder/dirshard.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go index c01566a..e462694 100644 --- a/data/builder/dirshard.go +++ b/data/builder/dirshard.go @@ -41,11 +41,11 @@ func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink if err != nil { return nil, err } - he := make([]hamtLink, 0, len(entries)) + hamtEntries := make([]hamtLink, 0, len(entries)) for _, e := range entries { name := e.Name.Must().String() sum := h.Sum([]byte(name)) - he = append(he, hamtLink{ + hamtEntries = append(hamtEntries, hamtLink{ sum, e, }) @@ -60,8 +60,8 @@ func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink children: make(map[int]entry), } - for _, e := range he { - sharder.add(e) + for _, entry := range hamtEntries { + sharder.add(entry) } return sharder.serialize(ls) From a6359adbfad7a4c6b4b87306a75f4beda095de03 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 13 Oct 2021 16:36:35 -0700 Subject: [PATCH 14/20] tests for files & directories round-tripping --- data/builder/dir_test.go | 72 +++++++ data/builder/dirshard.go | 33 ++- data/builder/file.go | 11 +- data/builder/file_test.go | 43 ++++ data/builder/util.go | 12 ++ go.mod | 8 +- go.sum | 432 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 575 insertions(+), 36 deletions(-) create mode 100644 data/builder/dir_test.go create mode 100644 data/builder/file_test.go diff --git a/data/builder/dir_test.go b/data/builder/dir_test.go new file mode 100644 index 0000000..954d2dd --- /dev/null +++ b/data/builder/dir_test.go @@ -0,0 +1,72 @@ +package builder + +import ( + "bytes" + "fmt" + "testing" + + "github.com/ipfs/go-unixfsnode" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +func mkEntries(cnt int, ls *ipld.LinkSystem) ([]dagpb.PBLink, error) { + entries := make([]dagpb.PBLink, 0, cnt) + for i := 0; i < cnt; i++ { + r := bytes.NewBufferString(fmt.Sprintf("%d", i)) + f, s, err := BuildUnixFSFile(r, "", ls) + if err != nil { + return nil, err + } + e, err := BuildUnixFSDirectoryEntry(fmt.Sprintf("file %d", i), int64(s), f) + if err != nil { + return nil, err + } + entries = append(entries, e) + } + return entries, nil +} + +func TestBuildUnixFSDirectory(t *testing.T) { + ls := cidlink.DefaultLinkSystem() + storage := cidlink.Memory{} + ls.StorageReadOpener = storage.OpenRead + ls.StorageWriteOpener = storage.OpenWrite + + testSizes := []int{100, 1000, 50000} + for _, cnt := range testSizes { + entries, err := mkEntries(cnt, &ls) + if err != nil { + t.Fatal(err) + } + + dl, err := BuildUnixFSDirectory(entries, &ls) + if err != nil { + t.Fatal(err) + } + + pbn, err := ls.Load(ipld.LinkContext{}, dl, dagpb.Type.PBNode) + if err != nil { + t.Fatal(err) + } + ufd, err := unixfsnode.Reify(ipld.LinkContext{}, pbn, &ls) + if err != nil { + t.Fatal(err) + } + observedCnt := 0 + + li := ufd.MapIterator() + for !li.Done() { + _, _, err := li.Next() + if err != nil { + t.Fatal(err) + } + observedCnt++ + } + if observedCnt != cnt { + fmt.Printf("%+v\n", ufd) + t.Fatalf("unexpected number of dir entries %d vs %d", observedCnt, cnt) + } + } +} diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go index e462694..a6dd63b 100644 --- a/data/builder/dirshard.go +++ b/data/builder/dirshard.go @@ -12,10 +12,11 @@ import ( type shard struct { // metadata about the shard - hasher uint64 - size int - width int - depth int + hasher uint64 + size int + sizeLg2 int + width int + depth int children map[int]entry } @@ -51,25 +52,35 @@ func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink }) } + sizeLg2, err := logtwo(size) + if err != nil { + return nil, err + } + sharder := shard{ - hasher: hasher, - size: size, - width: len(fmt.Sprintf("%X", size-1)), - depth: 0, + hasher: hasher, + size: size, + sizeLg2: sizeLg2, + width: len(fmt.Sprintf("%X", size-1)), + depth: 0, children: make(map[int]entry), } for _, entry := range hamtEntries { - sharder.add(entry) + err := sharder.add(entry) + if err != nil { + return nil, err + } } + fmt.Printf("sharder to serialize, %d children (%d)\n", len(sharder.children), len(hamtEntries)) return sharder.serialize(ls) } func (s *shard) add(lnk hamtLink) error { // get the bucket for lnk - bucket, err := lnk.hash.Slice(s.depth*s.size, s.size) + bucket, err := lnk.hash.Slice(s.depth*s.sizeLg2, s.sizeLg2) if err != nil { return err } @@ -86,6 +97,7 @@ func (s *shard) add(lnk hamtLink) error { &shard{ hasher: s.hasher, size: s.size, + sizeLg2: s.sizeLg2, width: s.width, depth: s.depth + 1, children: make(map[int]entry), @@ -123,6 +135,7 @@ func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, error) { Data(b, s.bitmap()) Fanout(b, uint64(s.size)) }) + fmt.Printf("shard: %+v\n", ufd) if err != nil { return nil, err } diff --git a/data/builder/file.go b/data/builder/file.go index 705fe70..4a32209 100644 --- a/data/builder/file.go +++ b/data/builder/file.go @@ -13,6 +13,9 @@ import ( basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/multiformats/go-multicodec" multihash "github.com/multiformats/go-multihash/core" + + // raw needed for opening as bytes + _ "github.com/ipld/go-ipld-prime/codec/raw" ) // BuildUnixFSFile creates a dag of ipld Nodes representing file data. @@ -151,11 +154,11 @@ func fileTreeRecursive(depth int, children []ipld.Link, childLen []uint64, src c rawlnk := cid.NewCidV1(uint64(multicodec.Raw), cl.Cid.Hash()) rn, err := ls.Load(ipld.LinkContext{}, cidlink.Link{Cid: rawlnk}, basicnode.Prototype__Bytes{}) if err != nil { - return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes: %w", err) } rnb, err := rn.AsBytes() if err != nil { - return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + return nil, 0, fmt.Errorf("could not parse dagpb node as bytes: %w", err) } return link, totalSize + uint64(len(rnb)), nil } @@ -227,11 +230,11 @@ func BuildUnixFSSymlink(content string, ls *ipld.LinkSystem) (ipld.Link, uint64, rawlnk := cid.NewCidV1(uint64(multicodec.Raw), cl.Cid.Hash()) rn, err := ls.Load(ipld.LinkContext{}, cidlink.Link{Cid: rawlnk}, basicnode.Prototype__Bytes{}) if err != nil { - return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes: %w", err) } rnb, err := rn.AsBytes() if err != nil { - return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes") + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes: %w", err) } return link, uint64(len(rnb)), nil } diff --git a/data/builder/file_test.go b/data/builder/file_test.go new file mode 100644 index 0000000..dc10b7e --- /dev/null +++ b/data/builder/file_test.go @@ -0,0 +1,43 @@ +package builder + +import ( + "bytes" + "testing" + + "github.com/ipfs/go-cid" + u "github.com/ipfs/go-ipfs-util" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +func TestBuildUnixFSFile(t *testing.T) { + buf := make([]byte, 10*1024*1024) + u.NewSeededRand(0xdeadbeef).Read(buf) + r := bytes.NewReader(buf) + + ls := cidlink.DefaultLinkSystem() + storage := cidlink.Memory{} + ls.StorageReadOpener = storage.OpenRead + ls.StorageWriteOpener = storage.OpenWrite + + f, _, err := BuildUnixFSFile(r, "", &ls) + if err != nil { + t.Fatal(err) + } + + // Note: this differs from the previous + // go-unixfs version of this test (https://github.com/ipfs/go-unixfs/blob/master/importer/importer_test.go#L50) + // because this library enforces CidV1 encoding. + expected, err := cid.Decode("bafybeieyxejezqto5xwcxtvh5tskowwxrn3hmbk3hcgredji3g7abtnfkq") + if err != nil { + t.Fatal(err) + } + if !expected.Equals(f.(cidlink.Link).Cid) { + t.Fatalf("expected CID %s, got CID %s", expected, f) + } + if _, err := storage.OpenRead(ipld.LinkContext{}, f); err != nil { + t.Fatal("expected top of file to be in store.") + } + + return +} diff --git a/data/builder/util.go b/data/builder/util.go index 4b9decb..8e5c0fb 100644 --- a/data/builder/util.go +++ b/data/builder/util.go @@ -2,6 +2,7 @@ package builder import ( "fmt" + "math/bits" ) // Common code from go-unixfs/hamt/util.go @@ -42,3 +43,14 @@ func (hb hashBits) slice(offset, width int) int { return out } } + +func logtwo(v int) (int, error) { + if v <= 0 { + return 0, fmt.Errorf("hamt size should be a power of two") + } + lg2 := bits.TrailingZeros(uint(v)) + if 1< Date: Wed, 13 Oct 2021 16:38:58 -0700 Subject: [PATCH 15/20] fix static --- data/builder/file_test.go | 2 - go.mod | 9 +- go.sum | 352 -------------------------------------- 3 files changed, 8 insertions(+), 355 deletions(-) diff --git a/data/builder/file_test.go b/data/builder/file_test.go index dc10b7e..9fbe8e0 100644 --- a/data/builder/file_test.go +++ b/data/builder/file_test.go @@ -38,6 +38,4 @@ func TestBuildUnixFSFile(t *testing.T) { if _, err := storage.OpenRead(ipld.LinkContext{}, f); err != nil { t.Fatal("expected top of file to be in store.") } - - return } diff --git a/go.mod b/go.mod index 92485d5..f6ee5d3 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,25 @@ go 1.16 require ( github.com/Stebalien/go-bitfield v0.0.1 github.com/ipfs/go-bitfield v1.0.0 + github.com/ipfs/go-block-format v0.0.3 // indirect github.com/ipfs/go-cid v0.1.0 + github.com/ipfs/go-ipfs-blockstore v1.0.3 // indirect github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipfs-util v0.0.2 + github.com/ipfs/go-ipld-cbor v0.0.5 // indirect github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-merkledag v0.3.2 github.com/ipfs/go-unixfs v0.2.4 github.com/ipld/go-codec-dagpb v1.3.0 github.com/ipld/go-ipld-prime v0.12.4-0.20211013215610-76353188a8a6 + github.com/mattn/go-colorable v0.1.8 // indirect github.com/multiformats/go-multicodec v0.3.0 github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2 - github.com/proofzero/go-ipld-linkstore v1.0.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 + go.uber.org/atomic v1.7.0 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.27.1 ) diff --git a/go.sum b/go.sum index 1216085..bf690e7 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,8 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50 h1:4i3KsuVA0o0KoBxAC5x+MY7RbteiMK1V7gf/G08NGIQ= @@ -42,22 +15,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -65,107 +24,39 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= @@ -186,7 +77,6 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.0.8-0.20210716091050-de6c03deae1c/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -247,17 +137,9 @@ github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs= -github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs= -github.com/ipld/go-car/v2 v2.0.2 h1:R1oIAPwrGp26mEFzcGf5bfTZAAHDOkaVnZTEVebaWX4= -github.com/ipld/go-car/v2 v2.0.2/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= -github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= -github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.12.0 h1:JapyKWTsJgmhrPI7hfx4V798c/RClr85sXfBZnH1VIw= -github.com/ipld/go-ipld-prime v0.12.0/go.mod h1:hy8b93WleDMRKumOJnTIrr0MbbFbx9GD6Kzxa53Xppc= github.com/ipld/go-ipld-prime v0.12.4-0.20211013215610-76353188a8a6 h1:mKlyUGDuu9HNn5EcP5Ck7Ln3LoQzeIQtUaB1ge+B1UQ= github.com/ipld/go-ipld-prime v0.12.4-0.20211013215610-76353188a8a6/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= @@ -271,29 +153,19 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.8 h1:bhR2mgIlno/Sfk4oUbH4sPlc83z1yGrN9bvqiq3C33I= -github.com/klauspost/cpuid/v2 v2.0.8/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -358,23 +230,14 @@ github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjV github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= @@ -384,17 +247,6 @@ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -418,7 +270,6 @@ github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.0 h1:tstDwfIjiHbnIjeM5Lp+pMrSeN+LCMsEwOrkPmWm03A= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -426,7 +277,6 @@ github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcM github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2 h1:gmJIXPl/LE3dXLii1Xce8QsTJax02FoJMPsXBtTg6Jw= github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= @@ -435,82 +285,21 @@ github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wS github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/proofzero/go-ipld-linkstore v1.0.0 h1:ZcQHPtiPf5wlBKGKEs+OG93XzdhjL5+11LI5HL9hP14= -github.com/proofzero/go-ipld-linkstore v1.0.0/go.mod h1:SK/37UY7ek8N14j75zddLThENn3WfArwT4BgbHHY2TY= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -518,37 +307,23 @@ github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a h1:/eS3yfGjQKG+9kayBkj0ip1BGhq6zJ3eaVksphxAaek= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= -github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= @@ -562,127 +337,47 @@ github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg5 github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= -golang.org/x/exp v0.0.0-20210615023648-acb5c1269671 h1:ddvpKwqE7dm58PoWjRCmaCiA3DANEW0zWGfNYQD212Y= -golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= @@ -693,76 +388,29 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From b3969b1b1657af27c42eef77b77621549ca04215 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 13 Oct 2021 17:30:13 -0700 Subject: [PATCH 16/20] add builder special case for murmur3 --- data/builder/dirshard.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go index a6dd63b..ba9c9cd 100644 --- a/data/builder/dirshard.go +++ b/data/builder/dirshard.go @@ -2,12 +2,15 @@ package builder import ( "fmt" + "hash" bitfield "github.com/ipfs/go-bitfield" "github.com/ipfs/go-unixfsnode/data" + "github.com/ipfs/go-unixfsnode/hamt" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" "github.com/multiformats/go-multihash" + "github.com/spaolacci/murmur3" ) type shard struct { @@ -38,9 +41,16 @@ type hamtLink struct { // than is typically allowed to fit in a standard IPFS single-block unixFS directory. func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { // hash the entries - h, err := multihash.GetHasher(hasher) - if err != nil { - return nil, err + var h hash.Hash + var err error + // TODO: use the multihash registry once murmur3 behavior is encoded there. + if hasher == hamt.HashMurmur3 { + h = murmur3.New64() + } else { + h, err = multihash.GetHasher(hasher) + if err != nil { + return nil, err + } } hamtEntries := make([]hamtLink, 0, len(entries)) for _, e := range entries { From 30bc1adc032f26ca6081e32d0038d9fb9c9ad17b Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 13 Oct 2021 18:35:40 -0700 Subject: [PATCH 17/20] clarity on multihash --- data/builder/dirshard.go | 1 + go.mod | 2 +- go.sum | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go index ba9c9cd..0138536 100644 --- a/data/builder/dirshard.go +++ b/data/builder/dirshard.go @@ -44,6 +44,7 @@ func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink var h hash.Hash var err error // TODO: use the multihash registry once murmur3 behavior is encoded there. + // https://github.com/multiformats/go-multihash/pull/150 if hasher == hamt.HashMurmur3 { h = murmur3.New64() } else { diff --git a/go.mod b/go.mod index f6ee5d3..88da156 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/ipld/go-ipld-prime v0.12.4-0.20211013215610-76353188a8a6 github.com/mattn/go-colorable v0.1.8 // indirect github.com/multiformats/go-multicodec v0.3.0 - github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2 + github.com/multiformats/go-multihash v0.0.17-0.20211012170226-654b06d7912a github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 go.uber.org/atomic v1.7.0 // indirect diff --git a/go.sum b/go.sum index bf690e7..c8eb02f 100644 --- a/go.sum +++ b/go.sum @@ -278,6 +278,8 @@ github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpK github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= +github.com/multiformats/go-multihash v0.0.17-0.20211012170226-654b06d7912a h1:zq4kIpOSKshyuBTNmPMRG/XIP5PlH4M+kk3kz7FbSoU= +github.com/multiformats/go-multihash v0.0.17-0.20211012170226-654b06d7912a/go.mod h1:rhm8ul8hQFEC1hEgrxrvkY/k/k8aOTRKanxCPVANMQs= github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2 h1:gmJIXPl/LE3dXLii1Xce8QsTJax02FoJMPsXBtTg6Jw= github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= From 5ef2170a693484d74b1884b64e46b35c90830bef Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 13 Oct 2021 18:42:45 -0700 Subject: [PATCH 18/20] tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index c8eb02f..90cfb54 100644 --- a/go.sum +++ b/go.sum @@ -280,8 +280,6 @@ github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.0.17-0.20211012170226-654b06d7912a h1:zq4kIpOSKshyuBTNmPMRG/XIP5PlH4M+kk3kz7FbSoU= github.com/multiformats/go-multihash v0.0.17-0.20211012170226-654b06d7912a/go.mod h1:rhm8ul8hQFEC1hEgrxrvkY/k/k8aOTRKanxCPVANMQs= -github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2 h1:gmJIXPl/LE3dXLii1Xce8QsTJax02FoJMPsXBtTg6Jw= -github.com/multiformats/go-multihash v0.0.17-0.20211013230636-971a35b6edf2/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= From ef5d36d2b2de23a890213f8f74858d2570310166 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 14 Oct 2021 16:02:51 -0700 Subject: [PATCH 19/20] update to ipld prime master --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 88da156..945954a 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/ipfs/go-merkledag v0.3.2 github.com/ipfs/go-unixfs v0.2.4 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.12.4-0.20211013215610-76353188a8a6 + github.com/ipld/go-ipld-prime v0.12.4-0.20211014180653-3ba656a3bc6b github.com/mattn/go-colorable v0.1.8 // indirect github.com/multiformats/go-multicodec v0.3.0 github.com/multiformats/go-multihash v0.0.17-0.20211012170226-654b06d7912a diff --git a/go.sum b/go.sum index 90cfb54..1e56aad 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZ github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.12.4-0.20211013215610-76353188a8a6 h1:mKlyUGDuu9HNn5EcP5Ck7Ln3LoQzeIQtUaB1ge+B1UQ= -github.com/ipld/go-ipld-prime v0.12.4-0.20211013215610-76353188a8a6/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= +github.com/ipld/go-ipld-prime v0.12.4-0.20211014180653-3ba656a3bc6b h1:rrGbdP3zuD8NmVJ/FKju8zSaskYNMXhgjJ2DsgKSdY4= +github.com/ipld/go-ipld-prime v0.12.4-0.20211014180653-3ba656a3bc6b/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= From 6a9bae0b791587dae74c2796e3886072669986cb Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 15 Oct 2021 11:04:41 -0700 Subject: [PATCH 20/20] more uniform error handling --- data/builder/directory.go | 17 +++++++---- data/builder/dirshard.go | 14 +++++---- data/builder/file.go | 63 +++++++++++++++++++++++++++++---------- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/data/builder/directory.go b/data/builder/directory.go index 635ae74..c6b0549 100644 --- a/data/builder/directory.go +++ b/data/builder/directory.go @@ -72,12 +72,13 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, func estimateDirSize(entries []dagpb.PBLink) int { s := 0 for _, e := range entries { + s += len(e.Name.Must().String()) lnk := e.Hash.Link() cl, ok := lnk.(cidlink.Link) if ok { - s += len(e.Name.Must().String()) + cl.ByteLen() + s += cl.ByteLen() } else { - s += len(e.Name.Must().String()) + len(lnk.String()) + s += len(lnk.Binary()) } } return s @@ -99,9 +100,15 @@ func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Lin if err != nil { return nil, err } - pbm.AssembleKey().AssignString("Data") - pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)) - pbm.AssembleKey().AssignString("Links") + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)); err != nil { + return nil, err + } + if err = pbm.AssembleKey().AssignString("Links"); err != nil { + return nil, err + } lnks, err := pbm.AssembleValue().BeginList(int64(len(entries))) if err != nil { return nil, err diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go index 0138536..a25aa66 100644 --- a/data/builder/dirshard.go +++ b/data/builder/dirshard.go @@ -84,7 +84,6 @@ func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink return nil, err } } - fmt.Printf("sharder to serialize, %d children (%d)\n", len(sharder.children), len(hamtEntries)) return sharder.serialize(ls) } @@ -146,7 +145,6 @@ func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, error) { Data(b, s.bitmap()) Fanout(b, uint64(s.size)) }) - fmt.Printf("shard: %+v\n", ufd) if err != nil { return nil, err } @@ -155,9 +153,15 @@ func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, error) { if err != nil { return nil, err } - pbm.AssembleKey().AssignString("Data") - pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)) - pbm.AssembleKey().AssignString("Links") + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)); err != nil { + return nil, err + } + if err = pbm.AssembleKey().AssignString("Links"); err != nil { + return nil, err + } lnkBuilder := dagpb.Type.PBLinks.NewBuilder() lnks, err := lnkBuilder.BeginList(int64(len(s.children))) diff --git a/data/builder/file.go b/data/builder/file.go index 4a32209..dcf3b81 100644 --- a/data/builder/file.go +++ b/data/builder/file.go @@ -37,7 +37,7 @@ func BuildUnixFSFile(r io.Reader, chunker string, ls *ipld.LinkSystem) (ipld.Lin var prevLen []uint64 depth := 1 for { - root, size, err := fileTreeRecursive(depth, prev[:], prevLen[:], s, ls) + root, size, err := fileTreeRecursive(depth, prev, prevLen, s, ls) if err != nil { return nil, 0, err } @@ -126,20 +126,39 @@ func fileTreeRecursive(depth int, children []ipld.Link, childLen []uint64, src c // Pack into the dagpb node. dpbb := dagpb.Type.PBNode.NewBuilder() - pbm, _ := dpbb.BeginMap(2) - pblb, _ := pbm.AssembleEntry("Links") - pbl, _ := pblb.BeginList(int64(len(children))) + pbm, err := dpbb.BeginMap(2) + if err != nil { + return nil, 0, err + } + pblb, err := pbm.AssembleEntry("Links") + if err != nil { + return nil, 0, err + } + pbl, err := pblb.BeginList(int64(len(children))) + if err != nil { + return nil, 0, err + } for i, c := range children { pbln, err := BuildUnixFSDirectoryEntry("", int64(blksizes[i]), c) if err != nil { return nil, 0, err } - _ = pbl.AssembleValue().AssignNode(pbln) + if err = pbl.AssembleValue().AssignNode(pbln); err != nil { + return nil, 0, err + } + } + if err = pbl.Finish(); err != nil { + return nil, 0, err + } + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, 0, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)); err != nil { + return nil, 0, err + } + if err = pbm.Finish(); err != nil { + return nil, 0, err } - _ = pbl.Finish() - _ = pbm.AssembleKey().AssignString("Data") - _ = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)) - _ = pbm.Finish() pbn := dpbb.Build() link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn) @@ -210,12 +229,26 @@ func BuildUnixFSSymlink(content string, ls *ipld.LinkSystem) (ipld.Link, uint64, if err != nil { return nil, 0, err } - pblb, _ := pbm.AssembleEntry("Links") - pbl, _ := pblb.BeginList(0) - _ = pbl.Finish() - _ = pbm.AssembleKey().AssignString("Data") - _ = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)) - _ = pbm.Finish() + pblb, err := pbm.AssembleEntry("Links") + if err != nil { + return nil, 0, err + } + pbl, err := pblb.BeginList(0) + if err != nil { + return nil, 0, err + } + if err = pbl.Finish(); err != nil { + return nil, 0, err + } + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, 0, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)); err != nil { + return nil, 0, err + } + if err = pbm.Finish(); err != nil { + return nil, 0, err + } pbn := dpbb.Build() link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn)