Merge pull request #13 from ipfs/fix/only-roots

fix reproviding only roots

go.mod

@@ -5,19 +5,22 @@ go 1.12
 require (
 	github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50 // indirect
 	github.com/cenkalti/backoff v2.1.1+incompatible
-	github.com/ipfs/go-block-format v0.0.2
+	github.com/ipfs/go-blockservice v0.1.0
 	github.com/ipfs/go-cid v0.0.2
 	github.com/ipfs/go-cidutil v0.0.2
 	github.com/ipfs/go-datastore v0.0.5
 	github.com/ipfs/go-ipfs-blockstore v0.0.1
 	github.com/ipfs/go-ipfs-blocksutil v0.0.1
+	github.com/ipfs/go-ipfs-exchange-offline v0.0.1
 	github.com/ipfs/go-ipfs-routing v0.1.0
+	github.com/ipfs/go-ipld-cbor v0.0.2
 	github.com/ipfs/go-ipld-format v0.0.2
 	github.com/ipfs/go-log v0.0.1
 	github.com/ipfs/go-merkledag v0.2.0
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/libp2p/go-libp2p-core v0.0.3
 	github.com/libp2p/go-libp2p-testing v0.0.4
+	github.com/multiformats/go-multihash v0.0.5
 	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 // indirect
 	golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae // indirect
 )

simple/reprovide.go

@@ -213,7 +213,9 @@ func pinSet(ctx context.Context, pinning Pinner, dag ipld.DAGService, onlyRoots
 		}
 
 		for _, key := range pinning.RecursiveKeys() {
-			if !onlyRoots {
+			if onlyRoots {
+				set.Visitor(ctx)(key)
+			} else {
 				err := merkledag.Walk(ctx, merkledag.GetLinksWithDAG(dag), key, set.Visitor(ctx))
 				if err != nil {
 					logR.Errorf("reprovide indirect pins: %s", err)

simple/reprovide_test.go

@@ -5,57 +5,160 @@ import (
 	"testing"
 	"time"
 
-	blocks "github.com/ipfs/go-block-format"
+	bsrv "github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
 	"github.com/ipfs/go-ipfs-blockstore"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
 	mock "github.com/ipfs/go-ipfs-routing/mock"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	merkledag "github.com/ipfs/go-merkledag"
 	peer "github.com/libp2p/go-libp2p-core/peer"
 	testutil "github.com/libp2p/go-libp2p-testing/net"
+	mh "github.com/multiformats/go-multihash"
 
 	. "github.com/ipfs/go-ipfs-provider/simple"
 )
 
-func TestReprovide(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
+func setupRouting(t *testing.T) (clA, clB mock.Client, idA, idB peer.ID) {
 	mrserv := mock.NewServer()
 
-	idA := testutil.RandIdentityOrFatal(t)
-	idB := testutil.RandIdentityOrFatal(t)
+	iidA := testutil.RandIdentityOrFatal(t)
+	iidB := testutil.RandIdentityOrFatal(t)
 
-	clA := mrserv.Client(idA)
-	clB := mrserv.Client(idB)
+	clA = mrserv.Client(iidA)
+	clB = mrserv.Client(iidB)
 
-	bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
+	return clA, clB, iidA.ID(), iidB.ID()
+}
 
-	blk := blocks.NewBlock([]byte("this is a test"))
-	err := bstore.Put(blk)
-	if err != nil {
-		t.Fatal(err)
+func setupDag(t *testing.T) (nodes []cid.Cid, bstore blockstore.Blockstore) {
+	bstore = blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
+	for _, data := range []string{"foo", "bar"} {
+		blk, err := cbor.WrapObject(data, mh.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = bstore.Put(blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+		nodes = append(nodes, blk.Cid())
+
+		blk, err = cbor.WrapObject(map[string]interface{}{
+			"child": blk.Cid(),
+		}, mh.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = bstore.Put(blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+		nodes = append(nodes, blk.Cid())
 	}
 
+	return nodes, bstore
+}
+
+func TestReprovide(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	clA, clB, idA, _ := setupRouting(t)
+	nodes, bstore := setupDag(t)
+
 	keyProvider := NewBlockstoreProvider(bstore)
 	reprov := NewReprovider(ctx, time.Hour, clA, keyProvider)
-	err = reprov.Reprovide()
+	err := reprov.Reprovide()
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	var providers []peer.AddrInfo
 	maxProvs := 100
 
-	provChan := clB.FindProvidersAsync(ctx, blk.Cid(), maxProvs)
-	for p := range provChan {
-		providers = append(providers, p)
-	}
+	for _, c := range nodes {
+		provChan := clB.FindProvidersAsync(ctx, c, maxProvs)
+		for p := range provChan {
+			providers = append(providers, p)
+		}
 
-	if len(providers) == 0 {
-		t.Fatal("Should have gotten a provider")
+		if len(providers) == 0 {
+			t.Fatal("Should have gotten a provider")
+		}
+
+		if providers[0].ID != idA {
+			t.Fatal("Somehow got the wrong peer back as a provider.")
+		}
 	}
+}
+
+type mockPinner struct {
+	recursive []cid.Cid
+	direct    []cid.Cid
+}
+
+func (mp *mockPinner) DirectKeys() []cid.Cid {
+	return mp.direct
+}
+
+func (mp *mockPinner) RecursiveKeys() []cid.Cid {
+	return mp.recursive
+}
+
+func TestReprovidePinned(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	nodes, bstore := setupDag(t)
+
+	dag := merkledag.NewDAGService(bsrv.New(bstore, offline.Exchange(bstore)))
+
+	for i := 0; i < 2; i++ {
+		clA, clB, idA, _ := setupRouting(t)
+
+		onlyRoots := i == 0
+		t.Logf("only roots: %v", onlyRoots)
+
+		var provide, dont []cid.Cid
+		if onlyRoots {
+			provide = []cid.Cid{nodes[1], nodes[3]}
+			dont = []cid.Cid{nodes[0], nodes[2]}
+		} else {
+			provide = []cid.Cid{nodes[0], nodes[1], nodes[3]}
+			dont = []cid.Cid{nodes[2]}
+		}
+
+		keyProvider := NewPinnedProvider(onlyRoots, &mockPinner{
+			recursive: []cid.Cid{nodes[1]},
+			direct:    []cid.Cid{nodes[3]},
+		}, dag)
+
+		reprov := NewReprovider(ctx, time.Hour, clA, keyProvider)
+		err := reprov.Reprovide()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for i, c := range provide {
+			prov, ok := <-clB.FindProvidersAsync(ctx, c, 1)
+			if !ok {
+				t.Errorf("Should have gotten a provider for %d", i)
+				continue
+			}
 
-	if providers[0].ID != idA.ID() {
-		t.Fatal("Somehow got the wrong peer back as a provider.")
+			if prov.ID != idA {
+				t.Errorf("Somehow got the wrong peer back as a provider.")
+				continue
+			}
+		}
+		for i, c := range dont {
+			prov, ok := <-clB.FindProvidersAsync(ctx, c, 1)
+			if ok {
+				t.Fatalf("found provider %s for %d, expected none", prov.ID, i)
+			}
+		}
 	}
 }