Merge fixes from v0.13.4

cmd/start.go

@@ -93,6 +93,8 @@ type Start struct {
 	Storage              string   `long:"storage" description:"set the outgoing message storage option [self-hosted, dropbox] default=self-hosted"`
 	BitcoinCash          bool     `long:"bitcoincash" description:"use a Bitcoin Cash wallet in a dedicated data directory"`
 	ZCash                string   `long:"zcash" description:"use a ZCash wallet in a dedicated data directory. To use this you must pass in the location of the zcashd binary."`
+
+	ForceKeyCachePurge bool `long:"forcekeypurge" description:"repair test for issue OpenBazaar/openbazaar-go#1593; use as instructed only"`
 }
 
 func (x *Start) Execute(args []string) error {
@@ -430,6 +432,11 @@ func (x *Start) Execute(args []string) error {
 		nd.Repo.Datastore().Delete(ipnskey)
 	}
 
+	if x.ForceKeyCachePurge {
+		log.Infof("forcing key purge from namesys cache...")
+		nd.Repo.Datastore().Delete(ipnskey)
+	}
+
 	// Wallet
 	mn, err := sqliteDB.Config().GetMnemonic()
 	if err != nil {

ipfs/resolve.go

@@ -2,20 +2,26 @@ package ipfs
 
 import (
 	"context"
-
 	ds "gx/ipfs/QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9/go-datastore"
 	"gx/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h/go-libp2p-peer"
 
 	"time"
 
 	ipath "gx/ipfs/QmQAgv6Gaoe2tQpcabqwKXKChp2MZ7i3UXv9DqTTaxCaTR/go-path"
+	ipnspb "gx/ipfs/QmUwMnKKjH3JwGKNVZ3TcP37W93xzqNA4ECFFiMo6sXkkc/go-ipns/pb"
+	"gx/ipfs/QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB/gogo-protobuf/proto"
+	"gx/ipfs/QmfVj3x4D6Jkq9SEoi5n2NmoUomLwoeiwnYz2KQa15wRw6/base32"
 
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/namesys"
 
 	nameopts "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options/namesys"
 )
 
+const (
+	persistentCacheDbPrefix = "/ipns/persistentcache/"
+)
+
 // Resolve an IPNS record. This is a multi-step process.
 // If the usecache flag is provided we will attempt to load the record from the database. If
 // it succeeds we will update the cache in a separate goroutine.
@@ -37,7 +43,7 @@ func Resolve(n *core.IpfsNode, p peer.ID, timeout time.Duration, quorum uint, us
 					return
 				}
 				if n.Identity != p {
-					if err := putToDatastore(n.Repo.Datastore(), p, pth); err != nil {
+					if err := putToDatastoreCache(n.Repo.Datastore(), p, pth); err != nil {
 						log.Error("Error putting IPNS record to datastore: %s", err.Error())
 					}
 				}
@@ -56,7 +62,7 @@ func Resolve(n *core.IpfsNode, p peer.ID, timeout time.Duration, quorum uint, us
 	}
 	// Resolving succeeded. Update the cache.
 	if n.Identity != p {
-		if err := putToDatastore(n.Repo.Datastore(), p, pth); err != nil {
+		if err := putToDatastoreCache(n.Repo.Datastore(), p, pth); err != nil {
 			log.Error("Error putting IPNS record to datastore: %s", err.Error())
 		}
 	}
@@ -87,18 +93,36 @@ func ResolveAltRoot(n *core.IpfsNode, p peer.ID, altRoot string, timeout time.Du
 	return pth.Segments()[1], nil
 }
 
+// getFromDatastore looks in two places in the database for a record. First is
+// under the /ipfs/<peerID> key which is sometimes used by the DHT. The value
+// returned by this location is a serialized protobuf record. The second is
+// under /ipfs/persistentcache/<peerID> which returns only the value (the path)
+// inside the protobuf record.
 func getFromDatastore(datastore ds.Datastore, p peer.ID) (ipath.Path, error) {
-	// resolve to what we may already have in the datastore
-	data, err := datastore.Get(namesys.IpnsDsKey(p))
+	ival, err := datastore.Get(namesys.IpnsDsKey(p))
 	if err != nil {
-		if err == ds.ErrNotFound {
-			return "", namesys.ErrResolveFailed
+		pth, err := datastore.Get(ipnsCacheDsKey(p))
+		if err != nil {
+			if err == ds.ErrNotFound {
+				return "", namesys.ErrResolveFailed
+			}
+			return "", err
 		}
+		return ipath.ParsePath(string(pth))
+	}
+
+	rec := new(ipnspb.IpnsEntry)
+	err = proto.Unmarshal(ival, rec)
+	if err != nil {
 		return "", err
 	}
-	return ipath.ParsePath(string(data))
+	return ipath.ParsePath(string(rec.Value))
+}
+
+func putToDatastoreCache(datastore ds.Datastore, p peer.ID, pth ipath.Path) error {
+	return datastore.Put(ipnsCacheDsKey(p), []byte(pth.String()))
 }
 
-func putToDatastore(datastore ds.Datastore, p peer.ID, pth ipath.Path) error {
-	return datastore.Put(namesys.IpnsDsKey(p), []byte(pth.String()))
+func ipnsCacheDsKey(id peer.ID) ds.Key {
+	return ds.NewKey(persistentCacheDbPrefix + base32.RawStdEncoding.EncodeToString([]byte(id)))
 }

ipfs/resolve_test.go

@@ -0,0 +1,79 @@
+package ipfs
+
+import (
+	"github.com/ipfs/go-ipfs/core/mock"
+	"github.com/ipfs/go-ipfs/namesys"
+	ipath "gx/ipfs/QmQAgv6Gaoe2tQpcabqwKXKChp2MZ7i3UXv9DqTTaxCaTR/go-path"
+	ipnspb "gx/ipfs/QmUwMnKKjH3JwGKNVZ3TcP37W93xzqNA4ECFFiMo6sXkkc/go-ipns/pb"
+	"gx/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h/go-libp2p-peer"
+	"gx/ipfs/QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB/gogo-protobuf/proto"
+	"testing"
+)
+
+// TestDatastoreCaching tests that the data can be inserted and retrieved from the
+// database using the IPNS namespace as well as our persistent cache namespace and
+// that the two do not conflict with each other.
+func TestDatastoreCaching(t *testing.T) {
+	n, err := coremock.NewMockNode()
+	if err != nil {
+		t.Error(err)
+	}
+	var (
+		pth       = "/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h"
+		rec       = &ipnspb.IpnsEntry{Value: []byte(pth), Signature: []byte{}}
+		peerIDStr = "QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB"
+	)
+
+	peerID, err := peer.IDB58Decode(peerIDStr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// First put to db using the IPNS namespace.
+	serializedRecord, err := proto.Marshal(rec)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := n.Repo.Datastore().Put(namesys.IpnsDsKey(peerID), serializedRecord); err != nil {
+		t.Fatal(err)
+	}
+
+	retreivedPath, err := getFromDatastore(n.Repo.Datastore(), peerID)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if retreivedPath.String() != pth {
+		t.Errorf("Retreived incorrect value. Expected %s, got %s", pth, retreivedPath.String())
+	}
+
+	// Next put to the database using the persistent cache namespace.
+	if err := putToDatastoreCache(n.Repo.Datastore(), peerID, ipath.Path(pth)); err != nil {
+		t.Fatal(err)
+	}
+
+	retreivedPath, err = getFromDatastore(n.Repo.Datastore(), peerID)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if retreivedPath.String() != pth {
+		t.Errorf("Retreived incorrect value. Expected %s, got %s", pth, retreivedPath.String())
+	}
+
+	// Test the persistent cache put did not override the IPNS namespace.
+	ival, err := n.Repo.Datastore().Get(namesys.IpnsDsKey(peerID))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rec2 := new(ipnspb.IpnsEntry)
+	err = proto.Unmarshal(ival, rec2)
+	if err != nil {
+		t.Error(err)
+	}
+	if string(rec2.Value) != pth {
+		t.Errorf("Retreived incorrect value. Expected %s, got %s", pth, string(rec2.Value))
+	}
+}

vendor/gx/ipfs/QmSVaJe1aRjc78cZARTtf4pqvXERYwihyYhZWoVWceHnsK/go-libp2p-secio/rw.go

@@ -167,7 +167,9 @@ func (r *etmReader) Read(buf []byte) (int, error) {
 
 	// If the destination buffer is too short, fill an internal buffer and then
 	// drain as much of that into the output buffer as will fit.
-	if cap(buf) < fullLen {
+	// Applies patch from v0.0.3 release, see
+	// https://github.com/libp2p/go-libp2p-secio/commit/98b8bd9f149802621eeca00d5a606f6788973038
+	if len(buf) < fullLen {
 		err := r.fill()
 		if err != nil {
 			return 0, err