Merge pull request #2904 from ipfs/feature/repo-runtime-check

Add option for datastore read rehashing

blocks/blockstore/blockstore.go

@@ -22,7 +22,8 @@ var log = logging.Logger("blockstore")
 // BlockPrefix namespaces blockstore datastores
 var BlockPrefix = ds.NewKey("blocks")
 
-var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
+var ValueTypeMismatch = errors.New("the retrieved value is not a Block")
+var ErrHashMismatch = errors.New("block in storage has different hash than requested")
 
 var ErrNotFound = errors.New("blockstore: block not found")
 
@@ -71,6 +72,12 @@ type blockstore struct {
 	lk      sync.RWMutex
 	gcreq   int32
 	gcreqlk sync.Mutex
+
+	rehash bool
+}
+
+func (bs *blockstore) RuntimeHashing(enabled bool) {
+	bs.rehash = enabled
 }
 
 func (bs *blockstore) Get(k key.Key) (blocks.Block, error) {
@@ -90,7 +97,16 @@ func (bs *blockstore) Get(k key.Key) (blocks.Block, error) {
 		return nil, ValueTypeMismatch
 	}
 
-	return blocks.NewBlockWithHash(bdata, mh.Multihash(k))
+	if bs.rehash {
+		rb := blocks.NewBlock(bdata)
+		if rb.Key() != k {
+			return nil, ErrHashMismatch
+		} else {
+			return rb, nil
+		}
+	} else {
+		return blocks.NewBlockWithHash(bdata, mh.Multihash(k))
+	}
 }
 
 func (bs *blockstore) Put(block blocks.Block) error {

blocks/blockstore/blockstore_test.go

@@ -53,6 +53,22 @@ func TestPutThenGetBlock(t *testing.T) {
 	}
 }
 
+func TestRuntimeHashing(t *testing.T) {
+	bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+	bl := blocks.NewBlock([]byte("some data"))
+	blBad, err := blocks.NewBlockWithHash([]byte("some other data"), bl.Key().ToMultihash())
+	if err != nil {
+		t.Fatal("Debug is enabled")
+	}
+
+	bs.Put(blBad)
+	bs.RuntimeHashing(true)
+
+	if _, err := bs.Get(bl.Key()); err != ErrHashMismatch {
+		t.Fatalf("Expected '%v' got '%v'\n", ErrHashMismatch, err)
+	}
+}
+
 func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []key.Key) {
 	if d == nil {
 		d = ds.NewMapDatastore()

core/builder.go

@@ -130,16 +130,22 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 	}
 
 	var err error
-	n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Repo.Datastore()), kSizeBlockstoreWriteCache)
+	bs := bstore.NewBlockstore(n.Repo.Datastore())
+	n.Blockstore, err = bstore.WriteCached(bs, kSizeBlockstoreWriteCache)
 	if err != nil {
 		return err
 	}
 
+	rcfg, err := n.Repo.Config()
+	if err != nil {
+		return err
+	}
+
+	if rcfg.Datastore.HashOnRead {
+		bs.RuntimeHashing(true)
+	}
+
 	if cfg.Online {
-		rcfg, err := n.Repo.Config()
-		if err != nil {
-			return err
-		}
 		do := setupDiscoveryOption(rcfg.Discovery)
 		if err := n.startOnlineServices(ctx, cfg.Routing, cfg.Host, do); err != nil {
 			return err

repo/config/datastore.go

@@ -15,8 +15,9 @@ type Datastore struct {
 	StorageGCWatermark int64  // in percentage to multiply on StorageMax
 	GCPeriod           string // in ns, us, ms, s, m, h
 
-	Params *json.RawMessage
-	NoSync bool
+	Params     *json.RawMessage
+	NoSync     bool
+	HashOnRead bool
 }
 
 func (d *Datastore) ParamData() []byte {

repo/config/init.go

@@ -84,6 +84,7 @@ func datastoreConfig() (Datastore, error) {
 		StorageMax:         "10GB",
 		StorageGCWatermark: 90, // 90%
 		GCPeriod:           "1h",
+		HashOnRead:         false,
 	}, nil
 }
 

test/sharness/t0084-repo-read-rehash.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+#
+# Copyright (c) Jakub Sztandera
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test ipfs blockstore repo read check."
+
+. lib/test-lib.sh
+
+rm -rf "$IPF_PATH/*"
+
+test_init_ipfs
+
+
+H_BLOCK1=$(echo "Block 1" | ipfs add -q)
+H_BLOCK2=$(echo "Block 2" | ipfs add -q)
+
+BS_BLOCK1="1220f18e/1220f18e07ebc69997909358f28b9d2c327eb032b0afab6bbc7fd7f399a7b7590be4.data"
+BS_BLOCK2="1220dc58/1220dc582e51f1f98b1f2d1c1baaa9f7b11602239ed42fbdf8f52d67e63cc03df12a.data"
+
+
+test_expect_success 'blocks are swapped' '
+	ipfs cat $H_BLOCK2 > noswap &&
+	cp -f "$IPFS_PATH/blocks/$BS_BLOCK1" "$IPFS_PATH/blocks/$BS_BLOCK2" &&
+	ipfs cat $H_BLOCK2 > swap &&
+	test_must_fail test_cmp noswap swap
+'
+
+ipfs config --bool Datastore.HashOnRead true
+
+test_expect_success 'getting modified block fails' '
+	(test_must_fail ipfs cat $H_BLOCK2 2> err_msg) &&
+	grep "block in storage has different hash than requested" err_msg
+'
+
+test_done