Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: blake3 support #9073

Closed
wants to merge 8 commits into from
Closed

feat: blake3 support #9073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8523afa
bumping dep versions so we can have nice things (like blake3 support)
laudiacay Jul 1, 2022
ce1f804
it goes
laudiacay Jul 6, 2022
f61104c
bumping versions correctly
laudiacay Jul 28, 2022
7d4fe18
:|
laudiacay Jul 28, 2022
09daa52
adding some tests
laudiacay Aug 1, 2022
62b3a26
fixing some more tests
laudiacay Aug 1, 2022
ec5c00d
updating multihash version
laudiacay Aug 3, 2022
7b90c98
fixing tests after changing default blake3 length to 32 in multihash
laudiacay Aug 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
module github.com/ipfs/kubo

replace github.com/multiformats/go-multihash => github.com/laudiacay/go-multihash v0.2.1-0.20220803155855-c7de973b5b4d

replace github.com/ipfs/go-verifcid => github.com/laudiacay/go-verifcid v0.0.2-0.20220728135245-e9ad4d714376

require (
bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc
contrib.go.opencensus.io/exporter/prometheus v0.4.0
Expand Down
23 changes: 4 additions & 19 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,8 +409,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8=
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY=
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY=
Expand Down Expand Up @@ -652,8 +650,6 @@ github.com/ipfs/go-unixfs v0.4.0/go.mod h1:I7Nqtm06HgOOd+setAoCU6rf/HgVFHE+peeNu
github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
github.com/ipfs/go-unixfsnode v1.4.0 h1:9BUxHBXrbNi8mWHc6j+5C580WJqtVw9uoeEKn4tMhwA=
github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo=
github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPmI+GksBlLE8js=
github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY=
Expand Down Expand Up @@ -754,6 +750,10 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/laudiacay/go-multihash v0.2.1-0.20220803155855-c7de973b5b4d h1:+sUFJ2egeVsQW+ZLCvJ2XtI1XmqGvxajns3pkdKrkBk=
github.com/laudiacay/go-multihash v0.2.1-0.20220803155855-c7de973b5b4d/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
github.com/laudiacay/go-verifcid v0.0.2-0.20220728135245-e9ad4d714376 h1:Pyp7SjAvTCrubxo3pSYer4owOoUkQxn7nLHGkdDRR40=
github.com/laudiacay/go-verifcid v0.0.2-0.20220728135245-e9ad4d714376/go.mod h1:rgQU0PiIyeXLeQXgZSJMyHgyEwZT6i2HdfnXaSf5OLo=
github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E=
github.com/libp2p/go-addr-util v0.1.0/go.mod h1:6I3ZYuFr2O/9D+SoyM0zEw0EF3YkldtTX406BpdQMqw=
Expand Down Expand Up @@ -1264,18 +1264,6 @@ github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mo
github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs=
github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues=
github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag=
github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
github.com/multiformats/go-multihash v0.2.0 h1:oytJb9ZA1OUW0r0f9ea18GiaPOo4SXyc7p2movyUuo4=
github.com/multiformats/go-multihash v0.2.0/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
Expand Down Expand Up @@ -1670,12 +1658,10 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down Expand Up @@ -1843,7 +1829,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down
4 changes: 2 additions & 2 deletions test/sharness/t0275-cid-security.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ test_expect_success "adding using too short of a hash function gives out an erro
'

test_expect_success "error reason is pointed out" '
grep "hashes must be at 20 least bytes long" block_out
grep "hashes must be at least 20 bytes long" block_out
'


Expand All @@ -45,7 +45,7 @@ test_cat_get() {
'

test_expect_success "error reason is pointed out" '
grep "hashes must be at 20 least bytes long" ipfs_get
grep "hashes must be at least 20 bytes long" ipfs_get
'
}

Expand Down
1 change: 1 addition & 0 deletions test/sharness/t0290-cid.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ cat <<EOF > hashes_expect
27 keccak-256
28 keccak-384
29 keccak-512
30 blake3
86 dbl-sha2-256
45588 blake2b-160
45589 blake2b-168
Expand Down
81 changes: 81 additions & 0 deletions test/sharness/t0800-blake3.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
#
# Copyright (c) 2020 Claudia Richoux
# MIT Licensed; see the LICENSE file in this repository.
#

test_description="Test blake3 mhash support"

. lib/test-lib.sh

test_init_ipfs

# the blake3 hash of "foo\n" in UTF8 (which is what comes out of echo when you pipe into `ipfs`) starts with "49dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92"
# without the newline it's "04e0bb39f30b1a3feb89f536c93be15055482df748674b00d26e5a75777702e9". so if you start seeing these values that's your problem
BLAKE3RAWCID32BYTE="bafkr4icj3sdq34o6p7lapfgoxtset5om3lsxll72uz5cjnrkzmb6aoo3si"
BLAKE3RAWCID64BYTE="bafkr4qcj3sdq34o6p7lapfgoxtset5om3lsxll72uz5cjnrkzmb6aoo3sknmbprpe27pbfrb67tonydgfot5ixuq4skiva76ppbgpjlzc4ua4"
BLAKE3RAWCID128BYTE="bafkr5aabjhoiodpr3z75mb4uz26oispvztnok5np7kthujfwflfqhybz3ojjvqf6f4tl54eweh36nzxamyv2pvc6sdsjjcud7z54ez5fpelsqdtax2k3rvuq3wdl5a4blxv3gvsroa3nakfzzknamhu2apf3vvytyiobabrn2bfnfajq66ikjy5lewsp5jyddsg5l7u3emr2ancimryay"

### block tests, including for various sizes of hash ###

test_expect_success "putting a block with an mhash blake3 succeeds (default 32 bytes)" '
HASH=$(echo "foo" | ipfs block put --mhtype=blake3 --cid-codec=raw | tee actual_out) &&
test $BLAKE3RAWCID32BYTE = "$HASH"
'

test_expect_success "block get output looks right" '
ipfs block get $BLAKE3RAWCID32BYTE > blk_get_out &&
echo "foo" > blk_get_exp &&
test_cmp blk_get_exp blk_get_out
'

test_expect_success "putting a block with an mhash blake3 succeeds: 64 bytes" '
HASH=$(echo "foo" | ipfs block put --mhtype=blake3 --mhlen=64 --cid-codec=raw | tee actual_out) &&
test $BLAKE3RAWCID64BYTE = "$HASH"
'

test_expect_success "64B block get output looks right" '
ipfs block get $BLAKE3RAWCID64BYTE > blk_get_out &&
echo "foo" > blk_get_exp &&
test_cmp blk_get_exp blk_get_out
'

test_expect_success "putting a block with an mhash blake3 succeeds: 128 bytes" '
HASH=$(echo "foo" | ipfs block put --mhtype=blake3 --mhlen=128 --cid-codec=raw | tee actual_out) &&
test $BLAKE3RAWCID128BYTE = "$HASH"
'

test_expect_success "32B block get output looks right" '
ipfs block get $BLAKE3RAWCID128BYTE > blk_get_out &&
echo "foo" > blk_get_exp &&
test_cmp blk_get_exp blk_get_out
'

### dag tests ###

test_expect_success "dag put works with blake3" '
HASH=$(echo "foo" | ipfs dag put --input-codec=raw --store-codec=raw --hash=blake3 | tee actual_out) &&
test $BLAKE3RAWCID32BYTE = "$HASH"
'

test_expect_success "dag get output looks right" '
ipfs dag get --output-codec=raw $BLAKE3RAWCID32BYTE > dag_get_out &&
echo "foo" > dag_get_exp &&
test_cmp dag_get_exp dag_get_out
'

### add and cat tests ###

test_expect_success "adding a file with just foo in it to ipfs" '
echo "foo" > afile &&
HASH=$(ipfs add -q --hash=blake3 --raw-leaves afile | tee actual_out) &&
test $BLAKE3RAWCID32BYTE = "$HASH"
'

test_expect_success "catting it" '
ipfs cat $BLAKE3RAWCID32BYTE > cat_out &&
echo "foo" > cat_exp &&
test_cmp cat_exp cat_out
'

test_done