Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: retrieval trustless #486

Merged
merged 3 commits into from
Nov 29, 2023
Merged

feat: retrieval trustless #486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e624d65
feat: trushless handler
simlecode Oct 10, 2023
aa901d1
test: add test for trustless handler
simlecode Nov 13, 2023
3a82908
chore: fix lint
simlecode Nov 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/droplet/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ func runDaemon(cctx *cli.Context) error {
if err = router.Handle("/resource", rpc.NewPieceStorageServer(resAPI.PieceStorageMgr)).GetError(); err != nil {
return fmt.Errorf("handle 'resource' failed: %w", err)
}
httpRetrievalServer, err := httpretrieval.NewServer(resAPI.PieceStorageMgr, resAPI)
httpRetrievalServer, err := httpretrieval.NewServer(ctx, resAPI.PieceStorageMgr, resAPI, resAPI.DAGStoreWrapper)
if err != nil {
return err
}
Expand Down
21 changes: 20 additions & 1 deletion dagstore/mock_dagstore_wrapper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
type Registration struct {
CarPath string
EagerInit bool
Index carindex.Index
}

// MockDagStoreWrapper is used to mock out the DAG store wrapper operations
Expand All @@ -42,6 +43,20 @@ func NewMockDagStoreWrapper() *MockDagStoreWrapper {
}
}

func (m *MockDagStoreWrapper) RegisterShardWithIndex(ctx context.Context, pieceCid cid.Cid, carPath string, eagerInit bool, resch chan dagstore.ShardResult, index carindex.Index) error {
m.lk.Lock()
defer m.lk.Unlock()

m.registrations[pieceCid] = Registration{
CarPath: carPath,
EagerInit: eagerInit,
Index: index,
}

resch <- dagstore.ShardResult{}
return nil
}

func (m *MockDagStoreWrapper) RegisterShard(ctx context.Context, pieceCid cid.Cid, carPath string, eagerInit bool, resch chan dagstore.ShardResult) error {
m.lk.Lock()
defer m.lk.Unlock()
Expand Down Expand Up @@ -98,7 +113,11 @@ func (m *MockDagStoreWrapper) LoadShard(ctx context.Context, pieceCid cid.Cid) (
if err != nil {
return nil, err
}
return getBlockstoreFromReader(fileR, pieceCid)
if pieceInfo.Index == nil {
return getBlockstoreFromReader(fileR, pieceCid)
}

return blockstore.NewReadOnly(fileR, pieceInfo.Index, carv2.ZeroLengthSectionAsEOF(true), blockstore.UseWholeCIDs(true))
}

func (m *MockDagStoreWrapper) DestroyShard(ctx context.Context, pieceCid cid.Cid, resch chan dagstore.ShardResult) error {
Expand Down
109 changes: 53 additions & 56 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/ipfs-force-community/droplet/v2
go 1.18

require (
github.com/BurntSushi/toml v1.2.0
github.com/BurntSushi/toml v1.3.2
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
Expand All @@ -18,7 +18,7 @@ require (
github.com/filecoin-project/go-bitfield v0.2.4
github.com/filecoin-project/go-cbor-util v0.0.1
github.com/filecoin-project/go-commp-utils v0.1.3
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc6
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7
github.com/filecoin-project/go-ds-versioning v0.1.2
github.com/filecoin-project/go-fil-commcid v0.1.0
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
Expand All @@ -30,7 +30,7 @@ require (
github.com/filecoin-project/go-statestore v0.2.0
github.com/filecoin-project/specs-actors/v2 v2.3.6
github.com/filecoin-project/specs-actors/v7 v7.0.1
github.com/filecoin-project/venus v1.14.0
github.com/filecoin-project/venus v1.14.1-0.20231128081003-8733a7eac76b
github.com/golang/mock v1.6.0
github.com/google/uuid v1.3.0
github.com/gorilla/mux v1.8.0
Expand All @@ -42,64 +42,56 @@ require (
github.com/ipfs-force-community/sophon-gateway v1.14.0
github.com/ipfs-force-community/sophon-messager v1.14.0
github.com/ipfs-force-community/venus-common-utils v0.0.0-20220217030526-e5e4c6bc14f7
github.com/ipfs/boxo v0.10.1
github.com/ipfs/go-blockservice v0.5.0
github.com/ipfs/boxo v0.12.0
github.com/ipfs/go-blockservice v0.5.1
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-cidutil v0.1.0
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-badger2 v0.1.3
github.com/ipfs/go-ds-leveldb v0.5.0
github.com/ipfs/go-ds-measure v0.2.0
github.com/ipfs/go-graphsync v0.14.6
github.com/ipfs/go-graphsync v0.15.1
github.com/ipfs/go-ipfs-blocksutil v0.0.1
github.com/ipfs/go-ipfs-chunker v0.0.5
github.com/ipfs/go-ipfs-exchange-offline v0.3.0
github.com/ipfs/go-ipld-cbor v0.0.6
github.com/ipfs/go-ipld-format v0.5.0
github.com/ipfs/go-ipld-cbor v0.1.0
github.com/ipfs/go-ipld-format v0.6.0
github.com/ipfs/go-libipfs v0.7.0
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-merkledag v0.11.0
github.com/ipfs/go-metrics-interface v0.0.1
github.com/ipfs/go-unixfs v0.4.5
github.com/ipld/frisbii v0.4.1
github.com/ipld/go-car v0.6.1
github.com/ipld/go-car/v2 v2.10.1
github.com/ipld/go-car/v2 v2.13.1
github.com/ipld/go-codec-dagpb v1.6.0
github.com/ipld/go-ipld-prime v0.20.0
github.com/ipld/go-ipld-prime v0.21.0
github.com/ipld/go-ipld-selector-text-lite v0.0.1
github.com/libp2p/go-libp2p v0.27.8
github.com/libp2p/go-libp2p v0.31.0
github.com/libp2p/go-maddr-filter v0.1.0
github.com/mitchellh/go-homedir v1.1.0
github.com/multiformats/go-multiaddr v0.9.0
github.com/multiformats/go-multiaddr v0.11.0
github.com/multiformats/go-multibase v0.2.0
github.com/multiformats/go-multihash v0.2.3
github.com/multiformats/go-varint v0.0.7
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.8.4
github.com/strikesecurity/strikememongo v0.2.4
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
github.com/urfave/cli/v2 v2.16.3
github.com/urfave/cli/v2 v2.25.7
github.com/whyrusleeping/cbor-gen v0.0.0-20230923211252-36a87e1ba72f
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
go.mongodb.org/mongo-driver v1.8.4
go.opencensus.io v0.24.0
go.uber.org/fx v1.19.2
go.uber.org/fx v1.20.0
go.uber.org/multierr v1.11.0
golang.org/x/sync v0.1.0
golang.org/x/sync v0.3.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
gorm.io/driver/mysql v1.3.5
gorm.io/driver/sqlite v1.1.4
gorm.io/gorm v1.23.8
)

require (
github.com/hashicorp/golang-lru/arc/v2 v2.0.7 // indirect
go.opentelemetry.io/otel/bridge/opencensus v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.opentelemetry.io/otel/sdk v1.16.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.39.0 // indirect
)

require (
contrib.go.opencensus.io/exporter/graphite v0.0.0-20200424223504-26b90655e0ce // indirect
contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
Expand All @@ -109,7 +101,7 @@ require (
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/awnumar/memcall v0.0.0-20191004114545-73db50fd9f80 // indirect
github.com/awnumar/memguard v0.22.2 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bep/debounce v1.2.1 // indirect
github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 // indirect
Expand All @@ -123,7 +115,7 @@ require (
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/deepmap/oapi-codegen v1.3.13 // indirect
github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
github.com/dgraph-io/badger/v3 v3.2103.5 // indirect
Expand Down Expand Up @@ -173,21 +165,22 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b // indirect
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/golang-lru/arc/v2 v2.0.7 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huin/goupnp v1.1.0 // indirect
github.com/huin/goupnp v1.2.0 // indirect
github.com/influxdata/influxdb-client-go/v2 v2.2.2 // indirect
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect
github.com/ipfs/go-block-format v0.1.2
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
github.com/ipfs/go-ipfs-blockstore v1.3.0
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
github.com/ipfs/go-ipfs-files v0.3.0 // indirect
Expand All @@ -197,11 +190,12 @@ require (
github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-unixfsnode v1.7.1 // indirect
github.com/ipfs/go-unixfsnode v1.9.0 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
github.com/ipni/go-libipni v0.0.7 // indirect
github.com/ipni/index-provider v0.12.0 // indirect
github.com/ipld/go-trustless-utils v0.4.1 // indirect
github.com/ipni/go-libipni v0.5.2 // indirect
github.com/ipni/index-provider v0.14.2 // indirect
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
Expand All @@ -212,7 +206,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.6 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
Expand All @@ -222,18 +216,18 @@ require (
github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect
github.com/libp2p/go-msgio v0.3.0 // indirect
github.com/libp2p/go-nat v0.1.0 // indirect
github.com/libp2p/go-nat v0.2.0 // indirect
github.com/libp2p/go-netroute v0.2.1 // indirect
github.com/libp2p/go-reuseport v0.2.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
github.com/libp2p/go-reuseport v0.4.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
github.com/magefile/mage v1.11.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-sqlite3 v1.14.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/miekg/dns v1.1.53 // indirect
github.com/miekg/dns v1.1.55 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
Expand All @@ -248,8 +242,8 @@ require (
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multicodec v0.9.0 // indirect
github.com/multiformats/go-multistream v0.4.1 // indirect
github.com/onsi/ginkgo/v2 v2.9.2 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/onsi/ginkgo/v2 v2.11.0 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
Expand All @@ -259,16 +253,15 @@ require (
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/prometheus/statsd_exporter v0.23.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-19 v0.3.3 // indirect
github.com/quic-go/qtls-go1-20 v0.2.3 // indirect
github.com/quic-go/quic-go v0.33.0 // indirect
github.com/quic-go/webtransport-go v0.5.2 // indirect
github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
github.com/quic-go/quic-go v0.38.1 // indirect
github.com/quic-go/webtransport-go v0.5.3 // indirect
github.com/raulk/clock v1.1.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
Expand All @@ -292,28 +285,32 @@ require (
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/bridge/opencensus v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.opentelemetry.io/otel/sdk v1.16.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.39.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/dig v1.16.1 // indirect
go.uber.org/zap v1.24.0
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.0 // indirect
go.uber.org/zap v1.25.0
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.1.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
google.golang.org/api v0.103.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.66.6 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
nhooyr.io/websocket v1.8.7 // indirect
)

replace (
Expand Down
Loading