Skip to content

Commit 25b5ece

Browse files
gammazerogammazero
committed
Read schema type hint from sync request
Head the schema type hint from the IPNI sync request header, and if provided, compare it against the type of data retrieved.
1 parent 69eb980 commit 25b5ece

File tree

3 files changed

+34
-15
lines changed

3 files changed

+34
-15
lines changed

‎engine/linksystem.go

+19
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/ipld/go-ipld-prime/codec/dagjson"
1111
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
1212
"github.com/ipld/go-ipld-prime/node/basicnode"
13+
"github.com/ipni/go-libipni/dagsync/ipnisync"
1314
"github.com/ipni/go-libipni/ingest/schema"
1415
provider "github.com/ipni/index-provider"
1516
"github.com/libp2p/go-libp2p/core/peer"
@@ -36,6 +37,14 @@ func (e *Engine) mkLinkSystem() ipld.LinkSystem {
3637
c := lnk.(cidlink.Link).Cid
3738
log.Debugf("Triggered ReadOpener from engine's linksystem with cid (%s)", c)
3839

40+
cidSchemaType, err := ipnisync.CidSchemaFromCtx(ctx)
41+
if err != nil {
42+
log.Errorf("CID schema type hint error: %s", err)
43+
}
44+
if cidSchemaType != "" {
45+
log.Infof("indexer requested %s", cidSchemaType)
46+
}
47+
3948
// Get the node from main datastore. If it is in the
4049
// main datastore it means it is an advertisement.
4150
val, err := e.ds.Get(ctx, datastore.NewKey(c.String()))
@@ -52,14 +61,24 @@ func (e *Engine) mkLinkSystem() ipld.LinkSystem {
5261
log.Errorf("Could not decode IPLD node for potential advertisement: %s", err)
5362
return nil, err
5463
}
64+
5565
// If this was an advertisement, then return it.
5666
if isAdvertisement(n) {
67+
if cidSchemaType != "" && cidSchemaType != ipnisync.CidSchemaAdvertisement {
68+
log.Errorf("Retrieved advertisement when asked for %s", cidSchemaType)
69+
return nil, errors.New("data does not match requested type")
70+
}
5771
log.Debugw("Retrieved advertisement from datastore", "cid", c, "size", len(val))
5872
return bytes.NewBuffer(val), nil
5973
}
6074
log.Debugw("Retrieved non-advertisement object from datastore", "cid", c, "size", len(val))
6175
}
6276

77+
if cidSchemaType != "" && cidSchemaType != ipnisync.CidSchemaEntryChunk {
78+
log.Errorf("Asked for %s when expecting %s", cidSchemaType, ipnisync.CidSchemaEntryChunk)
79+
return nil, errors.New("requested unexpected data type")
80+
}
81+
6382
// Not an advertisement, so this means we are receiving ingestion data.
6483

6584
// If no lister registered return error

‎go.mod

+5-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
77
github.com/golang/mock v1.6.0
88
github.com/hashicorp/go-multierror v1.1.1
9-
github.com/ipfs/boxo v0.22.0
9+
github.com/ipfs/boxo v0.23.0
1010
github.com/ipfs/go-cid v0.4.1
1111
github.com/ipfs/go-datastore v0.6.0
1212
github.com/ipfs/go-ds-leveldb v0.5.0
@@ -17,8 +17,8 @@ require (
1717
github.com/ipld/go-codec-dagpb v1.6.0
1818
github.com/ipld/go-ipld-adl-hamt v0.0.0-20240322071803-376decb85801
1919
github.com/ipld/go-ipld-prime v0.21.0
20-
github.com/ipni/go-libipni v0.6.11
21-
github.com/libp2p/go-libp2p v0.36.2
20+
github.com/ipni/go-libipni v0.6.13
21+
github.com/libp2p/go-libp2p v0.36.3
2222
github.com/libp2p/go-libp2p-pubsub v0.12.0
2323
github.com/mitchellh/go-homedir v1.1.0
2424
github.com/multiformats/go-multiaddr v0.13.0
@@ -27,7 +27,7 @@ require (
2727
github.com/prometheus/client_golang v1.20.0
2828
github.com/rogpeppe/go-internal v1.12.0
2929
github.com/stretchr/testify v1.9.0
30-
github.com/urfave/cli/v2 v2.27.2
30+
github.com/urfave/cli/v2 v2.27.4
3131
go.opentelemetry.io/otel v1.28.0
3232
go.opentelemetry.io/otel/exporters/prometheus v0.50.0
3333
go.opentelemetry.io/otel/metric v1.28.0
@@ -178,7 +178,7 @@ require (
178178
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
179179
github.com/twmb/murmur3 v1.1.6 // indirect
180180
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
181-
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
181+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
182182
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
183183
go.opentelemetry.io/otel/trace v1.28.0 // indirect
184184
go.uber.org/atomic v1.11.0 // indirect

‎go.sum

+10-10
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,8 @@ github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFck
262262
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
263263
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
264264
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
265-
github.com/ipfs/boxo v0.22.0 h1:QTC+P5uhsBNq6HzX728nsLyFW6rYDeR/5hggf9YZX78=
266-
github.com/ipfs/boxo v0.22.0/go.mod h1:yp1loimX0BDYOR0cyjtcXHv15muEh5V1FqO2QLlzykw=
265+
github.com/ipfs/boxo v0.23.0 h1:dY1PpcvPJ//VuUQ1TUd5TZvmaGuzxJ8dOP6mXaw+ke8=
266+
github.com/ipfs/boxo v0.23.0/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E=
267267
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
268268
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
269269
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -367,8 +367,8 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH
367367
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
368368
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
369369
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw=
370-
github.com/ipni/go-libipni v0.6.11 h1:i+a+OCVgtKd0FMg8L9PrNpJgq//MYTxsl7YlyCHKAJY=
371-
github.com/ipni/go-libipni v0.6.11/go.mod h1:hHkfaG5zP8M8RQX8C84gTUre5KODHGPxEbI8E2SgCNw=
370+
github.com/ipni/go-libipni v0.6.13 h1:6fQU6ZFu8fi0DZIs4VXZrIFbT9r97dNmNl7flWMVblE=
371+
github.com/ipni/go-libipni v0.6.13/go.mod h1:+hNohg7Tx8ML2a/Ei19zUxCnSqtqXiHySlqHIwPhQyQ=
372372
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
373373
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
374374
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@@ -419,8 +419,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
419419
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
420420
github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
421421
github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
422-
github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U=
423-
github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY=
422+
github.com/libp2p/go-libp2p v0.36.3 h1:NHz30+G7D8Y8YmznrVZZla0ofVANrvBl2c+oARfMeDQ=
423+
github.com/libp2p/go-libp2p v0.36.3/go.mod h1:4Y5vFyCUiJuluEPmpnKYf6WFx5ViKPUYs/ixe9ANFZ8=
424424
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
425425
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
426426
github.com/libp2p/go-libp2p-pubsub v0.12.0 h1:PENNZjSfk8KYxANRlpipdS7+BfLmOl3L2E/6vSNjbdI=
@@ -695,8 +695,8 @@ github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq
695695
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
696696
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
697697
github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
698-
github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
699-
github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
698+
github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
699+
github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
700700
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
701701
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
702702
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
@@ -729,8 +729,8 @@ github.com/wlynxg/anet v0.0.4/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguH
729729
github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I=
730730
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk=
731731
github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
732-
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
733-
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
732+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
733+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
734734
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
735735
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
736736
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

0 commit comments

Comments
 (0)