Skip to content

Commit 1e30f83

Browse files
gammazerogammazero
authored
Read schema type hint from sync request (#453)
* Read schema type hint from sync request Head the schema type hint from the IPNI sync request header, and if provided, compare it against the type of data retrieved. * log but do not fail if hint does not match data type
1 parent 69eb980 commit 1e30f83

File tree

3 files changed

+34
-16
lines changed

3 files changed

+34
-16
lines changed

‎engine/linksystem.go

+19-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/ipld/go-ipld-prime/codec/dagjson"
1111
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
1212
"github.com/ipld/go-ipld-prime/node/basicnode"
13+
"github.com/ipni/go-libipni/dagsync/ipnisync"
1314
"github.com/ipni/go-libipni/ingest/schema"
1415
provider "github.com/ipni/index-provider"
1516
"github.com/libp2p/go-libp2p/core/peer"
@@ -36,6 +37,14 @@ func (e *Engine) mkLinkSystem() ipld.LinkSystem {
3637
c := lnk.(cidlink.Link).Cid
3738
log.Debugf("Triggered ReadOpener from engine's linksystem with cid (%s)", c)
3839

40+
cidSchemaType, err := ipnisync.CidSchemaFromCtx(ctx)
41+
if err != nil {
42+
log.Errorf("CID schema type hint error: %s", err)
43+
}
44+
if cidSchemaType != "" {
45+
log.Infof("indexer requested %s", cidSchemaType)
46+
}
47+
3948
// Get the node from main datastore. If it is in the
4049
// main datastore it means it is an advertisement.
4150
val, err := e.ds.Get(ctx, datastore.NewKey(c.String()))
@@ -52,14 +61,23 @@ func (e *Engine) mkLinkSystem() ipld.LinkSystem {
5261
log.Errorf("Could not decode IPLD node for potential advertisement: %s", err)
5362
return nil, err
5463
}
64+
5565
// If this was an advertisement, then return it.
5666
if isAdvertisement(n) {
57-
log.Debugw("Retrieved advertisement from datastore", "cid", c, "size", len(val))
67+
if cidSchemaType != "" && cidSchemaType != ipnisync.CidSchemaAdvertisement {
68+
log.Errorf("Retrieved advertisement when asked for %s", cidSchemaType)
69+
} else {
70+
log.Debugw("Retrieved advertisement from datastore", "cid", c, "size", len(val))
71+
}
5872
return bytes.NewBuffer(val), nil
5973
}
6074
log.Debugw("Retrieved non-advertisement object from datastore", "cid", c, "size", len(val))
6175
}
6276

77+
if cidSchemaType != "" && cidSchemaType != ipnisync.CidSchemaEntryChunk {
78+
log.Errorf("Asked for %s when expecting %s", cidSchemaType, ipnisync.CidSchemaEntryChunk)
79+
}
80+
6381
// Not an advertisement, so this means we are receiving ingestion data.
6482

6583
// If no lister registered return error

‎go.mod

+5-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
77
github.com/golang/mock v1.6.0
88
github.com/hashicorp/go-multierror v1.1.1
9-
github.com/ipfs/boxo v0.22.0
9+
github.com/ipfs/boxo v0.23.0
1010
github.com/ipfs/go-cid v0.4.1
1111
github.com/ipfs/go-datastore v0.6.0
1212
github.com/ipfs/go-ds-leveldb v0.5.0
@@ -17,8 +17,8 @@ require (
1717
github.com/ipld/go-codec-dagpb v1.6.0
1818
github.com/ipld/go-ipld-adl-hamt v0.0.0-20240322071803-376decb85801
1919
github.com/ipld/go-ipld-prime v0.21.0
20-
github.com/ipni/go-libipni v0.6.11
21-
github.com/libp2p/go-libp2p v0.36.2
20+
github.com/ipni/go-libipni v0.6.13
21+
github.com/libp2p/go-libp2p v0.36.3
2222
github.com/libp2p/go-libp2p-pubsub v0.12.0
2323
github.com/mitchellh/go-homedir v1.1.0
2424
github.com/multiformats/go-multiaddr v0.13.0
@@ -27,7 +27,7 @@ require (
2727
github.com/prometheus/client_golang v1.20.0
2828
github.com/rogpeppe/go-internal v1.12.0
2929
github.com/stretchr/testify v1.9.0
30-
github.com/urfave/cli/v2 v2.27.2
30+
github.com/urfave/cli/v2 v2.27.4
3131
go.opentelemetry.io/otel v1.28.0
3232
go.opentelemetry.io/otel/exporters/prometheus v0.50.0
3333
go.opentelemetry.io/otel/metric v1.28.0
@@ -178,7 +178,7 @@ require (
178178
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
179179
github.com/twmb/murmur3 v1.1.6 // indirect
180180
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
181-
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
181+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
182182
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
183183
go.opentelemetry.io/otel/trace v1.28.0 // indirect
184184
go.uber.org/atomic v1.11.0 // indirect

‎go.sum

+10-10
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,8 @@ github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFck
262262
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
263263
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
264264
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
265-
github.com/ipfs/boxo v0.22.0 h1:QTC+P5uhsBNq6HzX728nsLyFW6rYDeR/5hggf9YZX78=
266-
github.com/ipfs/boxo v0.22.0/go.mod h1:yp1loimX0BDYOR0cyjtcXHv15muEh5V1FqO2QLlzykw=
265+
github.com/ipfs/boxo v0.23.0 h1:dY1PpcvPJ//VuUQ1TUd5TZvmaGuzxJ8dOP6mXaw+ke8=
266+
github.com/ipfs/boxo v0.23.0/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E=
267267
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
268268
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
269269
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -367,8 +367,8 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH
367367
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
368368
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
369369
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw=
370-
github.com/ipni/go-libipni v0.6.11 h1:i+a+OCVgtKd0FMg8L9PrNpJgq//MYTxsl7YlyCHKAJY=
371-
github.com/ipni/go-libipni v0.6.11/go.mod h1:hHkfaG5zP8M8RQX8C84gTUre5KODHGPxEbI8E2SgCNw=
370+
github.com/ipni/go-libipni v0.6.13 h1:6fQU6ZFu8fi0DZIs4VXZrIFbT9r97dNmNl7flWMVblE=
371+
github.com/ipni/go-libipni v0.6.13/go.mod h1:+hNohg7Tx8ML2a/Ei19zUxCnSqtqXiHySlqHIwPhQyQ=
372372
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
373373
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
374374
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@@ -419,8 +419,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
419419
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
420420
github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
421421
github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
422-
github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U=
423-
github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY=
422+
github.com/libp2p/go-libp2p v0.36.3 h1:NHz30+G7D8Y8YmznrVZZla0ofVANrvBl2c+oARfMeDQ=
423+
github.com/libp2p/go-libp2p v0.36.3/go.mod h1:4Y5vFyCUiJuluEPmpnKYf6WFx5ViKPUYs/ixe9ANFZ8=
424424
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
425425
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
426426
github.com/libp2p/go-libp2p-pubsub v0.12.0 h1:PENNZjSfk8KYxANRlpipdS7+BfLmOl3L2E/6vSNjbdI=
@@ -695,8 +695,8 @@ github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq
695695
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
696696
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
697697
github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
698-
github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
699-
github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
698+
github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
699+
github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
700700
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
701701
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
702702
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
@@ -729,8 +729,8 @@ github.com/wlynxg/anet v0.0.4/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguH
729729
github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I=
730730
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk=
731731
github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
732-
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
733-
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
732+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
733+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
734734
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
735735
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
736736
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

0 commit comments

Comments
 (0)