feat: use index claim (#10)

This PR makes use of the `assert/index` content claim, allowing a [w3-index](https://github.com/w3s-project/specs/blob/main/w3-index.md) to be used to help serve the data, which yields the following perf improvements to the batching blob fetcher:

| Export 500MiB DAG | Sub-requests | Time |
| -- | -- | -- |
| without index |  535 | 11.960s |
| with index | 37 | 7.554s |

package.json

@@ -51,17 +51,18 @@
   "dependencies": {
     "@ucanto/interface": "^10.0.1",
     "@web3-storage/blob-index": "^1.0.2",
-    "@web3-storage/content-claims": "^5.0.0",
+    "@web3-storage/content-claims": "^5.1.0",
     "multiformats": "^13.1.0",
     "multipart-byte-range": "^3.0.1",
     "p-defer": "^4.0.1",
     "p-queue": "^8.0.1"
   },
   "devDependencies": {
     "@ipld/unixfs": "^3.0.0",
-    "@web3-storage/public-bucket": "^1.0.0",
+    "@ucanto/principal": "^9.0.1",
+    "@web3-storage/public-bucket": "^1.1.0",
     "c8": "^9.1.0",
-    "carstream": "^2.1.0",
+    "carstream": "^2.2.0",
     "entail": "^2.1.2",
     "ipfs-unixfs-exporter": "^13.5.0",
     "standard": "^17.1.0",

src/fetcher/batching.js

@@ -13,7 +13,7 @@ import { resolveRange } from './lib.js'
  * @typedef {Map<RangeKey, PendingBlobRequest[]>} RangedRequests
  */
 
-const MAX_BATCH_SIZE = 12
+const MAX_BATCH_SIZE = 16
 
 /** @implements {API.Fetcher} */
 class BatchingFetcher {

src/locator/content-claims.js

@@ -1,7 +1,8 @@
 // eslint-disable-next-line
 import * as API from '../api.js'
 import * as Claims from '@web3-storage/content-claims/client'
-import { DigestMap } from '@web3-storage/blob-index'
+import { DigestMap, ShardedDAGIndex } from '@web3-storage/blob-index'
+import { fetchBlob } from '../fetcher/simple.js'
 import { NotFoundError } from '../lib.js'
 
 /**
@@ -85,6 +86,45 @@ export class ContentClaimsLocator {
           })
         }
       }
+
+      if (claim.type === 'assert/index') {
+        await this.#readClaims(claim.index.multihash)
+        const location = this.#cache.get(claim.index.multihash)
+        if (!location) continue
+
+        const fetchRes = await fetchBlob(location)
+        if (fetchRes.error) {
+          console.warn('failed to fetch index', fetchRes.error)
+          continue
+        }
+
+        const indexBytes = await fetchRes.ok.bytes()
+        const decodeRes = ShardedDAGIndex.extract(indexBytes)
+        if (decodeRes.error) {
+          console.warn('failed to decode index', decodeRes.error)
+          continue
+        }
+
+        const index = decodeRes.ok
+        await Promise.all([...index.shards].map(async ([shard, slices]) => {
+          await this.#readClaims(shard)
+          const location = this.#cache.get(shard)
+          if (!location) return
+
+          for (const [slice, pos] of slices) {
+            this.#cache.set(slice, {
+              digest: slice,
+              site: location.site.map(s => ({
+                location: s.location,
+                range: {
+                  offset: s.range.offset + pos[0],
+                  length: s.range.offset + pos[1]
+                }
+              }))
+            })
+          }
+        }))
+      }
     }
     this.#claimFetched.set(digest, true)
   }