feat: switch to KV (#57)

package.json

@@ -38,6 +38,7 @@
     "streaming-iterables": "^7.1.0"
   },
   "devDependencies": {
+    "@cloudflare/workers-types": "^4.20230518.0",
     "ava": "^5.2.0",
     "carbites": "^1.0.6",
     "esbuild": "^0.17.11",

src/bindings.d.ts

@@ -1,6 +1,7 @@
 import type { Link } from 'multiformats/link'
 import type { Context } from '@web3-storage/gateway-lib'
 import type { CARLink } from 'cardex/api'
+import type { R2Bucket, KVNamespace } from '@cloudflare/workers-types'
 import type { MemoryBudget } from './lib/mem-budget'
 
 export {}
@@ -10,7 +11,7 @@ export interface Environment {
   CARPARK: R2Bucket
   DUDEWHERE: R2Bucket
   SATNAV: R2Bucket
-  BLOCKLY: R2Bucket
+  BLOCKLY: KVNamespace
   MAX_SHARDS: string
 }
 
@@ -30,33 +31,3 @@ export interface IndexSource {
 export interface IndexSourcesContext extends Context {
   indexSources: IndexSource[]
 }
-
-export interface R2GetOptions {
-  range?: {
-    offset: number
-    length?: number
-  }
-}
-
-export interface R2ListOptions {
-  prefix?: string
-  cursor?: string
-}
-
-export interface R2Object {
-  body: ReadableStream
-  size: number
-  key: string
-}
-
-export interface R2Objects {
-  objects: R2Object[]
-  truncated: boolean
-  cursor?: string
-}
-
-export interface R2Bucket {
-  get (k: string, o?: R2GetOptions): Promise<R2Object | null>
-  head (k: string, o?: R2GetOptions): Promise<R2Object | null>
-  list (o?: R2ListOptions): Promise<R2Objects | null>
-}

src/lib/blockstore.js

@@ -8,7 +8,7 @@ import { OrderedCarBlockBatcher } from './block-batch.js'
  * @typedef {import('cardex/api').IndexItem} IndexEntry
  * @typedef {string} MultihashString
  * @typedef {import('dagula').Block} Block
- * @typedef {import('../bindings').R2Bucket} R2Bucket
+ * @typedef {import('@cloudflare/workers-types').R2Bucket} R2Bucket
  */
 
 // 2MB (max safe libp2p block size) + typical block header length + some leeway

src/lib/car-index.js

@@ -156,18 +156,18 @@ export class StreamingCarIndex {
 const mhToString = mh => base58btc.encode(mh.bytes)
 
 export class BlocklyIndex {
-  /** R2 bucket where indexes live. */
-  #bucket
+  /** Storage where indexes live. */
+  #store
   /** Cached index entries. */
   #cache
   /** Indexes that have been read. */
   #indexes
 
   /**
-   * @param {import('../bindings').R2Bucket} indexBucket
+   * @param {import('@cloudflare/workers-types').KVNamespace} indexStore
    */
-  constructor (indexBucket) {
-    this.#bucket = indexBucket
+  constructor (indexStore) {
+    this.#store = indexStore
     /** @type {Map<MultihashString, IndexEntry>} */
     this.#cache = new Map()
     this.#indexes = new Set()
@@ -213,10 +213,10 @@ export class BlocklyIndex {
     const key = mhToString(cid.multihash)
     if (this.#indexes.has(key)) return
 
-    const res = await this.#bucket.get(`${key}/.idx`)
+    const res = await this.#store.get(`${key}/.idx`, { type: 'stream' })
     if (!res) return
 
-    const reader = MultiIndexReader.createReader({ reader: res.body.getReader() })
+    const reader = MultiIndexReader.createReader({ reader: res.getReader() })
     while (true) {
       const { done, value } = await reader.read()
       if (done) break

test/helpers.js

@@ -51,10 +51,10 @@ export class Builder {
   #blockly
 
   /**
-   * @param {import('@miniflare/r2').R2Bucket} carpark
-   * @param {import('@miniflare/r2').R2Bucket} satnav
-   * @param {import('@miniflare/r2').R2Bucket} dudewhere
-   * @param {import('@miniflare/r2').R2Bucket} blockly
+   * @param {import('@cloudflare/workers-types').R2Bucket} carpark
+   * @param {import('@cloudflare/workers-types').R2Bucket} satnav
+   * @param {import('@cloudflare/workers-types').R2Bucket} dudewhere
+   * @param {import('@cloudflare/workers-types').KVNamespace} blockly
    */
   constructor (carpark, satnav, dudewhere, blockly) {
     this.#carpark = carpark
@@ -247,7 +247,7 @@ export async function collect (collectable) {
 }
 
 /**
- * @param {import('@miniflare/r2').R2Bucket} bucket
+ * @param {import('@cloudflare/workers-types').R2Bucket} bucket
  * @param {string} prefix
  */
 export async function listAll (bucket, prefix) {
@@ -276,7 +276,7 @@ function getAnyMapEntry (map) {
 }
 
 /**
- * @param {import('@miniflare/r2').R2Bucket} bucket
+ * @param {import('@cloudflare/workers-types').R2Bucket} bucket
  * @param {string} shardCID
  * @param {number} offset
  */

test/index.spec.js

@@ -13,7 +13,7 @@ describe('freeway', () => {
   let builder
 
   before(async () => {
-    const bucketNames = ['CARPARK', 'SATNAV', 'DUDEWHERE', 'BLOCKLY']
+    const bucketNames = ['CARPARK', 'SATNAV', 'DUDEWHERE']
 
     miniflare = new Miniflare({
       bindings: {},
@@ -26,12 +26,15 @@ describe('freeway', () => {
       buildCommand: undefined,
       wranglerConfigEnv: 'test',
       modules: true,
-      r2Buckets: bucketNames
+      r2Buckets: bucketNames,
       // r2Persist: true
+      kvNamespaces: ['BLOCKLY']
     })
 
     const buckets = await Promise.all(bucketNames.map(b => miniflare.getR2Bucket(b)))
-    builder = new Builder(buckets[0], buckets[1], buckets[2], buckets[3])
+    const blockly = await miniflare.getKVNamespace('BLOCKLY')
+    // @ts-expect-error
+    builder = new Builder(buckets[0], buckets[1], buckets[2], blockly)
   })
 
   it('should get a file', async () => {