From eefd885c200ce7f580126fc453fb9f2700e74606 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 15 May 2024 15:32:09 +0200 Subject: [PATCH] Revert "fix: revert blob add by default (#1456)" (#1458) This reverts commit b77ec750c3911f6f47fa32a5e1560858b347985c. To merge/release when ready --- packages/upload-client/src/index.js | 17 +- packages/upload-client/test/index.test.js | 264 +--------------------- packages/w3up-client/src/client.js | 4 +- packages/w3up-client/test/client.test.js | 25 +- 4 files changed, 35 insertions(+), 275 deletions(-) diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index a98fde498..ac714d5d5 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -8,8 +8,7 @@ import * as Upload from './upload.js' import * as UnixFS from './unixfs.js' import * as CAR from './car.js' import { ShardingStream, defaultFileComparator } from './sharding.js' -// TODO: enable Blob -// import { codec as carCodec } from '@ucanto/transport/car' +import { codec as carCodec } from '@ucanto/transport/car' export { Blob, Store, Upload, UnixFS, CAR } export * from './sharding.js' @@ -133,20 +132,16 @@ async function uploadBlockStream( new TransformStream({ async transform(car, controller) { const bytes = new Uint8Array(await car.arrayBuffer()) - // Invoke blob/add and write bytes to write target - // TODO: enable Blob - // const multihash = await Blob.add(conf, bytes, options) - const cid = await Store.add(conf, bytes, options) - // TODO: enable Blob - // const cid = Link.create(carCodec.code, multihash) + // Invoke blob/add and write bytes to write target + const multihash = await Blob.add(conf, bytes, options) + // Should this be raw instead? + const cid = Link.create(carCodec.code, multihash) let piece if (pieceHasher) { const multihashDigest = await pieceHasher.digest(bytes) /** @type {import('@web3-storage/capabilities/types').PieceLink} */ piece = Link.create(raw.code, multihashDigest) - // TODO: enable Blob - // const content = Link.create(raw.code, multihash) - const content = Link.create(raw.code, cid.multihash) + const content = Link.create(raw.code, multihash) // Invoke filecoin/offer for data const result = await Storefront.filecoinOffer( diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 587cf384f..77173d4f6 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -6,7 +6,6 @@ import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as UCAN from '@web3-storage/capabilities/ucan' import * as BlobCapabilities from '@web3-storage/capabilities/blob' -import * as StoreCapabilities from '@web3-storage/capabilities/store' import * as UploadCapabilities from '@web3-storage/capabilities/upload' import * as StorefrontCapabilities from '@web3-storage/capabilities/filecoin/storefront' import { Piece } from '@web3-storage/data-segment' @@ -42,12 +41,6 @@ describe('uploadFile', () => { let carCID const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -68,24 +61,6 @@ describe('uploadFile', () => { return { ok: { time: Date.now() } } }), }, - store: { - add: provide(StoreCapabilities.add, ({ invocation, capability }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - assert.equal(capability.can, StoreCapabilities.add.can) - assert.equal(capability.with, space.did()) - /** @type {Omit} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - link: expectedCar.cid, - with: space.did(), - } - - return { ok: { ...res, allocated: capability.nb.size } } - }), - }, blob: { add: provide( BlobCapabilities.add, @@ -155,11 +130,8 @@ describe('uploadFile', () => { } ) - // TODO: Blob Enable - // assert(service.blob.add.called) - // assert.equal(service.blob.add.callCount, 1) - assert(service.store.add.called) - assert.equal(service.store.add.callCount, 1) + assert(service.blob.add.called) + assert.equal(service.blob.add.callCount, 1) assert(service.filecoin.offer.called) assert.equal(service.filecoin.offer.callCount, 1) assert(service.upload.add.called) @@ -179,12 +151,6 @@ describe('uploadFile', () => { const carCIDs = [] const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -199,14 +165,6 @@ describe('uploadFile', () => { }), ]) - /** @type {Omit} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - with: space.did(), - } - const service = mockService({ ucan: { conclude: provide(UCAN.conclude, () => { @@ -222,17 +180,6 @@ describe('uploadFile', () => { ) }), }, - store: { - add: provide(StoreCapabilities.add, ({ capability }) => ({ - ok: { - ...res, - link: /** @type {import('../src/types.js').CARLink} */ ( - capability.nb.link - ), - allocated: capability.nb.size, - }, - })), - }, filecoin: { offer: Server.provideAdvanced({ capability: StorefrontCapabilities.filecoinOffer, @@ -291,12 +238,6 @@ describe('uploadFile', () => { const file = new Blob([bytes]) const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -330,29 +271,6 @@ describe('uploadFile', () => { ) }), }, - store: { - add: provide( - StoreCapabilities.add, - async ({ invocation, capability }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - assert.equal(capability.can, StoreCapabilities.add.can) - assert.equal(capability.with, space.did()) - - const expectedCar = await toCAR(bytes) - /** @type {Omit} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - link: expectedCar.cid, - with: space.did(), - } - - return { ok: { ...res, allocated: capability.nb.size } } - } - ), - }, filecoin: { offer: Server.provideAdvanced({ capability: StorefrontCapabilities.filecoinOffer, @@ -386,11 +304,8 @@ describe('uploadFile', () => { ) ) - assert(service.store.add.called) - assert.equal(service.store.add.callCount, 1) - // assert(service.blob.add.called) - // assert.equal(service.blob.add.callCount, 1) - + assert(service.blob.add.called) + assert.equal(service.blob.add.callCount, 1) assert(service.filecoin.offer.called) assert.equal(service.filecoin.offer.callCount, 1) }) @@ -410,12 +325,6 @@ describe('uploadDirectory', () => { let carCID = null const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -431,33 +340,6 @@ describe('uploadDirectory', () => { ]) const service = mockService({ - store: { - add: provide(StoreCapabilities.add, ({ capability, invocation }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.equal(invCap.can, StoreCapabilities.add.can) - assert.equal(invCap.with, space.did()) - - /** @type {Omit} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - with: space.did(), - } - - return { - ok: { - ...res, - link: /** @type {import('../src/types.js').CARLink} */ ( - capability.nb.link - ), - allocated: capability.nb.size, - }, - } - }), - }, ucan: { conclude: provide(UCAN.conclude, () => { return { ok: { time: Date.now() } } @@ -526,10 +408,8 @@ describe('uploadDirectory', () => { } ) - // assert(service.blob.add.called) - // assert.equal(service.blob.add.callCount, 1) - assert(service.store.add.called) - assert.equal(service.store.add.callCount, 1) + assert(service.blob.add.called) + assert.equal(service.blob.add.callCount, 1) assert(service.filecoin.offer.called) assert.equal(service.filecoin.offer.callCount, 1) assert(service.upload.add.called) @@ -551,12 +431,6 @@ describe('uploadDirectory', () => { const carCIDs = [] const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -571,13 +445,6 @@ describe('uploadDirectory', () => { }), ]) - /** @type {Omit} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - with: space.did(), - } const service = mockService({ ucan: { conclude: provide(UCAN.conclude, () => { @@ -593,17 +460,6 @@ describe('uploadDirectory', () => { ) }), }, - store: { - add: provide(StoreCapabilities.add, ({ capability }) => ({ - ok: { - ...res, - link: /** @type {import('../src/types.js').CARLink} */ ( - capability.nb.link - ), - allocated: capability.nb.size, - }, - })), - }, filecoin: { offer: Server.provideAdvanced({ capability: StorefrontCapabilities.filecoinOffer, @@ -655,12 +511,6 @@ describe('uploadDirectory', () => { const piece = Piece.fromPayload(someBytes).link const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -696,24 +546,6 @@ describe('uploadDirectory', () => { ) }), }, - store: { - add: provide(StoreCapabilities.add, ({ invocation, capability }) => { - // @ts-expect-error blob invocation - invocations.push(invocation) - return { - ok: { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - with: capability.with, - link: /** @type {import('../src/types.js').CARLink} */ ( - capability.nb.link - ), - allocated: capability.nb.size, - }, - } - }), - }, filecoin: { offer: Server.provideAdvanced({ capability: StorefrontCapabilities.filecoinOffer, @@ -799,16 +631,6 @@ describe('uploadDirectory', () => { : [] ) .map((cid) => cid.toString()) - // const shardsForUnordered = uploadServiceForUnordered.invocations - // .flatMap((i) => - // i.capability.can === 'upload/add' ? i.capability.nb.shards ?? [] : [] - // ) - // .map((cid) => cid.toString()) - // const shardsForOrdered = uploadServiceForOrdered.invocations - // .flatMap((i) => - // i.capability.can === 'upload/add' ? i.capability.nb.shards ?? [] : [] - // ) - // .map((cid) => cid.toString()) assert.deepEqual( shardsForUnordered, shardsForOrdered, @@ -869,12 +691,6 @@ describe('uploadCAR', () => { const carCIDs = [] const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -889,14 +705,6 @@ describe('uploadCAR', () => { }), ]) - /** @type {Omit} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - with: space.did(), - } - const service = mockService({ ucan: { conclude: provide(UCAN.conclude, () => { @@ -917,24 +725,6 @@ describe('uploadCAR', () => { ) }), }, - store: { - add: provide(StoreCapabilities.add, ({ capability, invocation }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.equal(invCap.can, StoreCapabilities.add.can) - assert.equal(invCap.with, space.did()) - return { - ok: { - ...res, - link: /** @type {import('../src/types.js').CARLink} */ ( - capability.nb.link - ), - allocated: capability.nb.size, - }, - } - }), - }, filecoin: { offer: Server.provideAdvanced({ capability: StorefrontCapabilities.filecoinOffer, @@ -988,10 +778,8 @@ describe('uploadCAR', () => { } ) - // assert(service.blob.add.called) - // assert.equal(service.blob.add.callCount, 2) - assert(service.store.add.called) - assert.equal(service.store.add.callCount, 2) + assert(service.blob.add.called) + assert.equal(service.blob.add.callCount, 2) assert(service.filecoin.offer.called) assert.equal(service.filecoin.offer.callCount, 2) assert(service.upload.add.called) @@ -1014,12 +802,6 @@ describe('uploadCAR', () => { const pieceCIDs = [] const proofs = await Promise.all([ - StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), BlobCapabilities.add.delegate({ issuer: space, audience: agent, @@ -1053,30 +835,6 @@ describe('uploadCAR', () => { ) }), }, - store: { - add: provide(StoreCapabilities.add, ({ capability, invocation }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - assert.equal(capability.can, StoreCapabilities.add.can) - assert.equal(capability.with, space.did()) - /** @type {Omit} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - with: space.did(), - } - return { - ok: { - ...res, - link: /** @type {import('../src/types.js').CARLink} */ ( - capability.nb.link - ), - allocated: capability.nb.size, - }, - } - }), - }, filecoin: { offer: Server.provideAdvanced({ capability: StorefrontCapabilities.filecoinOffer, @@ -1128,10 +886,8 @@ describe('uploadCAR', () => { } ) - // assert(service.blob.add.called) - // assert.equal(service.blob.add.callCount, 1) - assert(service.store.add.called) - assert.equal(service.store.add.callCount, 1) + assert(service.blob.add.called) + assert.equal(service.blob.add.callCount, 1) assert(service.filecoin.offer.called) assert.equal(service.filecoin.offer.callCount, 1) assert(service.upload.add.called) diff --git a/packages/w3up-client/src/client.js b/packages/w3up-client/src/client.js index 30743814c..0954dd22e 100644 --- a/packages/w3up-client/src/client.js +++ b/packages/w3up-client/src/client.js @@ -156,7 +156,7 @@ export class Client extends Base { /** * Get a receipt for an executed task by its CID. * - * @param {import('multiformats').UnknownLink} taskCid - Task CID for receipt. + * @param {import('multiformats').UnknownLink} taskCid */ async getReceipt(taskCid) { // Fetch receipt from endpoint @@ -353,8 +353,8 @@ export class Client extends Base { cause: error, }) } + /* c8 ignore stop */ } - /* c8 ignore stop */ }) ) } diff --git a/packages/w3up-client/test/client.test.js b/packages/w3up-client/test/client.test.js index a53945140..fda4a546a 100644 --- a/packages/w3up-client/test/client.test.js +++ b/packages/w3up-client/test/client.test.js @@ -12,7 +12,7 @@ export const testClient = { uploadFile: Test.withContext({ 'should upload a file to the service': async ( assert, - { connection, provisionsStorage, uploadTable, storeTable } + { connection, provisionsStorage, uploadTable, allocationsStorage } ) => { const bytes = await randomBytes(128) const file = new Blob([bytes]) @@ -51,9 +51,15 @@ export const testClient = { ok: true, }) - assert.deepEqual(await storeTable.exists(space.did(), expectedCar.cid), { - ok: true, - }) + assert.deepEqual( + await allocationsStorage.exists( + space.did(), + expectedCar.cid.multihash.bytes + ), + { + ok: true, + } + ) assert.equal(carCID?.toString(), expectedCar.cid.toString()) assert.equal(dataCID.toString(), expectedCar.roots[0].toString()) @@ -130,7 +136,7 @@ export const testClient = { uploadCar: Test.withContext({ 'uploads a CAR file to the service': async ( assert, - { connection, provisionsStorage, uploadTable, storeTable } + { connection, provisionsStorage, uploadTable, allocationsStorage } ) => { const car = await randomCAR(32) @@ -170,9 +176,12 @@ export const testClient = { return assert.ok(carCID) } - assert.deepEqual(await storeTable.exists(space.did(), carCID), { - ok: true, - }) + assert.deepEqual( + await allocationsStorage.exists(space.did(), carCID.multihash.bytes), + { + ok: true, + } + ) }, }), getRecipt: Test.withContext({