Merge pull request #1011 from streamich/cas-customizations

CAS storage customizations

src/cas/types.ts

@@ -1,10 +1,10 @@
 import type { CrudResourceInfo } from '../crud/types';
 
-export interface CasApi {
-  put(blob: Uint8Array): Promise<string>;
-  get(hash: string, options?: CasGetOptions): Promise<Uint8Array>;
-  del(hash: string, silent?: boolean): Promise<void>;
-  info(hash: string): Promise<CrudResourceInfo>;
+export interface CasApi<Hash> {
+  put(blob: Uint8Array): Promise<Hash>;
+  get(hash: Hash, options?: CasGetOptions): Promise<Uint8Array>;
+  del(hash: Hash, silent?: boolean): Promise<void>;
+  info(hash: Hash): Promise<CrudResourceInfo>;
 }
 
 export interface CasGetOptions {

src/crud-to-cas/CrudCas.ts

@@ -1,62 +1,18 @@
 import { hashToLocation } from './util';
-import type { CasApi, CasGetOptions } from '../cas/types';
-import type { CrudApi, CrudResourceInfo } from '../crud/types';
+import { CrudCasBase } from './CrudCasBase';
+import type { CrudApi } from '../crud/types';
 
 export interface CrudCasOptions {
   hash: (blob: Uint8Array) => Promise<string>;
 }
 
-const normalizeErrors = async <T>(code: () => Promise<T>): Promise<T> => {
-  try {
-    return await code();
-  } catch (error) {
-    if (error && typeof error === 'object') {
-      switch (error.name) {
-        case 'ResourceNotFound':
-        case 'CollectionNotFound':
-          throw new DOMException(error.message, 'BlobNotFound');
-      }
-    }
-    throw error;
-  }
-};
+const hashEqual = (h1: string, h2: string) => h1 === h2;
 
-export class CrudCas implements CasApi {
+export class CrudCas extends CrudCasBase<string> {
   constructor(
     protected readonly crud: CrudApi,
     protected readonly options: CrudCasOptions,
-  ) {}
-
-  public readonly put = async (blob: Uint8Array): Promise<string> => {
-    const digest = await this.options.hash(blob);
-    const [collection, resource] = hashToLocation(digest);
-    await this.crud.put(collection, resource, blob);
-    return digest;
-  };
-
-  public readonly get = async (hash: string, options?: CasGetOptions): Promise<Uint8Array> => {
-    const [collection, resource] = hashToLocation(hash);
-    return await normalizeErrors(async () => {
-      const blob = await this.crud.get(collection, resource);
-      if (!options?.skipVerification) {
-        const digest = await this.options.hash(blob);
-        if (hash !== digest) throw new DOMException('Blob contents does not match hash', 'Integrity');
-      }
-      return blob;
-    });
-  };
-
-  public readonly del = async (hash: string, silent?: boolean): Promise<void> => {
-    const [collection, resource] = hashToLocation(hash);
-    await normalizeErrors(async () => {
-      return await this.crud.del(collection, resource, silent);
-    });
-  };
-
-  public readonly info = async (hash: string): Promise<CrudResourceInfo> => {
-    const [collection, resource] = hashToLocation(hash);
-    return await normalizeErrors(async () => {
-      return await this.crud.info(collection, resource);
-    });
-  };
+  ) {
+    super(crud, options.hash, hashToLocation, hashEqual);
+  }
 }

src/crud-to-cas/CrudCasBase.ts

@@ -0,0 +1,60 @@
+import type { CasApi, CasGetOptions } from '../cas/types';
+import type { CrudApi, CrudResourceInfo } from '../crud/types';
+import type { FsLocation } from '../fsa-to-node/types';
+
+const normalizeErrors = async <T>(code: () => Promise<T>): Promise<T> => {
+  try {
+    return await code();
+  } catch (error) {
+    if (error && typeof error === 'object') {
+      switch (error.name) {
+        case 'ResourceNotFound':
+        case 'CollectionNotFound':
+          throw new DOMException(error.message, 'BlobNotFound');
+      }
+    }
+    throw error;
+  }
+};
+
+export class CrudCasBase<Hash> implements CasApi<Hash> {
+  constructor(
+    protected readonly crud: CrudApi,
+    protected readonly hash: (blob: Uint8Array) => Promise<Hash>,
+    protected readonly hash2Loc: (hash: Hash) => FsLocation,
+    protected readonly hashEqual: (h1: Hash, h2: Hash) => boolean,
+  ) {}
+
+  public readonly put = async (blob: Uint8Array): Promise<Hash> => {
+    const digest = await this.hash(blob);
+    const [collection, resource] = this.hash2Loc(digest);
+    await this.crud.put(collection, resource, blob);
+    return digest;
+  };
+
+  public readonly get = async (hash: Hash, options?: CasGetOptions): Promise<Uint8Array> => {
+    const [collection, resource] = this.hash2Loc(hash);
+    return await normalizeErrors(async () => {
+      const blob = await this.crud.get(collection, resource);
+      if (!options?.skipVerification) {
+        const digest = await this.hash(blob);
+        if (!this.hashEqual(digest, hash)) throw new DOMException('Blob contents does not match hash', 'Integrity');
+      }
+      return blob;
+    });
+  };
+
+  public readonly del = async (hash: Hash, silent?: boolean): Promise<void> => {
+    const [collection, resource] = this.hash2Loc(hash);
+    await normalizeErrors(async () => {
+      return await this.crud.del(collection, resource, silent);
+    });
+  };
+
+  public readonly info = async (hash: Hash): Promise<CrudResourceInfo> => {
+    const [collection, resource] = this.hash2Loc(hash);
+    return await normalizeErrors(async () => {
+      return await this.crud.info(collection, resource);
+    });
+  };
+}

src/crud-to-cas/__tests__/CrudCasBase.test.ts

@@ -0,0 +1,71 @@
+import { memfs } from '../..';
+import { onlyOnNode20 } from '../../__tests__/util';
+import { NodeFileSystemDirectoryHandle } from '../../node-to-fsa';
+import { FsaCrud } from '../../fsa-to-crud/FsaCrud';
+import { createHash } from 'crypto';
+import { hashToLocation } from '../util';
+import { CrudCasBase } from '../CrudCasBase';
+import { FsLocation } from '../../fsa-to-node/types';
+
+onlyOnNode20('CrudCas on FsaCrud', () => {
+  const setup = () => {
+    const { fs } = memfs();
+    const fsa = new NodeFileSystemDirectoryHandle(fs, '/', { mode: 'readwrite' });
+    const crud = new FsaCrud(fsa);
+    return { fs, fsa, crud, snapshot: () => (<any>fs).__vol.toJSON() };
+  };
+
+  test('can use a custom hashing digest type', async () => {
+    const { crud } = setup();
+    class Hash {
+      constructor(public readonly digest: string) {}
+    }
+    const hash = async (blob: Uint8Array): Promise<Hash> => {
+      const shasum = createHash('sha1');
+      shasum.update(blob);
+      const digest = shasum.digest('hex');
+      return new Hash(digest);
+    };
+    const cas = new CrudCasBase<Hash>(
+      crud,
+      hash,
+      (id: Hash) => hashToLocation(id.digest),
+      (h1: Hash, h2: Hash) => h1.digest === h2.digest,
+    );
+    const blob = Buffer.from('hello world');
+    const id = await cas.put(blob);
+    expect(id).toBeInstanceOf(Hash);
+    const id2 = await hash(blob);
+    expect(id.digest).toEqual(id2.digest);
+    const blob2 = await cas.get(id);
+    expect(String.fromCharCode(...blob2)).toEqual('hello world');
+    expect(await cas.info(id)).toMatchObject({ size: 11 });
+    await cas.del(id2);
+    expect(() => cas.info(id)).rejects.toThrowError();
+  });
+
+  test('can use custom folder sharding strategy', async () => {
+    const { crud } = setup();
+    const hash = async (blob: Uint8Array): Promise<string> => {
+      const shasum = createHash('sha1');
+      shasum.update(blob);
+      return shasum.digest('hex');
+    };
+    const theCustomFolderShardingStrategy = (h: string): FsLocation => [[h[0], h[1], h[2]], h[3]];
+    const cas = new CrudCasBase<string>(
+      crud,
+      hash,
+      theCustomFolderShardingStrategy,
+      (h1: string, h2: string) => h1 === h2,
+    );
+    const blob = Buffer.from('hello world');
+    const id = await cas.put(blob);
+    expect(typeof id).toBe('string');
+    const id2 = await hash(blob);
+    expect(id).toBe(id2);
+    const blob2 = await cas.get(id);
+    expect(String.fromCharCode(...blob2)).toEqual('hello world');
+    const blob3 = await crud.get([id2[0], id2[1], id2[2]], id2[3]);
+    expect(String.fromCharCode(...blob3)).toEqual('hello world');
+  });
+});

src/crud-to-cas/__tests__/testCasfs.ts

@@ -16,7 +16,7 @@ const b = (str: string) => {
 };
 
 export type Setup = () => {
-  cas: CasApi;
+  cas: CasApi<string>;
   crud: CrudApi;
   snapshot: () => Record<string, string | null>;
 };