copying of fix for objectstore from nats-io/nats.deno#746 (#205)

Signed-off-by: Alberto Ricart <[email protected]>

migration.md

@@ -89,7 +89,8 @@ these modules for cross-runtime consumption.
   make a request, the request could fail with a RequestError or TimeoutError.
   The RequestError in turn will contain the `cause` such as `NoRespondersError`.
   This also means that in TypeScript, the callback signature has been relaxed to
-  just `(Error, Msg)=>void`. For more information see the JsDocs.
+  just `(Error, Msg)=>void | Promise<never>`. For more information see the
+  JsDocs.
 - Previous versions of the client provided the enums `Events` and `DebugEvents`
   for the `type` values in the notification received via
   `status(): AsyncIterable<Status>` iterator. Starting with this release,
@@ -102,7 +103,8 @@ these modules for cross-runtime consumption.
 - RequestStrategy "Jitter" is now called "stall" to adopt the term used by new
   implementations in other clients and the RequestStrategy enum is now a type
   alias to simple strings "timer", "count", "stall", "sentinel".
-- `SHA256` a support type for object store has been moved to @nats-io/obj
+- `SHA256` a support type for object store has been removed. Client replaced
+  this library with a dependency.
 - `Base64Codec`, `Base64UrlCodec`, `Base64UrlPaddedCodec` support types for
   object store have been moved to @nats-io/obj.
 
@@ -207,6 +209,24 @@ Use `KvOptions.storage` and `KvStatus.storage`.
 > }
 > ```
 
+> [!IMPORTANT]
+>
+> Clients that use @nats-io/obj prior to 3.0.0-34, potentially generated sha-256
+> digests that were incorrect due to a bug on a 3rd party library bundled in the
+> client. The library has been replaced with npm:js-sha256. Errors in the
+> digests usually affected 500 MB or larger objects.
+>
+> If you are upgrading from a previous pre-release or from the previous
+> generation clients (these were also updated) you may want to re-put your
+> objects or use the tool
+> [fix-os-hashes](https://github.com/nats-io/nats.deno/blob/main/bin/fix-os-hashes.ts)
+> to be compatible with 3.0.0-34 and beyond (and other NATS tools). To run the
+> tool install [deno](https://deno.com/) and:
+>
+> ```
+> deno run -A https://github.com/nats-io/nats.deno/blob/main/bin/fix-os-hashes.ts -h
+> ```
+
 To use ObjectStore, you must install and import `@nats-io/obj`.
 
 ### Watch

obj/deno.json

@@ -34,6 +34,7 @@
   },
   "imports": {
     "@nats-io/nats-core": "jsr:@nats-io/nats-core@~3.0.0-50",
-    "@nats-io/jetstream": "jsr:@nats-io/jetstream@~3.0.0-37"
+    "@nats-io/jetstream": "jsr:@nats-io/jetstream@~3.0.0-37",
+    "js-sha256": "npm:[email protected]"
   }
 }

obj/package.json

@@ -34,7 +34,8 @@
   "description": "obj library - this library implements all the base functionality for NATS objectstore for javascript clients",
   "dependencies": {
     "@nats-io/jetstream": "3.0.0-37",
-    "@nats-io/nats-core": "3.0.0-50"
+    "@nats-io/nats-core": "3.0.0-50",
+    "js-sha256": "^0.11.0"
   },
   "devDependencies": {
     "@types/node": "^22.10.10",

obj/src/internal_mod.ts

@@ -16,6 +16,4 @@ export { StorageType } from "./types.ts";
 
 export { Objm } from "./objectstore.ts";
 
-export { SHA256, sha256 } from "./sha256.ts";
-
 export { Base64Codec, Base64UrlCodec, Base64UrlPaddedCodec } from "./base64.ts";

obj/src/objectstore.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2024 The NATS Authors
+ * Copyright 2022-2025 The NATS Authors
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -68,8 +68,9 @@ import type {
   ObjectStoreStatus,
   ObjectWatchInfo,
 } from "./types.ts";
-import { SHA256 } from "./sha256.ts";
 import { Base64UrlPaddedCodec } from "./base64.ts";
+import { sha256 } from "js-sha256";
+import { checkSha256, parseSha256 } from "./sha_digest.parser.ts";
 
 export const osPrefix = "OBJ_";
 export const digestType = "SHA-256=";
@@ -470,7 +471,7 @@ export class ObjectStoreImpl implements ObjectStore {
     const db = new DataBuffer();
     try {
       const reader = rs ? rs.getReader() : null;
-      const sha = new SHA256();
+      const sha = sha256.create();
 
       while (true) {
         const { done, value } = reader
@@ -491,10 +492,11 @@ export class ObjectStoreImpl implements ObjectStore {
 
           // prepare the metadata
           info.mtime = new Date().toISOString();
-          const digest = sha.digest("base64");
-          const pad = digest.length % 3;
-          const padding = pad > 0 ? "=".repeat(pad) : "";
-          info.digest = `${digestType}${digest}${padding}`;
+          const digest = Base64UrlPaddedCodec.encode(
+            Uint8Array.from(sha.digest()),
+          );
+          info.digest = `${digestType}${digest}`;
+
           info.deleted = false;
 
           // trailing md for the object
@@ -640,6 +642,16 @@ export class ObjectStoreImpl implements ObjectStore {
       return os.get(ln);
     }
 
+    if (!info.digest.startsWith(digestType)) {
+      return Promise.reject(new Error(`unknown digest type: ${info.digest}`));
+    }
+    const digest = parseSha256(info.digest.substring(8));
+    if (digest === null) {
+      return Promise.reject(
+        new Error(`unable to parse digest: ${info.digest}`),
+      );
+    }
+
     const d = deferred<Error | null>();
 
     const r: Partial<ObjectResult> = {
@@ -652,7 +664,7 @@ export class ObjectStoreImpl implements ObjectStore {
       return Promise.resolve(r as ObjectResult);
     }
 
-    const sha = new SHA256();
+    const sha = sha256.create();
     let controller: ReadableStreamDefaultController;
 
     const cc: Partial<ConsumerConfig> = {};
@@ -667,12 +679,8 @@ export class ObjectStoreImpl implements ObjectStore {
           controller!.enqueue(jm.data);
         }
         if (jm.info.pending === 0) {
-          const hash = sha.digest("base64");
-          // go pads the hash - which should be multiple of 3 - otherwise pads with '='
-          const pad = hash.length % 3;
-          const padding = pad > 0 ? "=".repeat(pad) : "";
-          const digest = `${digestType}${hash}${padding}`;
-          if (digest !== info.digest) {
+          const digest = Uint8Array.from(sha.digest());
+          if (!checkSha256(digest, Uint8Array.from(sha.digest()))) {
             controller!.error(
               new Error(
                 `received a corrupt object, digests do not match received: ${info.digest} calculated ${digest}`,