feat(backup): introduce new schema to minimize backup length (#333)

Signed-off-by: jeyem <[email protected]>

src/domain/backup/index.ts

@@ -4,7 +4,12 @@ import { Schema as v0_0_1 } from "./v0_0_1";
  * All supported backup schemas
  */
 export type Schema = v0_0_1;
+export type Version = "0.0.1";
 
 export const defaultVersion = "0.0.1";
 
+export const versions: Version[] = [
+  "0.0.1",
+];
+
 export { v0_0_1 };

src/domain/buildingBlocks/Pluto.ts

@@ -41,7 +41,7 @@ export interface Pluto {
   /**
    * create a Backup object from the stored data
    */
-  backup(): Promise<Backup.Schema>;
+  backup(version?: Backup.Version): Promise<Backup.Schema>;
 
   /**
    * load the given data into the store

src/edge-agent/Agent.Backup.ts

@@ -1,54 +1,96 @@
+import Pako from "pako";
 import * as Domain from "../domain";
 import Agent from "./Agent";
+import { Version } from "../domain/backup";
 import { isObject, validateSafe } from "../utils";
 
+
 /**
  * define Agent requirements for Backup
  */
 type BackupAgent = Pick<Agent, "apollo" | "pluto" | "pollux" | "seed">;
+type BackupExclude = "messages" | "mediators" | "link_secret";
+
+type MasterKey = Domain.PrivateKey & Domain.ExportableKey.Common & Domain.ExportableKey.JWK & Domain.ExportableKey.PEM
+
+export type BackupOptions = {
+  version?: Version
+  key?: MasterKey
+  compress?: boolean
+  excludes?: BackupExclude[]
+}
 
 export class AgentBackup {
   constructor(
     public readonly Agent: BackupAgent
   ) {}
 
   /**
-   * create JWE of data stored in Pluto
-   * 
-   * @returns {string}
-   * @see restore
-   */
-  async createJWE(): Promise<string> {
+ * Creates a JWE (JSON Web Encryption) containing the backup data stored in Pluto.
+ * The data can optionally be encrypted using a custom master key, compressed, 
+ * and filtered to exclude specified fields.
+ * 
+ * @param {BackupOptions} [options] - Optional settings for the backup.
+ * @param {Version} [options.version] - Specifies the version of the backup data.
+ * @param {MasterKey} [options.key] - Custom master key used for encrypting the backup.
+ * @param {boolean} [options.compress] - If true, compresses the JWE using DEFLATE.
+ * @param {BackupExclude[]} [options.excludes] - Keys to exclude from the backup data 
+ * (e.g., "messages", "mediators", "link_secret"). Arrays are cleared, and strings are set to empty strings.
+ * 
+ * @returns {Promise<string>} - A promise that resolves to the JWE string.
+ * 
+ * @see restore - Method to restore data from a JWE string.
+ */
+  async createJWE(options?: BackupOptions): Promise<string> {
     await this.Agent.pollux.start();
-    const backup = await this.Agent.pluto.backup();
-    const masterSk = await this.masterSk();
+
+    let backup = await this.Agent.pluto.backup(options?.version);
+    if (options?.excludes && Array.isArray(options.excludes)) {
+      backup = this.applyExclusions(backup, options.excludes);
+    }
+    const backupStr = options?.compress ? this.compress(JSON.stringify(backup)) : JSON.stringify(backup);
+    const masterSk = options?.key ?? await this.masterSk();
     const jwk = masterSk.to.JWK();
-    const jwe = this.Agent.pollux.jwe.JWE.encrypt(
-      JSON.stringify(backup),
+
+    return this.Agent.pollux.jwe.JWE.encrypt(
+      backupStr,
       JSON.stringify(jwk),
-      'backup'
+      'backup',
     );
-    return jwe;
   }
 
   /**
-   * decode JWE and save data to store
+   * Decodes a JWE (JSON Web Encryption) string and restores the backup data to the store.
+   * If the JWE is compressed (Base64-encoded), it will attempt to decompress it first.
+   * 
+   * @param {string} jwe - The JWE string containing the encrypted backup data.
+   * @param {BackupOptions} [options] - Optional settings for the backup.
+   * @param {Version} [options.version] - Specifies the version of the restore data.
+   * @param {MasterKey} [options.key] - Custom master key used for decrypting the backup.
+   * @param {boolean} [options.compress] - If true, compresses the JWE using INFLATE.
+   * 
+   * @returns {Promise<void>} - A promise that resolves when the data is successfully restored.
    * 
-   * @param jwe 
-   * @see backup
+   * @see createJWE - Method to create a JWE from the stored backup data.
    */
-  async restore(jwe: string) {
+  async restore(jwe: string, options?: BackupOptions) {
     await this.Agent.pollux.start();
-    const masterSk = await this.masterSk();
+    const masterSk = options?.key ?? await this.masterSk();
+
     const jwk = masterSk.to.JWK();
     const decoded = this.Agent.pollux.jwe.JWE.decrypt(
       jwe,
       'backup',
-      JSON.stringify(jwk)
+      JSON.stringify(jwk),
     );
-    const jsonStr = Buffer.from(decoded).toString();
-    const json = JSON.parse(jsonStr);
-    const backup = this.parseBackupJson(json);
+    let jsonStr: string;
+    if (options?.compress) {
+      jsonStr = this.decompress(new TextDecoder().decode(decoded));
+    } else {
+      jsonStr = Buffer.from(decoded).toString();
+    }    
+    const json = JSON.parse(jsonStr);    
+    const backup = this.parseBackupJson(json);    
     await this.Agent.pluto.restore(backup);
   }
 
@@ -60,12 +102,42 @@ export class AgentBackup {
           if (validateSafe(json, Domain.Backup.v0_0_1)) {
             return json;
           }
+          break;
       }
     }
-
     throw new Domain.AgentError.BackupVersionError();
   }
 
+  /**
+ * Compresses a JSON object into a Base64-encoded string using DEFLATE.
+ * 
+ * - Uses `level: 9` for maximum compression and `strategy: Z_FILTERED` 
+ *   (optimized for repetitive patterns, common in JSON data).
+ * - Converts the JSON to a string, compresses it, and encodes it in Base64.
+ * 
+ * @param {unknown} json - The JSON object to compress.
+ * @returns {string} - The Base64-encoded compressed string.
+ */
+  private compress(json: unknown): string {
+    // Strategy 1 is 
+    return Buffer.from(Pako.deflate(JSON.stringify(json), {level: 9, strategy: 1})).toString('base64');
+  }
+
+  /**
+ * Decompresses a Base64-encoded string into its original JSON representation.
+ * 
+ * - Decodes the Base64 string to a binary buffer.
+ * - Uses DEFLATE to decompress the data and converts it back to a JSON string.
+ * - Parses and returns the JSON object.
+ * 
+ * @param {string} data - The Base64-encoded compressed string.
+ * @returns {string} - The decompressed JSON string.
+ */
+  private decompress(data: string): string {
+    const compressedData = Buffer.from(data, 'base64');
+    return JSON.parse(Pako.inflate(compressedData, {to: 'string'}));
+  }
+
   /**
    * create a JWK for the MasterKey (X25519)
    * @returns JWK
@@ -82,4 +154,27 @@ export class AgentBackup {
     }
     return masterKey;
   }
+  /**
+ * Modifies the backup object by applying exclusions.
+ * Sets excluded array values to empty arrays and string values to empty strings.
+ *
+ * @param {Domain.Backup.Schema} backup - The backup object to be modified.
+ * @param {BackupExclude[]} excludes - An array of keys to exclude from the backup.
+ * @returns {Domain.Backup.Schema} The modified backup object.
+ */
+  private applyExclusions(backup: Domain.Backup.Schema, excludes: BackupExclude[]): Domain.Backup.Schema {
+    const tmp = {...backup};
+    for (const exclude of excludes) {
+      switch (exclude) {
+        case "messages":
+        case "mediators":
+          tmp[exclude] = [];
+          break;
+        case "link_secret":
+          tmp[exclude] = undefined;
+          break;
+      }
+    }
+    return tmp;
+  }
 }

src/pluto/Pluto.ts

@@ -5,6 +5,7 @@ import { PeerDID } from "../peer-did/PeerDID";
 import { BackupManager } from "./backup/BackupManager";
 import { PlutoRepositories, repositoryFactory } from "./repositories";
 import { Arrayable, asArray } from "../utils";
+import { Version } from "../domain/backup";
 
 
 /**
@@ -121,8 +122,8 @@ export class Pluto implements Domain.Pluto {
   }
 
   /** Backups **/
-  backup() {
-    return this.BackupMgr.backup();
+  backup(version?: Version) {
+    return this.BackupMgr.backup(version);
   }
 
   restore(backup: Domain.Backup.Schema) {

src/pluto/backup/BackupManager.ts

@@ -4,6 +4,7 @@ import { Pluto } from "../Pluto";
 import { PlutoRepositories } from "../repositories";
 import { isEmpty } from "../../utils";
 import { IBackupTask, IRestoreTask } from "./versions/interfaces";
+import { Version } from "../../domain/backup";
 
 /**
  * BackupManager
@@ -21,7 +22,7 @@ export class BackupManager {
    * @param version - backup schema version
    * @returns {Promise<Domain.Backup.Schema>}
    */
-  backup(version?: string) {
+  backup(version?: Version) {
     const task = this.getBackupTask(version);
     return task.run();
   }
@@ -37,7 +38,7 @@ export class BackupManager {
     await task.run();
   }
 
-  private getBackupTask(version: string = Domain.Backup.defaultVersion): IBackupTask {
+  private getBackupTask(version: Version = Domain.Backup.defaultVersion): IBackupTask {
     switch (version) {
       case "0.0.1":
         return new Versions.v0_0_1.BackupTask(this.Pluto, this.Repositories);
@@ -48,7 +49,6 @@ export class BackupManager {
 
   private getRestoreTask(backup: Domain.Backup.Schema): IRestoreTask {
     const version = backup.version ?? Domain.Backup.defaultVersion;
-
     switch (version) {
       case "0.0.1":
         return new Versions.v0_0_1.RestoreTask(this.Pluto, backup);

src/pluto/backup/versions/0_0_1/Backup.ts

@@ -68,7 +68,6 @@ export class BackupTask implements IBackupTask {
           index: key.index,
           did: did?.uuid,
         };
-
         return acc.concat(backup);
       }