docs: asset canister to split up very large asset contents into chunks for retrieval

docs/design/asset-canister.adoc

@@ -48,17 +48,18 @@ Store per asset:
 * Content type
 * Content for one or more content encodings
 ** Content encoding
-** Content blob
+** The actual content (bytes)
+*** Split up into chunks for retrieval
 
 === Implementation
 
-We will implement the canister in Motoko using existing stable memory data structures such as RBTree or Trie.
+We will implement the canister in Motoko, because the canister's functionality
+is within the realm of what might be found in a typical application canister.
 
 === Considered Solutions
 
 ==== Motoko canister
 
-* The API will be straightforward to implement using existing stable-memory data structures
 * Currently there is not a large body of Motoko libraries for things like compression
 ** Mitigated by: compression would probably be better done in dfx anyway
 
@@ -140,107 +141,113 @@ canister per block height.
 [source,candid]
 ----
 
-type Contents = blob;
-type BlobId = text;
+type BatchId = nat;
+type ChunkId = nat;
 type Key = text;
 
-type AssetInfo = record {
-  key: Key;
-  content_type: text;
-  encoding_details: vec record {
-    content_encoding: text;
-  };
-};
-
 // Create a new asset.  Contents will be attached later with SetContent.
 //   - No-op if asset already exists with the same content type.
 //   - Error if asset already exists with a different content type (delete first).
-type CreateAssetOperation = record {
+type CreateAssetArguments = record {
   key: Key;
   content_type: text;
 };
 
 // Add or change content for an asset, by content encoding
-type SetAssetContentOperation = record {
+type SetAssetContentArguments = record {
   key: Key;
   content_encoding: text;
-  blob_id: BlobId;
+  chunk_ids: vec ChunkId;
 };
 
 // Remove content for an asset, by content encoding
-type UnsetAssetContentOperation = record {
+type UnsetAssetContentArguments = record {
   key: Key;
   content_encoding: text;
 };
 
 // Delete an asset
-type DeleteAssetOperation = record {
+type DeleteAssetArguments = record {
   key: Key;
 };
 
 // Future: set up access control
-type SetAssetAclOperation = record {
+type SetAssetAclArguments = record {
   key: Key;
   tbd: text;
 };
 
 // Future: set a time after which to delete an asset
-type SetAssetExpiryOperation = record {
+type SetAssetExpiryArguments = record {
   key: Key;
   tbd: text;
 };
 
 // Reset everything
-type ClearOperation = record {};
+type ClearArguments = record {};
 
 type BatchOperationKind = variant {
-  Create: CreateAssetOperation;
-  SetContent: SetAssetContentOperation;
+  CreateAsset: CreateAssetArguments;
+  SetAssetContent: SetAssetContentArguments;
 
-  UnsetContent: UnsetAssetContentOperation;
-  Delete: DeleteAssetOperation;
+  UnsetAssetContent: UnsetAssetContentArguments;
+  DeleteAsset: DeleteAssetArguments;
 
-  SetAcl: SetAssetAclOperation;
-  SetExpiry: SetAssetExpiryOperation;
+  SetAssetAcl: SetAssetAclArguments;
+  SetAssetExpiry: SetAssetExpiryArguments;
 
-  Clear: ClearOperation;
+  Clear: ClearArguments;
 };
 
 service: {
 
   get: (record {
     key: Key,
     accept_encodings: vec text
-  }) -> (record { contents: blob; content_type: text; content_encoding: text }) query;
+  }) -> (record {
+    content: blob; // may be the entirety of the content, or just chunk index 0
+    content_type: text;
+    content_encoding: text,
+    total_length: nat // all chunks except last have size == content.size()
+  }) query;
+
+  // if get() returned chunks > 1, call this to retrieve them.
+  // chunks may or may not be split up at the same boundaries as presented to create_chunk().
+  get_chunk: (record {
+    key: Key,
+    content_encoding: text,
+    index: nat
+  }) -> (record { content: blob }) query;
 
-  list: (record {}) -> (vec AssetInfo) query;
+  list: (record {}) -> (vec record {
+    key: Key;
+    content_type: text;
+    encodings: vec record {
+      content_encoding: text;
+    };
+  }) query;
 
-  // allocate space for content
-  //
-  // first deletes all blobs created by an earlier call to create_blobs(), that have not been set to an asset,
-  // only if write_blob() has not been called for any of those blobs within the past 5 minutes.
-  create_blobs: (record { blob_info: vec record { length: nat32 } } ) -> (record { blob_ids: vec BlobId });
+  create_batch(record {}) -> (record { batch_id: BatchId });
 
-  // upload part of a blob's content
-  write_blob: (record { blob_id: BlobId; offset: nat32; contents: blob }) -> ();
+  create_chunk: (record { batch_id: BatchId; content: blob }) -> (record { chunk_id: ChunkId });
 
   // Perform all operations successfully, or reject
-  batch: (vec BatchOperationKind) -> ();
+  commit_batch: (record { batch_id: BatchId; operations: vec BatchOperationKind }) -> ();
 
-  create_asset: (CreateAssetOperation) -> ();
-  set_asset_content: (SetAssetContentOperation) -> ();
-  unset_asset_content: (UnsetAssetContentOperation) -> ();
+  create_asset: (CreateAssetArguments) -> ();
+  set_asset_content: (SetAssetContentArguments) -> ();
+  unset_asset_content: (UnsetAssetContentArguments) -> ();
 
-  delete_asset: (DeleteAssetOperation) -> ();
+  delete_asset: (DeleteAssetArguments) -> ();
 
-  set_asset_acl: (SetAssetAclOperation) -> ();
-  set_asset_expiry: (SetAssetExpiryOperation) -> ();
+  set_asset_acl: (SetAssetAclArguments) -> ();
+  set_asset_expiry: (SetAssetExpiryArguments) -> ();
 
-  clear: (ClearOperation) -> ();
+  clear: (ClearArguments) -> ();
 
   // Single call to create an asset with content for a single content encoding that
   // fits within the message ingress limit.
-  store: (record { key: Key; content_type: text; content_encoding: text; contents: blob }) -> ();
+  store: (record { key: Key; content_type: text; content_encoding: text; content: blob }) -> ();
 }
 
 ----