Feat: Add MEDIA_UPLOAD_CHUNK_SIZE

Vija02 · Jan 24, 2025 · 80fed38 · 80fed38
1 parent 2dcebf4
commit 80fed38
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 10 deletions.
diff --git a/backend/server/src/middleware/installRemote.ts b/backend/server/src/middleware/installRemote.ts
@@ -76,10 +76,11 @@ function transformer(html: string, req: Request) {
 
   // Extra data
   const extraEnv = [
-    { ROOT_URL: process.env.ROOT_URL, CSRF_TOKEN: req.csrfToken() } as Record<
-      string,
-      string
-    >,
+    {
+      ROOT_URL: process.env.ROOT_URL,
+      CSRF_TOKEN: req.csrfToken(),
+      MEDIA_UPLOAD_CHUNK_SIZE: process.env.MEDIA_UPLOAD_CHUNK_SIZE,
+    } as Record<string, string>,
   ]
     .concat(registeredEnvToViews.map((x) => x.envVars))
     .reduce((acc, val) => ({ ...acc, ...val }), {} as Record<string, string>);

diff --git a/backend/server/src/middleware/installRenderer.ts b/backend/server/src/middleware/installRenderer.ts
@@ -78,10 +78,11 @@ function transformer(html: string, req: Request) {
 
   // Extra data
   const extraEnv = [
-    { ROOT_URL: process.env.ROOT_URL, CSRF_TOKEN: req.csrfToken() } as Record<
-      string,
-      string
-    >,
+    {
+      ROOT_URL: process.env.ROOT_URL,
+      CSRF_TOKEN: req.csrfToken(),
+      MEDIA_UPLOAD_CHUNK_SIZE: process.env.MEDIA_UPLOAD_CHUNK_SIZE,
+    } as Record<string, string>,
   ]
     .concat(registeredEnvToViews.map((x) => x.envVars))
     .reduce((acc, val) => ({ ...acc, ...val }), {} as Record<string, string>);

diff --git a/packages/lib/src/appData.ts b/packages/lib/src/appData.ts
@@ -1,7 +1,22 @@
 const getRootURL = () => (window as any)?.__APP_DATA__?.ROOT_URL;
 const getCSRFToken = () => (window as any)?.__APP_DATA__?.CSRF_TOKEN;
+const getMediaUploadChunkSize = () => {
+  const val = parseInt(
+    (window as any)?.__APP_DATA__?.MEDIA_UPLOAD_CHUNK_SIZE,
+    10,
+  );
+  if (Number.isSafeInteger(val)) {
+    return val;
+  }
+  return Infinity;
+};
 
 const getCustomEnv = (envName: string) =>
   (window as any)?.__APP_DATA__?.[envName];
 
-export const appData = { getRootURL, getCSRFToken, getCustomEnv };
+export const appData = {
+  getRootURL,
+  getCSRFToken,
+  getMediaUploadChunkSize,
+  getCustomEnv,
+};
diff --git a/plugins/audio-recorder/view/useAudioRecording.tsx b/plugins/audio-recorder/view/useAudioRecording.tsx
@@ -299,7 +299,7 @@ function startUpload(
 ) {
   const endpoint = pluginApi.media.tusUploadUrl;
   // DEBT: Maybe need bigger chunkSize
-  const chunkSize = 15000; // 15kb. Roughly every second
+  const chunkSize = Math.min(15000, pluginApi.env.getMediaUploadChunkSize()); // 15kb. Roughly every second
 
   const options: ConstructorParameters<typeof tus.Upload>[1] = {
     endpoint,

diff --git a/plugins/simple-image/view/ImageRemote/UploadModal.tsx b/plugins/simple-image/view/ImageRemote/UploadModal.tsx
@@ -17,6 +17,7 @@ export const UploadModal = () => {
         "csrf-token": appData.getCSRFToken(),
         "organization-id": pluginApi.pluginContext.organizationId,
       },
+      chunkSize: pluginApi.env.getMediaUploadChunkSize(),
     }),
   );
   const sceneData = pluginApi.scene.useValtioData();

diff --git a/plugins/video-player/view/Remote/UploadVideoModal.tsx b/plugins/video-player/view/Remote/UploadVideoModal.tsx
@@ -32,13 +32,15 @@ const UploadVideoModal = ({
   ...props
 }: UploadVideoModalPropTypes) => {
   const pluginApi = usePluginAPI();
+
   const [uppy] = useState(() =>
     new Uppy().use(Tus, {
       endpoint: pluginApi.media.tusUploadUrl,
       headers: {
         "csrf-token": appData.getCSRFToken(),
         "organization-id": pluginApi.pluginContext.organizationId,
       },
+      chunkSize: pluginApi.env.getMediaUploadChunkSize(),
     }),
   );
   const mutableSceneData = pluginApi.scene.useValtioData();

diff --git a/scripts/_setup_utils.js b/scripts/_setup_utils.js
@@ -190,6 +190,14 @@ exports.updateDotenv = function updateDotenv(add, answers) {
 # For any other values, this option is disabled`,
   );
 
+  add(
+    "MEDIA_UPLOAD_CHUNK_SIZE",
+    "100000000",
+    `\
+# How big each chunk of a file upload should be. This is useful in environments where there is a limit to how big a file upload can be.
+# Big files will be chunked into the provided size. Default is 100000000 which is 100mb`,
+  );
+
   add(
     "VITE_APP_OPENOBSERVE_CLIENT_TOKEN",
     `\