Merge pull request #335 from zurdi15/fix-322

Fix multi rom zipping + streaming

backend/endpoints/rom.py

@@ -1,12 +1,13 @@
-import io
-import tempfile
-import zipfile
+from datetime import datetime
 from fastapi import APIRouter, Request, status, HTTPException
 from fastapi_pagination.ext.sqlalchemy import paginate
 from fastapi_pagination.cursor import CursorPage, CursorParams
 from fastapi.responses import FileResponse
 from pydantic import BaseModel, BaseConfig
 
+from stat import S_IFREG
+from stream_zip import ZIP_64, stream_zip
+
 from logger.logger import log
 from handler import dbh
 from utils import fs, get_file_name_with_no_tags
@@ -15,6 +16,8 @@
 from models.platform import Platform
 from config import LIBRARY_BASE_PATH
 
+from .utils import CustomStreamingResponse
+
 router = APIRouter()
 
 
@@ -76,31 +79,29 @@ def download_rom(id: int, files: str):
     rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}"
 
     if not rom.multi:
-        return FileResponse(
-            path=rom_path,
-            filename=rom.file_name,
-            media_type="application/octet-stream",
-        )
-
-    mf = io.BytesIO()
-    with zipfile.ZipFile(mf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
-        try:
-            for file_name in files.split(","):
-                zf.write(f"{rom_path}/{file_name}", file_name)
-        except FileNotFoundError as e:
-            log.error(str(e))
-        finally:
-            zf.close()
-
-    tmp = tempfile.NamedTemporaryFile(delete=False)
-    with open(tmp.name, "wb") as f:
-        f.write(mf.getvalue())
-
-        return FileResponse(
-            path=tmp.name,
-            filename=f"{rom.r_name}.zip",
-            media_type="application/octet-stream",
-        )
+        return FileResponse(path=rom_path, filename=rom.file_name)
+
+    # Builds a generator of tuples for each member file
+    def local_files():
+        def contents(file_name):
+            with open(f"{rom_path}/{file_name}", "rb") as f:
+                while chunk := f.read(65536):
+                    yield chunk
+
+        return [
+            (file_name, datetime.now(), S_IFREG | 0o600, ZIP_64, contents(file_name))
+            for file_name in files.split(",")
+        ]
+
+    zipped_chunks = stream_zip(local_files())
+
+    # Streams the zip file to the client
+    return CustomStreamingResponse(
+        zipped_chunks,
+        media_type="application/zip",
+        headers={"Content-Disposition": f"attachment; filename={rom.r_name}.zip"},
+        emit_body={"id": rom.id},
+    )
 
 
 @router.get("/platforms/{p_slug}/roms", status_code=200)

backend/endpoints/utils.py

@@ -0,0 +1,13 @@
+from fastapi.responses import StreamingResponse
+
+from utils.socket import socket_server
+
+
+class CustomStreamingResponse(StreamingResponse):
+    def __init__(self, *args, **kwargs) -> None:
+        self.emit_body = kwargs.pop("emit_body", None)
+        super().__init__(*args, **kwargs)
+
+    async def stream_response(self, *args, **kwargs) -> None:
+        await super().stream_response(*args, **kwargs)
+        await socket_server.emit("download:complete", self.emit_body)

frontend/src/components/Game/Card/ActionBar.vue

@@ -18,7 +18,7 @@ const downloadUrl = `${window.location.origin}${props.rom.download_path}`;
         <template v-if="rom.multi">
           <v-btn
             @click="downloadRomApi(rom)"
-            :disabled="downloadStore.value.includes(rom.file_name)"
+            :disabled="downloadStore.value.includes(rom.id)"
             icon="mdi-download"
             size="x-small"
             rounded="0"

frontend/src/components/Game/Card/Cover.vue

@@ -14,7 +14,7 @@ const props = defineProps(["rom", "isHovering", "hoverProps", "size"]);
   >
     <v-progress-linear
       color="rommAccent1"
-      :active="downloadStore.value.includes(rom.file_name)"
+      :active="downloadStore.value.includes(rom.id)"
       :indeterminate="true"
       absolute
     />

frontend/src/components/Game/ListItem/Item.vue

@@ -45,7 +45,7 @@ const downloadUrl = `${window.location.origin}${props.rom.download_path}`;
           <v-avatar :rounded="0">
             <v-progress-linear
               color="rommAccent1"
-              :active="downloadStore.value.includes(rom.file_name)"
+              :active="downloadStore.value.includes(rom.id)"
               :indeterminate="true"
               absolute
             />
@@ -69,7 +69,7 @@ const downloadUrl = `${window.location.origin}${props.rom.download_path}`;
       <template v-if="rom.multi">
         <v-btn
           @click="downloadRomApi(rom)"
-          :disabled="downloadStore.value.includes(rom.file_name)"
+          :disabled="downloadStore.value.includes(rom.id)"
           icon="mdi-download"
           size="x-small"
           rounded="0"

frontend/src/services/api.js

@@ -1,5 +1,6 @@
 import axios from "axios";
 import useDownloadStore from "@/stores/download.js";
+import socket from "@/services/socket.js";
 
 export async function fetchPlatformsApi() {
   return axios.get("/api/platforms");
@@ -20,31 +21,31 @@ export async function fetchRomApi(platform, rom) {
   return axios.get(`/api/platforms/${platform}/roms/${rom}`);
 }
 
+// Listen for multi-file download completion events
+socket.on("download:complete", ({ id }) => {
+  const downloadStore = useDownloadStore();
+  useDownloadStore().remove(id);
+
+  // Disconnect socket when no more downloads are in progress
+  if (downloadStore.value.length === 0) socket.disconnect();
+});
+
+// Used only for multi-file downloads
 export async function downloadRomApi(rom, files) {
   // Force download of all multirom-parts when no part is selected
   if (files != undefined && files.length == 0) {
     files = undefined;
   }
 
-  const downloadStore = useDownloadStore();
-  downloadStore.add(rom.file_name);
+  const a = document.createElement("a");
+  a.href = `/api/platforms/${rom.p_slug}/roms/${rom.id}/download?files=${
+    files || rom.files
+  }`;
+  a.download = `${rom.r_name}.zip`;
+  a.click();
 
-  axios
-    .get(
-      `/api/platforms/${rom.p_slug}/roms/${rom.id}/download?files=${
-        files || rom.files
-      }`,
-      {
-        responseType: "blob",
-      }
-    )
-    .then((response) => {
-      const a = document.createElement("a");
-      a.href = window.URL.createObjectURL(new Blob([response.data]));
-      a.download = `${rom.r_name}.zip`;
-      a.click();
-      downloadStore.remove(rom.file_name);
-    });
+  if (!socket.connected) socket.connect();
+  useDownloadStore().add(rom.id);
 }
 
 export async function updateRomApi(rom, updatedData, renameAsIGDB) {

frontend/src/stores/download.js

@@ -4,11 +4,11 @@ export default defineStore("download", {
   state: () => ({ value: [] }),
 
   actions: {
-    add(filename) {
-      this.value.push(filename);
+    add(id) {
+      this.value.push(id);
     },
-    remove(filename) {
-      this.value.splice(this.value.indexOf(filename), 1);
+    remove(id) {
+      this.value.splice(this.value.indexOf(id), 1);
     },
   },
 });

frontend/src/views/Details.vue

@@ -67,7 +67,7 @@ onBeforeMount(async () => {
             <v-card
               elevation="2"
               :loading="
-                downloadStore.value.includes(rom.file_name)
+                downloadStore.value.includes(rom.id)
                   ? 'rommAccent1'
                   : null
               "
@@ -96,7 +96,7 @@ onBeforeMount(async () => {
             <template v-if="rom.multi">
               <v-btn
                 @click="downloadRomApi(rom, filesToDownload)"
-                :disabled="downloadStore.value.includes(rom.file_name)"
+                :disabled="downloadStore.value.includes(rom.id)"
                 rounded="0"
                 color="primary"
                 block

pyproject.toml

@@ -33,6 +33,7 @@ types-pyyaml = "^6.0.12.11"
 types-requests = "^2.31.0.2"
 mypy = "^1.4.1"
 types-redis = "^4.6.0.3"
+stream-zip = "^0.0.67"
 
 [build-system]
 requires = ["poetry-core"]