[Reputation Oracle] Separate streams for hashing and uploading a file (…

…#1388)

* Using two separate streams for calculating file hash and uploading to s3 bucket

* `copyFileFromURLToBucket` testcase updated to match `copyFileFromURLToBucket` method

packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts

@@ -222,10 +222,15 @@ describe('StorageService', () => {
 
   describe('copyFileFromURLToBucket', () => {
     it('should copy a file from a valid URL to a bucket', async () => {
-      const streamResponseData = new stream.Readable();
-      streamResponseData.push(JSON.stringify(MOCK_MANIFEST));
-      streamResponseData.push(null);
-      (axios.get as any).mockResolvedValueOnce({ data: streamResponseData });
+      const hashStreamData = new stream.Readable();
+      hashStreamData.push(JSON.stringify(MOCK_MANIFEST));
+      hashStreamData.push(null);
+      const uploadStreamData = new stream.Readable();
+      uploadStreamData.push(JSON.stringify(MOCK_MANIFEST));
+      uploadStreamData.push(null);
+      (axios.get as any)
+        .mockResolvedValueOnce({ data: hashStreamData })
+        .mockResolvedValueOnce({ data: uploadStreamData });
 
       const uploadedFile =
         await storageService.copyFileFromURLToBucket(MOCK_FILE_URL);

packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts

@@ -95,14 +95,18 @@ export class StorageService {
    */
   public async copyFileFromURLToBucket(url: string): Promise<UploadedFile> {
     try {
-      const { data } = await axios.get(url, { responseType: 'stream' });
-      const stream = new PassThrough();
-      data.pipe(stream);
+      const { data: hStream } = await axios.get(url, {
+        responseType: 'stream',
+      });
+      const hash = await hashStream(hStream);
 
-      const hash = await hashStream(stream);
       const key = `s3${hash}.zip`;
 
-      await this.minioClient.putObject(this.s3Config.bucket, key, stream, {
+      // Creating a second readable stream for uploading a file to the bucket
+      const { data: uStream } = await axios.get(url, {
+        responseType: 'stream',
+      });
+      await this.minioClient.putObject(this.s3Config.bucket, key, uStream, {
         'Cache-Control': 'no-store',
       });