[Reputation Oracle] Storage module

packages/apps/reputation-oracle/server/src/common/config/index.ts

@@ -1,2 +1,3 @@
 export * from './env';
 export * from './networks';
+export * from './s3';

packages/apps/reputation-oracle/server/src/common/config/s3.ts

@@ -0,0 +1,13 @@
+import { ConfigType, registerAs } from '@nestjs/config';
+
+export const s3Config = registerAs('s3', () => ({
+  endPoint: process.env.S3_ENDPOINT!,
+  port: +process.env.S3_PORT!,
+  accessKey: process.env.S3_ACCESS_KEY!,
+  secretKey: process.env.S3_SECRET_KEY!,
+  bucket: process.env.S3_BUCKET!,
+  useSSL: process.env.S3_USE_SSL === 'true',
+}));
+
+export const s3ConfigKey = s3Config.KEY;
+export type S3ConfigType = ConfigType<typeof s3Config>;

packages/apps/reputation-oracle/server/src/common/interfaces/s3.ts

@@ -0,0 +1,4 @@
+export class UploadedFile {
+  public url: string;
+  public hash: string;
+}

packages/apps/reputation-oracle/server/src/common/utils/index.ts

@@ -1,68 +1,7 @@
-import {
-  StorageCredentials,
-  StorageParams,
-  UploadFile,
-} from '@human-protocol/sdk';
-import * as Minio from 'minio';
 import * as crypto from 'crypto';
-import { PassThrough, Readable } from 'stream';
-import axios from 'axios';
-import { Logger } from '@nestjs/common';
+import { Readable } from 'stream';
 
-/**
- * **Copy file from a URL to cloud storage**
- *
- * @param {string} url - URL of the source file
- * @param {string} destBucket - Destination bucket name
- * @param {StorageParams} clientParams - Configuration parameters for cloud storage
- * @param {StorageCredentials} [credentials] - Optional. Cloud storage access data. If credentials are not provided, use anonymous access to the bucket
- * @returns {Promise<UploadFile>} - Uploaded file with key/hash
- */
-export async function copyFileFromURLToBucket(
-  url: string,
-  destBucket: string,
-  clientParams: StorageParams,
-  credentials?: StorageCredentials,
-): Promise<UploadFile> {
-  try {
-    const client: Minio.Client = new Minio.Client({
-      ...clientParams,
-      accessKey: credentials?.accessKey ?? '',
-      secretKey: credentials?.secretKey ?? '',
-    });
-
-    const { data } = await axios.get(url, { responseType: 'stream' });
-    const stream = new PassThrough();
-    data.pipe(stream);
-
-    const hash = await hashStream(data);
-    const key = `s3${hash}.zip`;
-
-    await new Promise((resolve, reject) => {
-      client.putObject(destBucket, key, stream, (err, etag) => {
-        if (err) reject(err);
-        else resolve(etag);
-      });
-    });
-
-    Logger.log(`File from ${url} copied to ${destBucket}/${key}`);
-
-    return {
-      key,
-      url: `${clientParams.useSSL ? 'https' : 'http'}://${
-        clientParams.endPoint
-      }${
-        clientParams.port ? `:${clientParams.port}` : ''
-      }/${destBucket}/${key}`,
-      hash,
-    };
-  } catch (error) {
-    Logger.error('Error copying file:', error);
-    throw new Error('File not uploaded');
-  }
-}
-
-function hashStream(stream: Readable): Promise<string> {
+export function hashStream(stream: Readable): Promise<string> {
   return new Promise((resolve, reject) => {
     const hash = crypto.createHash('sha1');
 

packages/apps/reputation-oracle/server/src/modules/storage/storage.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { StorageService } from './storage.service';
+import { ConfigModule } from '@nestjs/config';
+import { s3Config } from '../../common/config';
+
+@Module({
+  imports: [ConfigModule.forFeature(s3Config)],
+  providers: [StorageService],
+  exports: [StorageService],
+})
+export class StorageModule {}

packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts

@@ -0,0 +1,229 @@
+import { ChainId, StorageClient } from '@human-protocol/sdk';
+import { ConfigModule, registerAs } from '@nestjs/config';
+import { Test } from '@nestjs/testing';
+import {
+  MOCK_FILE_URL,
+  MOCK_MANIFEST,
+  MOCK_S3_ACCESS_KEY,
+  MOCK_S3_BUCKET,
+  MOCK_S3_ENDPOINT,
+  MOCK_S3_PORT,
+  MOCK_S3_SECRET_KEY,
+  MOCK_S3_USE_SSL,
+} from '../../../test/constants';
+import { StorageService } from './storage.service';
+import crypto from 'crypto';
+import axios from 'axios';
+import stream from 'stream';
+
+jest.mock('@human-protocol/sdk', () => ({
+  ...jest.requireActual('@human-protocol/sdk'),
+  StorageClient: {
+    downloadFileFromUrl: jest.fn(),
+  },
+}));
+
+jest.mock('minio', () => {
+  class Client {
+    putObject = jest.fn();
+    bucketExists = jest.fn();
+    constructor() {
+      (this as any).protocol = 'http:';
+      (this as any).host = 'localhost';
+      (this as any).port = 9000;
+    }
+  }
+
+  return { Client };
+});
+
+jest.mock('axios');
+
+describe('Web3Service', () => {
+  let storageService: StorageService;
+
+  beforeAll(async () => {
+    const moduleRef = await Test.createTestingModule({
+      imports: [
+        ConfigModule.forFeature(
+          registerAs('s3', () => ({
+            accessKey: MOCK_S3_ACCESS_KEY,
+            secretKey: MOCK_S3_SECRET_KEY,
+            endPoint: MOCK_S3_ENDPOINT,
+            port: MOCK_S3_PORT,
+            useSSL: MOCK_S3_USE_SSL,
+            bucket: MOCK_S3_BUCKET,
+          })),
+        ),
+      ],
+      providers: [StorageService],
+    }).compile();
+
+    storageService = moduleRef.get<StorageService>(StorageService);
+  });
+
+  describe('uploadJobSolutions', () => {
+    it('should upload the solutions correctly', async () => {
+      const workerAddress = '0x1234567890123456789012345678901234567891';
+      const escrowAddress = '0x1234567890123456789012345678901234567890';
+      const chainId = ChainId.LOCALHOST;
+      const solution = 'test';
+
+      storageService.minioClient.bucketExists = jest
+        .fn()
+        .mockResolvedValueOnce(true);
+
+      const jobSolution = {
+        workerAddress,
+        solution,
+      };
+      const fileData = await storageService.uploadJobSolutions(
+        escrowAddress,
+        chainId,
+        [jobSolution],
+      );
+      expect(fileData).toEqual({
+        url: `http://${MOCK_S3_ENDPOINT}:${MOCK_S3_PORT}/${MOCK_S3_BUCKET}/${escrowAddress}-${chainId}.json`,
+        hash: crypto
+          .createHash('sha1')
+          .update(JSON.stringify([jobSolution]))
+          .digest('hex'),
+      });
+      expect(storageService.minioClient.putObject).toHaveBeenCalledWith(
+        MOCK_S3_BUCKET,
+        `${escrowAddress}-${chainId}.json`,
+        expect.stringContaining(solution),
+        {
+          'Content-Type': 'application/json',
+        },
+      );
+    });
+
+    it('should fail if the bucket does not exist', async () => {
+      const workerAddress = '0x1234567890123456789012345678901234567891';
+      const escrowAddress = '0x1234567890123456789012345678901234567890';
+      const chainId = ChainId.LOCALHOST;
+      const solution = 'test';
+
+      storageService.minioClient.bucketExists = jest
+        .fn()
+        .mockResolvedValueOnce(false);
+
+      const jobSolution = {
+        workerAddress,
+        solution,
+      };
+      await expect(
+        storageService.uploadJobSolutions(escrowAddress, chainId, [
+          jobSolution,
+        ]),
+      ).rejects.toThrow('Bucket not found');
+    });
+    it('should fail if the file cannot be uploaded', async () => {
+      const workerAddress = '0x1234567890123456789012345678901234567891';
+      const escrowAddress = '0x1234567890123456789012345678901234567890';
+      const chainId = ChainId.LOCALHOST;
+      const solution = 'test';
+
+      storageService.minioClient.bucketExists = jest
+        .fn()
+        .mockResolvedValueOnce(true);
+      storageService.minioClient.putObject = jest
+        .fn()
+        .mockRejectedValueOnce('Network error');
+
+      const jobSolution = {
+        workerAddress,
+        solution,
+      };
+
+      await expect(
+        storageService.uploadJobSolutions(escrowAddress, chainId, [
+          jobSolution,
+        ]),
+      ).rejects.toThrow('File not uploaded');
+    });
+  });
+
+  describe('download', () => {
+    it('should download the file correctly', async () => {
+      const exchangeAddress = '0x1234567890123456789012345678901234567892';
+      const workerAddress = '0x1234567890123456789012345678901234567891';
+      const solution = 'test';
+
+      const expectedJobFile = {
+        exchangeAddress,
+        solutions: [
+          {
+            workerAddress,
+            solution,
+          },
+        ],
+      };
+
+      StorageClient.downloadFileFromUrl = jest
+        .fn()
+        .mockResolvedValueOnce(expectedJobFile);
+      const solutionsFile = await storageService.download(MOCK_FILE_URL);
+      expect(solutionsFile).toBe(expectedJobFile);
+    });
+
+    it('should return empty array when file cannot be downloaded', async () => {
+      StorageClient.downloadFileFromUrl = jest
+        .fn()
+        .mockRejectedValue('Network error');
+
+      const solutionsFile = await storageService.download(MOCK_FILE_URL);
+      expect(solutionsFile).toStrictEqual([]);
+    });
+  });
+
+  describe('copyFileFromURLToBucket', () => {
+    it('should copy a file from a valid URL to a bucket', async () => {
+      const streamResponseData = new stream.Readable();
+      streamResponseData.push(JSON.stringify(MOCK_MANIFEST));
+      streamResponseData.push(null);
+      (axios.get as any).mockResolvedValueOnce({ data: streamResponseData });
+
+      const uploadedFile = await storageService.copyFileFromURLToBucket(
+        MOCK_FILE_URL,
+      );
+
+      expect(
+        uploadedFile.url.includes(
+          `http://${MOCK_S3_ENDPOINT}:${MOCK_S3_PORT}/${MOCK_S3_BUCKET}/`,
+        ),
+      ).toBeTruthy();
+      expect(uploadedFile.hash).toBeDefined();
+      expect(storageService.minioClient.putObject).toBeCalledWith(
+        MOCK_S3_BUCKET,
+        expect.any(String),
+        expect.any(stream),
+      );
+    });
+
+    it('should handle an invalid URL', async () => {
+      (axios.get as any).mockRejectedValue('Network error');
+
+      await expect(
+        storageService.copyFileFromURLToBucket(MOCK_FILE_URL),
+      ).rejects.toThrow('File not uploaded');
+    });
+
+    it('should handle errors when copying the file', async () => {
+      const streamResponseData = new stream.Readable();
+      streamResponseData.push(JSON.stringify(MOCK_MANIFEST));
+      streamResponseData.push(null);
+      (axios.get as any).mockResolvedValueOnce({ data: streamResponseData });
+      storageService.minioClient.putObject = jest
+        .fn()
+        .mockRejectedValue('Network error');
+
+      await expect(
+        storageService.copyFileFromURLToBucket(
+          'https://example.com/archivo.zip',
+        ),
+      ).rejects.toThrow('File not uploaded');
+    });
+  });
+});