[Reputation Oracle] Enhanced webhook processing logic (#877)

* Enhanced webhook processing logic * Removed unused comments * Removed unused comments * Updated method name * Updated methods after local test * Added new unit tests * Updated migrations * Removed comments * Updated filepath and final results logic * Updated copyFileFromURLToBucket method * Updated unit tests * Using Logger instead console.log * Improved error handling in utils uploadFiles method * [Reputation oracle] Added signature verification (#879) * Added signature verification * Added event type * Added guard and util * Updated event paths * Chenged dto * Updated migrations * Removed unnecessary method * Added oracle type parameter * Added oracle type parameter to swagger * Updated webhook verification for reputation oracle * Updated webhook verification for job launcher * Updated signature util unit tests * [Common] Removed `oracleType` verification (#889) * Updated webhook verification for reputation oracle * Updated webhook verification for job launcher * Updated signature util unit tests * Resolved conflicts * Removed unused event type
humanprotocol · Sep 28, 2023 · fc1326d · fc1326d
1 parent 08f46ea
commit fc1326d
Show file tree

Hide file tree

Showing 27 changed files with 941 additions and 849 deletions.
diff --git a/packages/apps/job-launcher/server/src/common/config/env.ts b/packages/apps/job-launcher/server/src/common/config/env.ts
@@ -90,7 +90,7 @@ export const envValidator = Joi.object({
   S3_ACCESS_KEY: Joi.string().required(),
   S3_SECRET_KEY: Joi.string().required(),
   S3_BUCKET: Joi.string().default('launcher'),
-  S3_USE_SSL: Joi.string().default(false),
+  S3_USE_SSL: Joi.string().default('false'),
   // Stripe
   STRIPE_SECRET_KEY: Joi.string().required(),
   STRIPE_API_VERSION: Joi.string().default('2022-11-15'),

diff --git a/packages/apps/job-launcher/server/src/common/guards/signature.auth.spec.ts b/packages/apps/job-launcher/server/src/common/guards/signature.auth.spec.ts
@@ -50,15 +50,13 @@ describe('SignatureAuthGuard', () => {
 
     it('should return true if signature is verified', async () => {
       mockRequest.headers['header-signature-key'] = 'validSignature';
-      jest.spyOn(guard, 'determineAddress').mockReturnValue('someAddress');
       (verifySignature as jest.Mock).mockReturnValue(true);
 
       const result = await guard.canActivate(context as any);
       expect(result).toBeTruthy();
     });
 
     it('should throw unauthorized exception if signature is not verified', async () => {
-      jest.spyOn(guard, 'determineAddress').mockReturnValue('someAddress');
       (verifySignature as jest.Mock).mockReturnValue(false);
 
       await expect(guard.canActivate(context as any)).rejects.toThrow(UnauthorizedException);
@@ -69,35 +67,4 @@ describe('SignatureAuthGuard', () => {
       await expect(guard.canActivate(context as any)).rejects.toThrow(UnauthorizedException);
     });
   });
-
-  describe('determineAddress', () => {
-    it('should return the correct address if originalUrl contains the fortune oracle type', () => {
-      const mockRequest = { originalUrl: '/somepath/fortune/anotherpath' };
-      const expectedAddress = MOCK_ADDRESS;
-      mockConfigService.get = jest.fn().mockReturnValue(expectedAddress);
-
-      const result = guard.determineAddress(mockRequest);
-
-      expect(result).toEqual(expectedAddress);
-    });
-
-    it('should return the correct address if originalUrl contains the cvat oracle type', () => {
-      const mockRequest = { originalUrl: '/somepath/cvat/anotherpath' };
-      const expectedAddress = MOCK_ADDRESS;
-      mockConfigService.get = jest.fn().mockReturnValue(expectedAddress);
-
-      const result = guard.determineAddress(mockRequest);
-
-      expect(result).toEqual(expectedAddress);
-    });
-
-    it('should throw BadRequestException for unrecognized oracle type', () => {
-      const mockRequest = { originalUrl: '/some/random/path' };
-
-      expect(() => {
-        guard.determineAddress(mockRequest);
-      }).toThrow(BadRequestException);
-    });
-
-  });
 });
diff --git a/packages/apps/job-launcher/server/src/common/guards/signature.auth.ts b/packages/apps/job-launcher/server/src/common/guards/signature.auth.ts
@@ -9,7 +9,6 @@ import { verifySignature } from '../utils/signature';
 import { HEADER_SIGNATURE_KEY } from '../constants';
 import { ConfigService } from '@nestjs/config';
 import { ConfigNames } from '../config';
-import { OracleType } from '../enums/webhook';
 
 @Injectable()
 export class SignatureAuthGuard implements CanActivate {
@@ -19,11 +18,18 @@ export class SignatureAuthGuard implements CanActivate {
     const request = context.switchToHttp().getRequest();
 
     const data = request.body;
-    const signature = request.headers[HEADER_SIGNATURE_KEY];
-
+    const signature = request.headers[HEADER_SIGNATURE_KEY]; 
+    const oracleAdresses = [
+      this.configService.get<string>(
+        ConfigNames.FORTUNE_EXCHANGE_ORACLE_ADDRESS,
+      )!,
+      this.configService.get<string>(
+        ConfigNames.CVAT_EXCHANGE_ORACLE_ADDRESS,
+      )!
+    ]
+
     try {
-      const address = this.determineAddress(request);
-      const isVerified = verifySignature(data, signature, address);
+      const isVerified = verifySignature(data, signature, oracleAdresses)
 
       if (isVerified) {
         return true;
@@ -34,24 +40,4 @@ export class SignatureAuthGuard implements CanActivate {
 
     throw new UnauthorizedException('Unauthorized');
   }
-
-  public determineAddress(request: any): string {
-    const originalUrl = request.originalUrl;
-    const parts = originalUrl.split('/');
-    const oracleType = parts[2];
-
-    if (oracleType === OracleType.FORTUNE) {
-      return this.configService.get<string>(
-        ConfigNames.FORTUNE_EXCHANGE_ORACLE_ADDRESS,
-      )!;
-    } else if (oracleType === OracleType.CVAT) {
-      return this.configService.get<string>(
-        ConfigNames.CVAT_EXCHANGE_ORACLE_ADDRESS,
-      )!;
-    } else {
-      throw new BadRequestException(
-        'Unable to determine address from origin URL',
-      );
-    }
-  }
 }
diff --git a/packages/apps/job-launcher/server/src/common/utils/signature.spec.ts b/packages/apps/job-launcher/server/src/common/utils/signature.spec.ts
@@ -2,7 +2,6 @@ import { verifySignature, recoverSigner, signMessage } from './signature';
 import { MOCK_ADDRESS, MOCK_PRIVATE_KEY } from '../../../test/constants';
 import { ErrorSignature } from '../constants/errors';
 
-//ethers.utils.verifyMessage = jest.fn().mockResolvedValue(true);
 jest.doMock('ethers', () => {
   return {
     utils: {
@@ -25,7 +24,7 @@ describe('Signature utility', () => {
       const message = 'Hello, this is a signed message!';
       const signature = await signMessage(message, MOCK_PRIVATE_KEY);
 
-      const result = verifySignature(message, signature, MOCK_ADDRESS);
+      const result = verifySignature(message, signature, [MOCK_ADDRESS]);
 
       expect(result).toBe(true);
     });
@@ -37,7 +36,7 @@ describe('Signature utility', () => {
       const invalidAddress = '0x1234567890123456789012345678901234567892';
 
       expect(() => {
-        verifySignature(message, invalidSignature, invalidAddress);
+        verifySignature(message, invalidSignature, [invalidAddress]);
       }).toThrow(ErrorSignature.SignatureNotVerified);
     });
 
@@ -46,7 +45,7 @@ describe('Signature utility', () => {
       const invalidSignature = '0xInvalidSignature';
 
       expect(() => {
-        verifySignature(message, invalidSignature, MOCK_ADDRESS);
+        verifySignature(message, invalidSignature, [MOCK_ADDRESS]);
       }).toThrow(ErrorSignature.InvalidSignature);
     });
   });

diff --git a/packages/apps/job-launcher/server/src/common/utils/signature.ts b/packages/apps/job-launcher/server/src/common/utils/signature.ts
@@ -5,11 +5,11 @@ import { ErrorSignature } from '../constants/errors';
 export function verifySignature(
   message: object | string,
   signature: string,
-  address: string,
+  addresses: string[],
 ): boolean {
   const signer = recoverSigner(message, signature);
 
-  if (signer.toLowerCase() !== address.toLowerCase()) {
+  if (!addresses.some(address => address.toLowerCase() === signer.toLowerCase())) {
     throw new ConflictException(ErrorSignature.SignatureNotVerified);
   }
 

diff --git a/packages/apps/job-launcher/server/src/modules/job/job.controller.ts b/packages/apps/job-launcher/server/src/modules/job/job.controller.ts
@@ -94,8 +94,8 @@ export class JobController {
 
   @Public()
   @UseGuards(SignatureAuthGuard)
-  @Post('/:oracleType/escrow-failed-webhook')
-  public async(
+  @Post('/escrow-failed-webhook')
+  public async (
     @Headers(HEADER_SIGNATURE_KEY) _: string,
     @Body() data: EscrowFailedWebhookDto,
   ): Promise<any> {

diff --git a/packages/apps/job-launcher/server/src/modules/job/job.dto.ts b/packages/apps/job-launcher/server/src/modules/job/job.dto.ts
@@ -332,15 +332,15 @@ export class EscrowFailedWebhookDto {
     enum: ChainId,
   })
   @IsEnum(ChainId)
-  public chain_id: ChainId;
+  public chainId: ChainId;
 
   @ApiProperty()
   @IsString()
-  public escrow_address: string;
+  public escrowAddress: string;
 
   @ApiProperty()
   @IsEnum(EventType)
-  public event_type: EventType;
+  public eventType: EventType;
 
   @ApiProperty()
   @IsString()

diff --git a/packages/apps/job-launcher/server/src/modules/job/job.service.ts b/packages/apps/job-launcher/server/src/modules/job/job.service.ts
@@ -641,14 +641,14 @@ export class JobService {
   public async escrowFailedWebhook(
     dto: EscrowFailedWebhookDto,
   ): Promise<boolean> {
-    if (dto.event_type !== EventType.TASK_CREATION_FAILED) {
+    if (dto.eventType !== EventType.TASK_CREATION_FAILED) {
       this.logger.log(ErrorJob.InvalidEventType, JobService.name);
       throw new BadRequestException(ErrorJob.InvalidEventType);
     }
 
     const jobEntity = await this.jobRepository.findOne({
-      chainId: dto.chain_id,
-      escrowAddress: dto.escrow_address,
+      chainId: dto.chainId,
+      escrowAddress: dto.escrowAddress,
     });
 
     if (!jobEntity) {

diff --git a/packages/apps/reputation-oracle/server/.env.example b/packages/apps/reputation-oracle/server/.env.example
@@ -30,3 +30,7 @@ WEB3_PRIVATE_KEY=
 # Reputation Level
 REPUTATION_LEVEL_LOW=
 REPUTATION_LEVEL_HIGH=
+
+# Oracles
+FORTUNE_RECORDING_ORACLE_ADDRESS=
+CVAT_RECORDING_ORACLE_ADDRESS=
diff --git a/packages/apps/reputation-oracle/server/src/common/config/env.ts b/packages/apps/reputation-oracle/server/src/common/config/env.ts
@@ -25,6 +25,8 @@ export const ConfigNames = {
   S3_USE_SSL: 'S3_USE_SSL',
   REPUTATION_LEVEL_LOW: 'REPUTATION_LEVEL_LOW',
   REPUTATION_LEVEL_HIGH: 'REPUTATION_LEVEL_HIGH',
+  FORTUNE_RECORDING_ORACLE_ADDRESS: 'FORTUNE_RECORDING_ORACLE_ADDRESS',
+  CVAT_RECORDING_ORACLE_ADDRESS: 'CVAT_RECORDING_ORACLE_ADDRESS'
 };
 
 export const envValidator = Joi.object({
@@ -41,8 +43,8 @@ export const envValidator = Joi.object({
   POSTGRES_PASSWORD: Joi.string().default('qwerty'),
   POSTGRES_DATABASE: Joi.string().default('job-launcher'),
   POSTGRES_PORT: Joi.string().default('5432'),
-  POSTGRES_SYNC: Joi.string().default(false),
-  POSTGRES_SSL: Joi.string().default(false),
+  POSTGRES_SYNC: Joi.string().default('false'),
+  POSTGRES_SSL: Joi.string().default('false'),
   // Web3
   WEB3_PRIVATE_KEY: Joi.string().required(),
   // S3
@@ -51,8 +53,11 @@ export const envValidator = Joi.object({
   S3_ACCESS_KEY: Joi.string().required(),
   S3_SECRET_KEY: Joi.string().required(),
   S3_BUCKET: Joi.string().default('launcher'),
-  S3_USE_SSL: Joi.string().default(false),
+  S3_USE_SSL: Joi.string().default('false'),
   // Reputation Level
   REPUTATION_LEVEL_LOW: Joi.number().default(300),
   REPUTATION_LEVEL_HIGH: Joi.number().default(700),
+  // Oracles
+  FORTUNE_RECORDING_ORACLE_ADDRESS: Joi.string().required(),
+  CVAT_RECORDING_ORACLE_ADDRESS: Joi.string().required(),
 });
diff --git a/packages/apps/reputation-oracle/server/src/common/constants/errors.ts b/packages/apps/reputation-oracle/server/src/common/constants/errors.ts
@@ -4,6 +4,7 @@
 export enum ErrorWebhook {
   NotFound = 'Webhook not found',
   NotCreated = 'Webhook has not been created',
+  InvalidEventType = 'Invalid event type',
 }
 
 /**
@@ -18,6 +19,7 @@ export enum ErrorReputation {
  * Represents error messages related to results.
  */
 export enum ErrorResults {
+  IntermediateResultsURLNotSet = 'Intermediate results URL is not set',
   NoIntermediateResultsFound = 'No intermediate results found',
   NoResultsHaveBeenVerified = 'No results have been verified',
 }
@@ -27,4 +29,13 @@ export enum ErrorResults {
  */
 export enum ErrorManifest {
   ManifestUrlDoesNotExist = 'Manifest url does not exist',
+  UnsupportedManifestType = 'Unsupported manifest type'
+}
+
+/**
+ * Represents error messages related to signature.
+ */
+export enum ErrorSignature {
+  SignatureNotVerified = 'Signature not verified',
+  InvalidSignature = 'Invalid signature',
 }
diff --git a/packages/apps/reputation-oracle/server/src/common/constants/index.ts b/packages/apps/reputation-oracle/server/src/common/constants/index.ts
@@ -1,7 +1,16 @@
+import { JobRequestType } from "../enums";
+
 export const NS = 'hmt';
 export const RETRIES_COUNT_THRESHOLD = 3;
 export const INITIAL_REPUTATION = 0;
 
+export const CVAT_RESULTS_ANNOTATIONS_FILENAME = 'resulting_annotations.zip'
+export const CVAT_VALIDATION_META_FILENAME = 'validation_meta.json'
+
+export const CVAT_JOB_TYPES = [JobRequestType.IMAGE_LABEL_BINARY, JobRequestType.IMAGE_BOXES, JobRequestType.IMAGE_POINTS]
+
+export const HEADER_SIGNATURE_KEY = 'human-signature';
+
 export const CURSE_WORDS = [
   '4r5e',
   '5h1t',

diff --git a/packages/apps/reputation-oracle/server/src/common/enums/job.ts b/packages/apps/reputation-oracle/server/src/common/enums/job.ts
@@ -1,4 +1,6 @@
 export enum JobRequestType {
   IMAGE_LABEL_BINARY = 'IMAGE_LABEL_BINARY',
+  IMAGE_BOXES = 'IMAGE_BOXES',
+  IMAGE_POINTS = 'IMAGE_POINTS',
   FORTUNE = 'FORTUNE',
 }
diff --git a/packages/apps/reputation-oracle/server/src/common/enums/webhook.ts b/packages/apps/reputation-oracle/server/src/common/enums/webhook.ts
@@ -1,6 +1,15 @@
+export enum EventType {
+  TASK_FINISHED = 'task_finished'
+}
+
 export enum WebhookStatus {
   PENDING = 'PENDING',
   COMPLETED = 'COMPLETED',
   FAILED = 'FAILED',
   PAID = 'PAID',
 }
+
+export enum OracleType {
+  FORTUNE = 'fortune',
+  CVAT = 'cvat',
+}
diff --git a/packages/apps/reputation-oracle/server/src/common/guards/index.ts b/packages/apps/reputation-oracle/server/src/common/guards/index.ts
@@ -0,0 +1 @@
+export * from './signature.auth';