From b35ed6d48ca2b1d01152ddc435532a1bb6d70cdb Mon Sep 17 00:00:00 2001 From: Eugene Kozlov <68875428+kozlove-aws@users.noreply.github.com> Date: Thu, 8 Jul 2021 16:01:40 -0500 Subject: [PATCH] feat(deadline): create Deadline Groups and Pools on deploy for ConfigureSpotEventPlugin (#470) --- .../python/README.md | 2 - .../ts/README.md | 2 - packages/aws-rfdk/lib/deadline/README.md | 6 - .../lib/configure-spot-event-plugin.ts | 6 +- .../deadline/lib/spot-event-plugin-fleet.ts | 16 +- .../configure-spot-event-plugin/handler.ts | 8 + .../test/handler.test.ts | 121 ++++++++--- .../configure-spot-event-plugin/types.ts | 10 + .../spot-event-plugin-client.ts | 101 +++++++++ .../test/spot-event-plugin-client.test.ts | 203 +++++++++++++++++- 10 files changed, 422 insertions(+), 53 deletions(-) diff --git a/examples/deadline/All-In-AWS-Infrastructure-SEP/python/README.md b/examples/deadline/All-In-AWS-Infrastructure-SEP/python/README.md index 2b0253cfb..f3c1220e2 100644 --- a/examples/deadline/All-In-AWS-Infrastructure-SEP/python/README.md +++ b/examples/deadline/All-In-AWS-Infrastructure-SEP/python/README.md @@ -73,8 +73,6 @@ These instructions assume that your working directory is `examples/deadline/All- 7. You can now [connect to the farm](https://docs.aws.amazon.com/rfdk/latest/guide/connecting-to-render-farm.html) and [submit rendering jobs](https://docs.aws.amazon.com/rfdk/latest/guide/first-rfdk-app.html#_optional_submit_a_job_to_the_render_farm). **Note:** In order for the Spot Event Plugin to create a Spot Fleet Request you need to: - * Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). - * Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). * Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs). **Note:** Disable 'Allow Workers to Perform House Cleaning If Pulse is not Running' in the 'Configure Repository Options' when using Spot Event Plugin. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#prerequisites). diff --git a/examples/deadline/All-In-AWS-Infrastructure-SEP/ts/README.md b/examples/deadline/All-In-AWS-Infrastructure-SEP/ts/README.md index e58b4b246..79bc805dc 100644 --- a/examples/deadline/All-In-AWS-Infrastructure-SEP/ts/README.md +++ b/examples/deadline/All-In-AWS-Infrastructure-SEP/ts/README.md @@ -66,8 +66,6 @@ These instructions assume that your working directory is `examples/deadline/All- 7. You can now [connect to the farm](https://docs.aws.amazon.com/rfdk/latest/guide/connecting-to-render-farm.html) and [submit rendering jobs](https://docs.aws.amazon.com/rfdk/latest/guide/first-rfdk-app.html#_optional_submit_a_job_to_the_render_farm). **Note:** In order for the Spot Event Plugin to create a Spot Fleet Request you need to: - * Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). - * Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). * Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs). **Note:** Disable 'Allow Workers to Perform House Cleaning If Pulse is not Running' in the 'Configure Repository Options' when using Spot Event Plugin. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#prerequisites). diff --git a/packages/aws-rfdk/lib/deadline/README.md b/packages/aws-rfdk/lib/deadline/README.md index 884dce11d..a50780bf7 100644 --- a/packages/aws-rfdk/lib/deadline/README.md +++ b/packages/aws-rfdk/lib/deadline/README.md @@ -45,8 +45,6 @@ The `ConfigureSpotEventPlugin` construct has two main responsibilities: --- **Note:** This construct will configure the Spot Event Plugin, but the Spot Fleet Requests will not be created unless you: -- Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). -- Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). - Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs). --- @@ -232,8 +230,6 @@ This construct represents a Spot Fleet launched by the [Deadline's Spot Event Pl This construct is expected to be used as an input to the [ConfigureSpotEventPlugin](#configure-spot-event-plugin) construct. `ConfigureSpotEventPlugin` construct will generate a Spot Fleet Request Configuration from each provided `SpotEventPluginFleet` and will set these configurations to the Spot Event Plugin. -_**Note:** You will have to create the groups manually using Deadline before submitting jobs. See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html - ```ts const vpc = new Vpc(/* ... */); const renderQueue = new RenderQueue(stack, 'RenderQueue', /* ... */); @@ -277,8 +273,6 @@ const fleet = new SpotEventPluginFleet(this, 'SpotEventPluginFleet', { You can add the Workers to Deadline's Pools providing a list of pools as following: -_**Note:** You will have to create the pools manually using Deadline before submitting jobs. See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html - ```ts const fleet = new SpotEventPluginFleet(this, 'SpotEventPluginFleet', { vpc, diff --git a/packages/aws-rfdk/lib/deadline/lib/configure-spot-event-plugin.ts b/packages/aws-rfdk/lib/deadline/lib/configure-spot-event-plugin.ts index 0abc9a39b..44516b8e9 100644 --- a/packages/aws-rfdk/lib/deadline/lib/configure-spot-event-plugin.ts +++ b/packages/aws-rfdk/lib/deadline/lib/configure-spot-event-plugin.ts @@ -317,8 +317,6 @@ export interface ConfigureSpotEventPluginProps { * Logs for all AWS Lambdas are automatically recorded in Amazon CloudWatch. * * This construct will configure the Spot Event Plugin, but the Spot Fleet Requests will not be created unless you: - * - Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). - * - Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html). * - Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs). * * Important: Disable 'Allow Workers to Perform House Cleaning If Pulse is not Running' in the 'Configure Repository Options' @@ -459,6 +457,8 @@ export class ConfigureSpotEventPlugin extends Construct { }; const spotFleetRequestConfigs = this.mergeSpotFleetRequestConfigs(props.spotFleets); + const deadlineGroups = Array.from(new Set(props.spotFleets?.map(fleet => fleet.deadlineGroups).reduce((p, c) => p.concat(c), []))); + const deadlinePools = Array.from(new Set(props.spotFleets?.map(fleet => fleet.deadlinePools).reduce((p, c) => p?.concat(c ?? []), []))); const properties: SEPConfiguratorResourceProps = { connection: { hostname: props.renderQueue.endpoint.hostname, @@ -468,6 +468,8 @@ export class ConfigureSpotEventPlugin extends Construct { }, spotFleetRequestConfigurations: spotFleetRequestConfigs, spotPluginConfigurations: pluginConfig, + deadlineGroups, + deadlinePools, }; const resource = new CustomResource(this, 'Default', { diff --git a/packages/aws-rfdk/lib/deadline/lib/spot-event-plugin-fleet.ts b/packages/aws-rfdk/lib/deadline/lib/spot-event-plugin-fleet.ts index 2b5732010..8393f009b 100644 --- a/packages/aws-rfdk/lib/deadline/lib/spot-event-plugin-fleet.ts +++ b/packages/aws-rfdk/lib/deadline/lib/spot-event-plugin-fleet.ts @@ -90,9 +90,6 @@ export interface SpotEventPluginFleetProps { /** * Deadline groups these workers need to be assigned to. * - * Note that you will have to create the groups manually using Deadline before submitting jobs. - * See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html - * * Also, note that the Spot Fleet configuration does not allow using wildcards as part of the Group name * as described here https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#wildcards */ @@ -101,9 +98,6 @@ export interface SpotEventPluginFleetProps { /** * Deadline pools these workers need to be assigned to. * - * Note that you will have to create the pools manually using Deadline before submitting jobs. - * See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html - * * @default - Workers are not assigned to any pool. */ readonly deadlinePools?: string[]; @@ -387,6 +381,13 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF */ public readonly deadlineGroups: string[]; + /** + * Deadline pools the workers need to be assigned to. + * + * @default - Workers are not assigned to any pool + */ + public readonly deadlinePools?: string[]; + /** * Name of SSH keypair to grant access to instances. * @@ -413,6 +414,7 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF super(scope, id); this.deadlineGroups = props.deadlineGroups.map(group => group.toLocaleLowerCase()); + this.deadlinePools = props.deadlinePools?.map(pool => pool.toLocaleLowerCase()); this.validateProps(props); this.securityGroups = props.securityGroups ?? [ new SecurityGroup(this, 'SpotFleetSecurityGroup', { vpc: props.vpc }) ]; @@ -463,7 +465,7 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF renderQueue: props.renderQueue, workerSettings: { groups: this.deadlineGroups, - pools: props.deadlinePools?.map(pool => pool.toLocaleLowerCase()), + pools: this.deadlinePools, region: props.deadlineRegion, }, userDataProvider: props.userDataProvider, diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/handler.ts b/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/handler.ts index f504af242..792cf990d 100644 --- a/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/handler.ts +++ b/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/handler.ts @@ -47,6 +47,14 @@ export class SEPConfiguratorResource extends SimpleCustomResource { public async doCreate(_physicalId: string, resourceProperties: SEPConfiguratorResourceProps): Promise { const spotEventPluginClient = await this.spotEventPluginClient(resourceProperties.connection); + if (!await spotEventPluginClient.addGroups(resourceProperties.deadlineGroups)) { + throw new Error(`Failed to add Deadline group(s) ${resourceProperties.deadlineGroups}`); + } + + if (!await spotEventPluginClient.addPools(resourceProperties.deadlinePools)) { + throw new Error(`Failed to add Deadline pool(s) ${resourceProperties.deadlinePools}`); + } + if (resourceProperties.spotFleetRequestConfigurations) { const convertedSpotFleetRequestConfigs = convertSpotFleetRequestConfiguration(resourceProperties.spotFleetRequestConfigurations); const stringConfigs = JSON.stringify(convertedSpotFleetRequestConfigs); diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/test/handler.test.ts b/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/test/handler.test.ts index 7ed0de251..65adeb63b 100644 --- a/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/test/handler.test.ts +++ b/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/test/handler.test.ts @@ -205,24 +205,44 @@ describe('SEPConfiguratorResource', () => { connection: validConnection, }; + const deadlineGroups = ['group_name']; + const deadlinePools = ['pool_name']; + const allConfigs: SEPConfiguratorResourceProps = { spotPluginConfigurations: validSpotEventPluginConfig, connection: validConnection, spotFleetRequestConfigurations: validSpotFleetRequestConfig, + deadlineGroups, + deadlinePools, }; const noConfigs: SEPConfiguratorResourceProps = { connection: validConnection, }; + async function returnTrue(_v1: any): Promise { + return true; + } + + async function returnFalse(_v1: any): Promise { + return false; + } + describe('doCreate', () => { let handler: SEPConfiguratorResource; - let mockSpotEventPluginClient: { saveServerData: jest.Mock; configureSpotEventPlugin: jest.Mock; }; + let mockSpotEventPluginClient: { + saveServerData: jest.Mock; + configureSpotEventPlugin: jest.Mock; + addGroups: jest.Mock; + addPools: jest.Mock; + }; beforeEach(() => { mockSpotEventPluginClient = { - saveServerData: jest.fn(), - configureSpotEventPlugin: jest.fn(), + saveServerData: jest.fn( (a) => returnTrue(a) ), + configureSpotEventPlugin: jest.fn( (a) => returnTrue(a) ), + addGroups: jest.fn( (a) => returnTrue(a) ), + addPools: jest.fn( (a) => returnTrue(a) ), }; handler = new SEPConfiguratorResource(new AWS.SecretsManager()); @@ -242,9 +262,6 @@ describe('SEPConfiguratorResource', () => { test('with no configs', async () => { // GIVEN - async function returnTrue(_v1: any): Promise { - return true; - } const mockSaveServerData = jest.fn( (a) => returnTrue(a) ); mockSpotEventPluginClient.saveServerData = mockSaveServerData; const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) ); @@ -261,9 +278,6 @@ describe('SEPConfiguratorResource', () => { test('save spot fleet request configs', async () => { // GIVEN - async function returnTrue(_v1: any): Promise { - return true; - } const mockSaveServerData = jest.fn( (a) => returnTrue(a) ); mockSpotEventPluginClient.saveServerData = mockSaveServerData; @@ -281,9 +295,6 @@ describe('SEPConfiguratorResource', () => { test('save spot fleet request configs without BlockDeviceMappings', async () => { // GIVEN - async function returnTrue(_v1: any): Promise { - return true; - } const mockSaveServerData = jest.fn( (a) => returnTrue(a) ); mockSpotEventPluginClient.saveServerData = mockSaveServerData; @@ -326,9 +337,6 @@ describe('SEPConfiguratorResource', () => { test('save spot fleet request configs without Ebs', async () => { // GIVEN - async function returnTrue(_v1: any): Promise { - return true; - } const mockSaveServerData = jest.fn( (a) => returnTrue(a) ); mockSpotEventPluginClient.saveServerData = mockSaveServerData; @@ -375,9 +383,6 @@ describe('SEPConfiguratorResource', () => { test('save spot event plugin configs', async () => { // GIVEN - async function returnTrue(_v1: any): Promise { - return true; - } const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) ); mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin; @@ -407,14 +412,22 @@ describe('SEPConfiguratorResource', () => { expect(mockConfigureSpotEventPlugin.mock.calls[0][0]).toEqual([...configs, ...securitySettings]); }); - test('save both configs', async () => { + test('save server data', async () => { // GIVEN - async function returnTrue(_v1: any): Promise { - return true; - } const mockSaveServerData = jest.fn( (a) => returnTrue(a) ); mockSpotEventPluginClient.saveServerData = mockSaveServerData; + // WHEN + const result = await handler.doCreate('physicalId', allConfigs); + + // THEN + expect(result).toBeUndefined(); + expect(mockSaveServerData.mock.calls.length).toBe(1); + expect(mockSaveServerData.mock.calls[0][0]).toEqual(JSON.stringify(validConvertedSpotFleetRequestConfig)); + }); + + test('configure spot event plugin', async () => { + // GIVEN const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) ); mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin; @@ -436,22 +449,67 @@ describe('SEPConfiguratorResource', () => { }]; // WHEN - const result = await handler.doCreate('physicalId', allConfigs); + await handler.doCreate('physicalId', allConfigs); // THEN - expect(result).toBeUndefined(); - expect(mockSaveServerData.mock.calls.length).toBe(1); - expect(mockSaveServerData.mock.calls[0][0]).toEqual(JSON.stringify(validConvertedSpotFleetRequestConfig)); - expect(mockConfigureSpotEventPlugin.mock.calls.length).toBe(1); expect(mockConfigureSpotEventPlugin.mock.calls[0][0]).toEqual([...configs, ...securitySettings]); }); + test('create groups', async () => { + // GIVEN + const mockAddGroups = jest.fn( (a) => returnTrue(a) ); + mockSpotEventPluginClient.addGroups = mockAddGroups; + + // WHEN + await handler.doCreate('physicalId', allConfigs); + + // THEN + expect(mockAddGroups.mock.calls.length).toBe(1); + expect(mockAddGroups).toHaveBeenCalledWith(deadlineGroups); + }); + + test('create pools', async () => { + // GIVEN + const mockAddPools = jest.fn( (a) => returnTrue(a) ); + mockSpotEventPluginClient.addPools = mockAddPools; + + // WHEN + await handler.doCreate('physicalId', allConfigs); + + // THEN + expect(mockAddPools.mock.calls.length).toBe(1); + expect(mockAddPools).toHaveBeenCalledWith(deadlinePools); + }); + + test('throw when cannot add groups', async () => { + // GIVEN + mockSpotEventPluginClient.addGroups = jest.fn( (a) => returnFalse(a) ); + + // WHEN + const promise = handler.doCreate('physicalId', allConfigs); + + // THEN + await expect(promise) + .rejects + .toThrowError(`Failed to add Deadline group(s) ${allConfigs.deadlineGroups}`); + }); + + test('throw when cannot add pools', async () => { + // GIVEN + mockSpotEventPluginClient.addPools = jest.fn( (a) => returnFalse(a) ); + + // WHEN + const promise = handler.doCreate('physicalId', allConfigs); + + // THEN + await expect(promise) + .rejects + .toThrowError(`Failed to add Deadline pool(s) ${allConfigs.deadlinePools}`); + }); + test('throw when cannot save spot fleet request configs', async () => { // GIVEN - async function returnFalse(_v1: any): Promise { - return false; - } const mockSaveServerData = jest.fn( (a) => returnFalse(a) ); mockSpotEventPluginClient.saveServerData = mockSaveServerData; @@ -466,9 +524,6 @@ describe('SEPConfiguratorResource', () => { test('throw when cannot save spot event plugin configs', async () => { // GIVEN - async function returnFalse(_v1: any): Promise { - return false; - } const mockConfigureSpotEventPlugin = jest.fn( (a) => returnFalse(a) ); mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin; diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/types.ts b/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/types.ts index 8887f0287..c08f1d822 100644 --- a/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/types.ts +++ b/packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/types.ts @@ -23,6 +23,16 @@ export interface SEPConfiguratorResourceProps { * See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#event-plugin-configuration-options */ readonly spotPluginConfigurations?: PluginSettings; + + /** + * Deadline groups that are used by these fleets. + */ + readonly deadlineGroups?: string[]; + + /** + * Deadline pools that are used by these fleets. + */ + readonly deadlinePools?: string[]; } /** diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/spot-event-plugin-client.ts b/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/spot-event-plugin-client.ts index 65e18f8d3..50f8a8b41 100644 --- a/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/spot-event-plugin-client.ts +++ b/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/spot-event-plugin-client.ts @@ -25,6 +25,30 @@ interface DescribeServerDataResponse { readonly ServerData: DescribedServerData[]; } +/** + * A response from get pool/group request + */ +export interface PoolGroupCollections { + /** + * The collection of user-created Pools/Groups that are currently active + */ + readonly Pools: string []; + + /** + * The collection of Pools/Groups that are currently obsolete + */ + readonly ObsoletePools: string []; +} + +/** + * A type of collection to get/recive from Deadline. + */ +export enum CollectionType { + Pool = 'pool', + + Group = 'group', +} + /** * Provides a simple interface to send requests to the Render Queue API related to the Deadline Spot Event Plugin. */ @@ -94,6 +118,83 @@ export class SpotEventPluginClient { } } + public async addGroups(newGroups?: string[]): Promise { + if (newGroups && newGroups.length) { + const deadlineGroups = await this.getCollection(CollectionType.Group); + if (deadlineGroups) { + const newDeadlineGroups = deadlineGroups.Pools + .concat(newGroups + .filter(group => !deadlineGroups.Pools.includes(group))); + return await this.saveCollection({ + Pools: newDeadlineGroups, + ObsoletePools: deadlineGroups.ObsoletePools, + } as PoolGroupCollections, + CollectionType.Group); + } + return false; + } + return true; + } + + public async addPools(newPools?: string[]): Promise { + if (newPools && newPools.length) { + const deadlinePools = await this.getCollection(CollectionType.Pool); + if (deadlinePools) { + const newDeadlinePools = deadlinePools.Pools + .concat(newPools + .filter(pool => !deadlinePools.Pools.includes(pool))); + return await this.saveCollection({ + Pools: newDeadlinePools, + ObsoletePools: deadlinePools.ObsoletePools, + } as PoolGroupCollections, + CollectionType.Pool); + } + return false; + } + return true; + } + + private async getCollection(type: CollectionType): Promise { + console.log(`Getting ${type} collection:`); + try { + const response = await this.deadlineClient.GetRequest(`/db/settings/collections/${type}s?invalidateCache=true`, { + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }); + const deadlinePools: PoolGroupCollections = response.data; + if (!deadlinePools.Pools || !Array.isArray(deadlinePools.Pools)) { + console.error(`Failed to receive a ${type} collection. Invalid response: ${JSON.stringify(response.data)}.`); + return undefined; + } + return deadlinePools; + } catch(e) { + console.error(`Failed to get ${type} collection. Reason: ${(e).message}`); + return undefined; + } + } + + private async saveCollection(pools: PoolGroupCollections, type: CollectionType): Promise { + console.log(`Saving ${type} collection:`); + console.log(pools); + + try { + await this.deadlineClient.PostRequest(`/db/settings/collections/${type}s/save`, { + Pools: pools.Pools, + ObsoletePools: pools.ObsoletePools, + }, + { + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }); + return true; + } catch(e) { + console.error(`Failed to save ${type} collection. Reason: ${(e).message}`); + return false; + } + } + private async describeServerData(): Promise { return await this.deadlineClient.PostRequest('/rcs/v1/describeServerData', { ServerDataIds: [ diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/test/spot-event-plugin-client.test.ts b/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/test/spot-event-plugin-client.test.ts index 99cf5f17f..111b8f851 100644 --- a/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/test/spot-event-plugin-client.test.ts +++ b/packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/test/spot-event-plugin-client.test.ts @@ -6,9 +6,27 @@ import { IncomingMessage } from 'http'; import { Socket } from 'net'; import { DeadlineClient, Response } from '../../deadline-client'; -import { SpotEventPluginClient } from '../spot-event-plugin-client'; +import { SpotEventPluginClient, CollectionType } from '../spot-event-plugin-client'; describe('SpotEventPluginClient', () => { + const poolsColection = { + Pools: ['pool_name'], + ObsoletePools: [], + }; + const groupsColection = { + Pools: ['group_name'], + ObsoletePools: [], + }; + const successfulPoolResponse: Response = { + data: { ...poolsColection }, + fullResponse: new IncomingMessage(new Socket()), + }; + + const successfulGroupResponse: Response = { + data: { ...groupsColection }, + fullResponse: new IncomingMessage(new Socket()), + }; + let spotEventPluginClient: SpotEventPluginClient; let describeDataResponse: Response; let successfulResponse: Response; @@ -238,4 +256,187 @@ describe('SpotEventPluginClient', () => { // THEN await expect(promise).rejects.toEqual(statusMessage); }); + + test.each([ + [CollectionType.Group, successfulGroupResponse, groupsColection], + [CollectionType.Pool, successfulPoolResponse, poolsColection], + ])('Successful getCollection for %s', async (type: CollectionType, response: Response, expectedResult: any) => { + // GIVEN + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].GetRequest = jest.fn().mockResolvedValue(response); + + // WHEN + // eslint-disable-next-line dot-notation + const result = await spotEventPluginClient['getCollection'](type); + + // THEN + expect(result).toEqual(expectedResult); + // eslint-disable-next-line dot-notation + expect(spotEventPluginClient['deadlineClient'].GetRequest).toBeCalledTimes(1); + expect(consoleLogMock).toBeCalledTimes(1); + expect(consoleLogMock).toBeCalledWith(expect.stringMatching(`Getting ${type} collection:`)); + }); + + test('failed getCollection', async () => { + // GIVEN + const statusMessage = 'error message'; + + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].GetRequest = jest.fn().mockRejectedValue(new Error(statusMessage)); + // eslint-disable-next-line dot-notation + const result = await spotEventPluginClient['getCollection'](CollectionType.Group); + + // THEN + expect(result).toBeUndefined(); + expect(consoleErrorMock).toBeCalledTimes(1); + expect(consoleErrorMock).toBeCalledWith(expect.stringMatching(`Failed to get group collection. Reason: ${statusMessage}`)); + }); + + test('failed getCollection with invalid response', async () => { + // GIVEN + const invalidGroupResponse = { + data: { + Pools: {}, + }, + }; + + // WHEN + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].GetRequest = jest.fn().mockResolvedValue(invalidGroupResponse); + // eslint-disable-next-line dot-notation + const result = await spotEventPluginClient['getCollection'](CollectionType.Group); + + // THEN + expect(result).toBeUndefined(); + expect(consoleErrorMock).toBeCalledTimes(1); + expect(consoleErrorMock).toBeCalledWith(expect.stringMatching(`Failed to receive a group collection. Invalid response: ${JSON.stringify(invalidGroupResponse.data)}.`)); + }); + + test.each([ + [CollectionType.Group, groupsColection], + [CollectionType.Pool, poolsColection], + ])('successful saveCollection for %s', async (type: CollectionType, expectedResult: any) => { + // GIVEN + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].PostRequest = jest.fn().mockResolvedValue({}); + + // WHEN + // eslint-disable-next-line dot-notation + const result = await spotEventPluginClient['saveCollection'](expectedResult, type); + + // THEN + expect(result).toBeTruthy(); + // eslint-disable-next-line dot-notation + expect(spotEventPluginClient['deadlineClient'].PostRequest).toBeCalledTimes(1); + expect(consoleLogMock).toBeCalledTimes(2); + expect(consoleLogMock).toBeCalledWith(expect.stringMatching(`Saving ${type} collection:`)); + expect(consoleLogMock).toBeCalledWith(expectedResult); + }); + + test('failed saveCollection', async () => { + // GIVEN + const statusMessage = 'error message'; + + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].PostRequest = jest.fn().mockRejectedValue(new Error(statusMessage)); + + // WHEN + // eslint-disable-next-line dot-notation + const result = await spotEventPluginClient['saveCollection'](groupsColection, CollectionType.Group); + + // THEN + expect(result).toBeFalsy(); + expect(consoleErrorMock).toBeCalledTimes(1); + expect(consoleErrorMock).toBeCalledWith(expect.stringMatching(`Failed to save group collection. Reason: ${statusMessage}`)); + }); + + test.each([ + [ [], ['gr1', 'gr2'] ], + [['gr1', 'gr2'],['gr1', 'gr2']], + [['gr1', 'gr2'],['gr1', 'gr3']], + [['gr1', 'gr2'],[]], + ])('successful call addGroup with existing groups %s and added groups %s', async (currentGroupsCollection: string[], addedGroupsCollection: string[]) => { + // GIVEN + const obsoletePools = ['obsolete_pool']; + const groupResponse: Response = { + data: { + Pools: currentGroupsCollection, + ObsoletePools: obsoletePools, + }, + fullResponse: new IncomingMessage(new Socket()), + }; + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].GetRequest = jest.fn().mockResolvedValue(groupResponse); + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].PostRequest = jest.fn().mockReturnValue(true); + + // WHEN + await spotEventPluginClient.addGroups(addedGroupsCollection); + + // THEN + const requestsCount = addedGroupsCollection.length > 0 ? 1 : 0; + // eslint-disable-next-line dot-notation + expect(spotEventPluginClient['deadlineClient'].GetRequest).toBeCalledTimes(requestsCount); + + // eslint-disable-next-line dot-notation + expect(spotEventPluginClient['deadlineClient'].PostRequest).toBeCalledTimes(requestsCount); + if (requestsCount>0) { + // eslint-disable-next-line dot-notation, jest/no-conditional-expect + expect(spotEventPluginClient['deadlineClient'].PostRequest).toBeCalledWith( + '/db/settings/collections/groups/save', + { + Pools: Array.from(new Set(currentGroupsCollection.concat(addedGroupsCollection))), + ObsoletePools: obsoletePools, + }, + { + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + }, + ); + } + }); + + test.each([ + [ [], ['pool1', 'pool2'] ], + [['pool1', 'pool2'],['pool1', 'pool2']], + [['pool1', 'pool2'],['pool1', 'pool3']], + [['pool1', 'pool2'],[]], + ])('successful call addPool with existing pools %s and added pools %s', async (currentPoolsCollection: string[], addedPoolsCollection: string[]) => { + // GIVEN + const obsoletePools = ['obsolete_pool']; + const poolResponse: Response = { + data: { + Pools: currentPoolsCollection, + ObsoletePools: obsoletePools, + }, + fullResponse: new IncomingMessage(new Socket()), + }; + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].GetRequest = jest.fn().mockResolvedValue(poolResponse); + // eslint-disable-next-line dot-notation + spotEventPluginClient['deadlineClient'].PostRequest = jest.fn().mockReturnValue(true); + + // WHEN + await spotEventPluginClient.addPools(addedPoolsCollection); + + // THEN + const requestsCount = addedPoolsCollection.length > 0 ? 1 : 0; + // eslint-disable-next-line dot-notation + expect(spotEventPluginClient['deadlineClient'].GetRequest).toBeCalledTimes(requestsCount); + + // eslint-disable-next-line dot-notation + expect(spotEventPluginClient['deadlineClient'].PostRequest).toBeCalledTimes(requestsCount); + if (requestsCount>0) { + // eslint-disable-next-line dot-notation, jest/no-conditional-expect + expect(spotEventPluginClient['deadlineClient'].PostRequest).toBeCalledWith( + '/db/settings/collections/pools/save', + { + Pools: Array.from(new Set(currentPoolsCollection.concat(addedPoolsCollection))), + ObsoletePools: obsoletePools, + }, + { + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + }, + ); + } + }); });