feat(deadline): create Deadline Groups and Pools on deploy for Config…

…ureSpotEventPlugin (#470)

examples/deadline/All-In-AWS-Infrastructure-SEP/python/README.md

@@ -73,8 +73,6 @@ These instructions assume that your working directory is `examples/deadline/All-
 7. You can now [connect to the farm](https://docs.aws.amazon.com/rfdk/latest/guide/connecting-to-render-farm.html) and [submit rendering jobs](https://docs.aws.amazon.com/rfdk/latest/guide/first-rfdk-app.html#_optional_submit_a_job_to_the_render_farm).
 
     **Note:** In order for the Spot Event Plugin to create a Spot Fleet Request you need to:
-    * Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
-    * Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
     * Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs).
 
     **Note:** Disable 'Allow Workers to Perform House Cleaning If Pulse is not Running' in the 'Configure Repository Options' when using Spot Event Plugin. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#prerequisites).

examples/deadline/All-In-AWS-Infrastructure-SEP/ts/README.md

@@ -66,8 +66,6 @@ These instructions assume that your working directory is `examples/deadline/All-
 7. You can now [connect to the farm](https://docs.aws.amazon.com/rfdk/latest/guide/connecting-to-render-farm.html) and [submit rendering jobs](https://docs.aws.amazon.com/rfdk/latest/guide/first-rfdk-app.html#_optional_submit_a_job_to_the_render_farm).
 
     **Note:** In order for the Spot Event Plugin to create a Spot Fleet Request you need to:
-    * Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
-    * Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
     * Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs).
 
     **Note:** Disable 'Allow Workers to Perform House Cleaning If Pulse is not Running' in the 'Configure Repository Options' when using Spot Event Plugin. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#prerequisites).

packages/aws-rfdk/lib/deadline/README.md

@@ -45,8 +45,6 @@ The `ConfigureSpotEventPlugin` construct has two main responsibilities:
 ---
 
 **Note:** This construct will configure the Spot Event Plugin, but the Spot Fleet Requests will not be created unless you:
-- Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
-- Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
 - Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs).
 
 ---
@@ -232,8 +230,6 @@ This construct represents a Spot Fleet launched by the [Deadline's Spot Event Pl
 
 This construct is expected to be used as an input to the [ConfigureSpotEventPlugin](#configure-spot-event-plugin) construct. `ConfigureSpotEventPlugin` construct will generate a Spot Fleet Request Configuration from each provided `SpotEventPluginFleet` and will set these configurations to the Spot Event Plugin.
 
-_**Note:** You will have to create the groups manually using Deadline before submitting jobs. See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html
-
 ```ts
 const vpc = new Vpc(/* ... */);
 const renderQueue = new RenderQueue(stack, 'RenderQueue', /* ... */);
@@ -277,8 +273,6 @@ const fleet = new SpotEventPluginFleet(this, 'SpotEventPluginFleet', {
 
 You can add the Workers to Deadline's Pools providing a list of pools as following:
 
-_**Note:** You will have to create the pools manually using Deadline before submitting jobs. See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html
-
 ```ts
 const fleet = new SpotEventPluginFleet(this, 'SpotEventPluginFleet', {
   vpc,

packages/aws-rfdk/lib/deadline/lib/configure-spot-event-plugin.ts

@@ -317,8 +317,6 @@ export interface ConfigureSpotEventPluginProps {
  * Logs for all AWS Lambdas are automatically recorded in Amazon CloudWatch.
  *
  * This construct will configure the Spot Event Plugin, but the Spot Fleet Requests will not be created unless you:
- * - Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
- * - Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
  * - Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs).
  *
  * Important: Disable 'Allow Workers to Perform House Cleaning If Pulse is not Running' in the 'Configure Repository Options'
@@ -459,6 +457,8 @@ export class ConfigureSpotEventPlugin extends Construct {
     };
     const spotFleetRequestConfigs = this.mergeSpotFleetRequestConfigs(props.spotFleets);
 
+    const deadlineGroups = Array.from(new Set(props.spotFleets?.map(fleet => fleet.deadlineGroups).reduce((p, c) => p.concat(c), [])));
+    const deadlinePools = Array.from(new Set(props.spotFleets?.map(fleet => fleet.deadlinePools).reduce((p, c) => p?.concat(c ?? []), [])));
     const properties: SEPConfiguratorResourceProps = {
       connection: {
         hostname: props.renderQueue.endpoint.hostname,
@@ -468,6 +468,8 @@ export class ConfigureSpotEventPlugin extends Construct {
       },
       spotFleetRequestConfigurations: spotFleetRequestConfigs,
       spotPluginConfigurations: pluginConfig,
+      deadlineGroups,
+      deadlinePools,
     };
 
     const resource = new CustomResource(this, 'Default', {

packages/aws-rfdk/lib/deadline/lib/spot-event-plugin-fleet.ts

@@ -90,9 +90,6 @@ export interface SpotEventPluginFleetProps {
   /**
    * Deadline groups these workers need to be assigned to.
    *
-   * Note that you will have to create the groups manually using Deadline before submitting jobs.
-   * See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html
-   *
    * Also, note that the Spot Fleet configuration does not allow using wildcards as part of the Group name
    * as described here https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#wildcards
    */
@@ -101,9 +98,6 @@ export interface SpotEventPluginFleetProps {
   /**
    * Deadline pools these workers need to be assigned to.
    *
-   * Note that you will have to create the pools manually using Deadline before submitting jobs.
-   * See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html
-   *
    * @default - Workers are not assigned to any pool.
    */
   readonly deadlinePools?: string[];
@@ -387,6 +381,13 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
    */
   public readonly deadlineGroups: string[];
 
+  /**
+   * Deadline pools the workers need to be assigned to.
+   *
+   * @default - Workers are not assigned to any pool
+   */
+  public readonly deadlinePools?: string[];
+
   /**
    * Name of SSH keypair to grant access to instances.
    *
@@ -413,6 +414,7 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
     super(scope, id);
 
     this.deadlineGroups = props.deadlineGroups.map(group => group.toLocaleLowerCase());
+    this.deadlinePools = props.deadlinePools?.map(pool => pool.toLocaleLowerCase());
     this.validateProps(props);
 
     this.securityGroups = props.securityGroups ?? [ new SecurityGroup(this, 'SpotFleetSecurityGroup', { vpc: props.vpc }) ];
@@ -463,7 +465,7 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
       renderQueue: props.renderQueue,
       workerSettings: {
         groups: this.deadlineGroups,
-        pools: props.deadlinePools?.map(pool => pool.toLocaleLowerCase()),
+        pools: this.deadlinePools,
         region: props.deadlineRegion,
       },
       userDataProvider: props.userDataProvider,

packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/handler.ts

@@ -47,6 +47,14 @@ export class SEPConfiguratorResource extends SimpleCustomResource {
   public async doCreate(_physicalId: string, resourceProperties: SEPConfiguratorResourceProps): Promise<object|undefined> {
     const spotEventPluginClient = await this.spotEventPluginClient(resourceProperties.connection);
 
+    if (!await spotEventPluginClient.addGroups(resourceProperties.deadlineGroups)) {
+      throw new Error(`Failed to add Deadline group(s) ${resourceProperties.deadlineGroups}`);
+    }
+
+    if (!await spotEventPluginClient.addPools(resourceProperties.deadlinePools)) {
+      throw new Error(`Failed to add Deadline pool(s) ${resourceProperties.deadlinePools}`);
+    }
+
     if (resourceProperties.spotFleetRequestConfigurations) {
       const convertedSpotFleetRequestConfigs = convertSpotFleetRequestConfiguration(resourceProperties.spotFleetRequestConfigurations);
       const stringConfigs = JSON.stringify(convertedSpotFleetRequestConfigs);

packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/test/handler.test.ts

@@ -205,24 +205,44 @@ describe('SEPConfiguratorResource', () => {
     connection: validConnection,
   };
 
+  const deadlineGroups =  ['group_name'];
+  const deadlinePools =  ['pool_name'];
+
   const allConfigs: SEPConfiguratorResourceProps = {
     spotPluginConfigurations: validSpotEventPluginConfig,
     connection: validConnection,
     spotFleetRequestConfigurations: validSpotFleetRequestConfig,
+    deadlineGroups,
+    deadlinePools,
   };
 
   const noConfigs: SEPConfiguratorResourceProps = {
     connection: validConnection,
   };
 
+  async function returnTrue(_v1: any): Promise<boolean> {
+    return true;
+  }
+
+  async function returnFalse(_v1: any): Promise<boolean> {
+    return false;
+  }
+
   describe('doCreate', () => {
     let handler: SEPConfiguratorResource;
-    let mockSpotEventPluginClient: { saveServerData: jest.Mock<any, any>; configureSpotEventPlugin: jest.Mock<any, any>; };
+    let mockSpotEventPluginClient: {
+      saveServerData: jest.Mock<any, any>;
+      configureSpotEventPlugin: jest.Mock<any, any>;
+      addGroups: jest.Mock<any, any>;
+      addPools: jest.Mock<any, any>;
+    };
 
     beforeEach(() => {
       mockSpotEventPluginClient = {
-        saveServerData: jest.fn(),
-        configureSpotEventPlugin: jest.fn(),
+        saveServerData: jest.fn( (a) => returnTrue(a) ),
+        configureSpotEventPlugin: jest.fn( (a) => returnTrue(a) ),
+        addGroups: jest.fn( (a) => returnTrue(a) ),
+        addPools: jest.fn( (a) => returnTrue(a) ),
       };
 
       handler = new SEPConfiguratorResource(new AWS.SecretsManager());
@@ -242,9 +262,6 @@ describe('SEPConfiguratorResource', () => {
 
     test('with no configs', async () => {
       // GIVEN
-      async function returnTrue(_v1: any): Promise<boolean> {
-        return true;
-      }
       const mockSaveServerData = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.saveServerData = mockSaveServerData;
       const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) );
@@ -261,9 +278,6 @@ describe('SEPConfiguratorResource', () => {
 
     test('save spot fleet request configs', async () => {
       // GIVEN
-      async function returnTrue(_v1: any): Promise<boolean> {
-        return true;
-      }
       const mockSaveServerData = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.saveServerData = mockSaveServerData;
 
@@ -281,9 +295,6 @@ describe('SEPConfiguratorResource', () => {
 
     test('save spot fleet request configs without BlockDeviceMappings', async () => {
       // GIVEN
-      async function returnTrue(_v1: any): Promise<boolean> {
-        return true;
-      }
       const mockSaveServerData = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.saveServerData = mockSaveServerData;
 
@@ -326,9 +337,6 @@ describe('SEPConfiguratorResource', () => {
 
     test('save spot fleet request configs without Ebs', async () => {
       // GIVEN
-      async function returnTrue(_v1: any): Promise<boolean> {
-        return true;
-      }
       const mockSaveServerData = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.saveServerData = mockSaveServerData;
 
@@ -375,9 +383,6 @@ describe('SEPConfiguratorResource', () => {
 
     test('save spot event plugin configs', async () => {
       // GIVEN
-      async function returnTrue(_v1: any): Promise<boolean> {
-        return true;
-      }
       const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin;
 
@@ -407,14 +412,22 @@ describe('SEPConfiguratorResource', () => {
       expect(mockConfigureSpotEventPlugin.mock.calls[0][0]).toEqual([...configs, ...securitySettings]);
     });
 
-    test('save both configs', async () => {
+    test('save server data', async () => {
       // GIVEN
-      async function returnTrue(_v1: any): Promise<boolean> {
-        return true;
-      }
       const mockSaveServerData = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.saveServerData = mockSaveServerData;
 
+      // WHEN
+      const result = await handler.doCreate('physicalId', allConfigs);
+
+      // THEN
+      expect(result).toBeUndefined();
+      expect(mockSaveServerData.mock.calls.length).toBe(1);
+      expect(mockSaveServerData.mock.calls[0][0]).toEqual(JSON.stringify(validConvertedSpotFleetRequestConfig));
+    });
+
+    test('configure spot event plugin', async () => {
+      // GIVEN
       const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin;
 
@@ -436,22 +449,67 @@ describe('SEPConfiguratorResource', () => {
       }];
 
       // WHEN
-      const result = await handler.doCreate('physicalId', allConfigs);
+      await handler.doCreate('physicalId', allConfigs);
 
       // THEN
-      expect(result).toBeUndefined();
-      expect(mockSaveServerData.mock.calls.length).toBe(1);
-      expect(mockSaveServerData.mock.calls[0][0]).toEqual(JSON.stringify(validConvertedSpotFleetRequestConfig));
-
       expect(mockConfigureSpotEventPlugin.mock.calls.length).toBe(1);
       expect(mockConfigureSpotEventPlugin.mock.calls[0][0]).toEqual([...configs, ...securitySettings]);
     });
 
+    test('create groups', async () => {
+      // GIVEN
+      const mockAddGroups = jest.fn( (a) => returnTrue(a) );
+      mockSpotEventPluginClient.addGroups = mockAddGroups;
+
+      // WHEN
+      await handler.doCreate('physicalId', allConfigs);
+
+      // THEN
+      expect(mockAddGroups.mock.calls.length).toBe(1);
+      expect(mockAddGroups).toHaveBeenCalledWith(deadlineGroups);
+    });
+
+    test('create pools', async () => {
+      // GIVEN
+      const mockAddPools = jest.fn( (a) => returnTrue(a) );
+      mockSpotEventPluginClient.addPools = mockAddPools;
+
+      // WHEN
+      await handler.doCreate('physicalId', allConfigs);
+
+      // THEN
+      expect(mockAddPools.mock.calls.length).toBe(1);
+      expect(mockAddPools).toHaveBeenCalledWith(deadlinePools);
+    });
+
+    test('throw when cannot add groups', async () => {
+      // GIVEN
+      mockSpotEventPluginClient.addGroups = jest.fn( (a) => returnFalse(a) );
+
+      // WHEN
+      const promise = handler.doCreate('physicalId', allConfigs);
+
+      // THEN
+      await expect(promise)
+        .rejects
+        .toThrowError(`Failed to add Deadline group(s) ${allConfigs.deadlineGroups}`);
+    });
+
+    test('throw when cannot add pools', async () => {
+      // GIVEN
+      mockSpotEventPluginClient.addPools = jest.fn( (a) => returnFalse(a) );
+
+      // WHEN
+      const promise = handler.doCreate('physicalId', allConfigs);
+
+      // THEN
+      await expect(promise)
+        .rejects
+        .toThrowError(`Failed to add Deadline pool(s) ${allConfigs.deadlinePools}`);
+    });
+
     test('throw when cannot save spot fleet request configs', async () => {
       // GIVEN
-      async function returnFalse(_v1: any): Promise<boolean> {
-        return false;
-      }
       const mockSaveServerData = jest.fn( (a) => returnFalse(a) );
       mockSpotEventPluginClient.saveServerData = mockSaveServerData;
 
@@ -466,9 +524,6 @@ describe('SEPConfiguratorResource', () => {
 
     test('throw when cannot save spot event plugin configs', async () => {
       // GIVEN
-      async function returnFalse(_v1: any): Promise<boolean> {
-        return false;
-      }
       const mockConfigureSpotEventPlugin = jest.fn( (a) => returnFalse(a) );
       mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin;
 

packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/types.ts

@@ -23,6 +23,16 @@ export interface SEPConfiguratorResourceProps {
    * See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#event-plugin-configuration-options
    */
   readonly spotPluginConfigurations?: PluginSettings;
+
+  /**
+   * Deadline groups that are used by these fleets.
+   */
+  readonly deadlineGroups?: string[];
+
+  /**
+   * Deadline pools that are used by these fleets.
+   */
+  readonly deadlinePools?: string[];
 }
 
 /**