fix(core): Add Groups and Pools for SpotEventPluginFleet

packages/aws-rfdk/lib/deadline/lib/configure-spot-event-plugin.ts

@@ -459,6 +459,8 @@ export class ConfigureSpotEventPlugin extends Construct {
     };
     const spotFleetRequestConfigs = this.mergeSpotFleetRequestConfigs(props.spotFleets);
 
+    const deadlineGroups = Array.from(new Set(props.spotFleets?.map(fleet => fleet.deadlineGroups).reduce((p, c) => p.concat(c), [])));
+    const deadlinePools = Array.from(new Set(props.spotFleets?.map(fleet => fleet.deadlinePools).reduce((p, c) => p?.concat(c ?? []), [])));
     const properties: SEPConfiguratorResourceProps = {
       connection: {
         hostname: props.renderQueue.endpoint.hostname,
@@ -468,6 +470,8 @@ export class ConfigureSpotEventPlugin extends Construct {
       },
       spotFleetRequestConfigurations: spotFleetRequestConfigs,
       spotPluginConfigurations: pluginConfig,
+      deadlineGroups,
+      deadlinePools,
     };
 
     const resource = new CustomResource(this, 'Default', {

packages/aws-rfdk/lib/deadline/lib/spot-event-plugin-fleet.ts

@@ -387,6 +387,13 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
    */
   public readonly deadlineGroups: string[];
 
+  /**
+   * Deadline pools the workers need to be assigned to.
+   *
+   * @default - Workers are not assigned to any pool
+   */
+  public readonly deadlinePools?: string[];
+
   /**
    * Name of SSH keypair to grant access to instances.
    *
@@ -413,6 +420,7 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
     super(scope, id);
 
     this.deadlineGroups = props.deadlineGroups.map(group => group.toLocaleLowerCase());
+    this.deadlinePools = props.deadlinePools?.map(pool => pool.toLocaleLowerCase());
     this.validateProps(props);
 
     this.securityGroups = props.securityGroups ?? [ new SecurityGroup(this, 'SpotFleetSecurityGroup', { vpc: props.vpc }) ];
@@ -463,7 +471,7 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
       renderQueue: props.renderQueue,
       workerSettings: {
         groups: this.deadlineGroups,
-        pools: props.deadlinePools?.map(pool => pool.toLocaleLowerCase()),
+        pools: this.deadlinePools,
         region: props.deadlineRegion,
       },
       userDataProvider: props.userDataProvider,

packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/handler.ts

@@ -6,7 +6,11 @@
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { SecretsManager } from 'aws-sdk';
 import { LambdaContext } from '../lib/aws-lambda';
-import { SpotEventPluginClient } from '../lib/configure-spot-event-plugin';
+import {
+  CollectionType,
+  PoolCollections,
+  SpotEventPluginClient,
+} from '../lib/configure-spot-event-plugin';
 import { CfnRequestEvent, SimpleCustomResource } from '../lib/custom-resource';
 import { DeadlineClient } from '../lib/deadline-client';
 import {
@@ -47,6 +51,37 @@ export class SEPConfiguratorResource extends SimpleCustomResource {
   public async doCreate(_physicalId: string, resourceProperties: SEPConfiguratorResourceProps): Promise<object|undefined> {
     const spotEventPluginClient = await this.spotEventPluginClient(resourceProperties.connection);
 
+    if (resourceProperties.deadlineGroups && resourceProperties.deadlineGroups.length) {
+      const deadlineGroups = await spotEventPluginClient.getCollection(CollectionType.Group);
+      const newDeadlineGroups = deadlineGroups.Pools
+        .concat(resourceProperties.deadlineGroups
+          .filter(group => !deadlineGroups.Pools.includes(group)));
+      const response = await spotEventPluginClient
+        .saveCollection({
+          Pools: newDeadlineGroups,
+          ObsoletePools: deadlineGroups.ObsoletePools,
+        } as PoolCollections,
+        CollectionType.Group);
+      if (!response) {
+        throw new Error('Failed to save group collection');
+      }
+    }
+    if (resourceProperties.deadlinePools && resourceProperties.deadlinePools.length) {
+      const deadlinePools = await spotEventPluginClient.getCollection(CollectionType.Pool);
+      const newDeadlinePools = deadlinePools.Pools
+        .concat(resourceProperties.deadlinePools
+          .filter(pool => !deadlinePools.Pools.includes(pool)));
+      const response = await spotEventPluginClient
+        .saveCollection({
+          Pools: newDeadlinePools,
+          ObsoletePools: deadlinePools.ObsoletePools,
+        } as PoolCollections,
+        CollectionType.Pool);
+      if (!response) {
+        throw new Error('Failed to save pool collection');
+      }
+    }
+
     if (resourceProperties.spotFleetRequestConfigurations) {
       const convertedSpotFleetRequestConfigs = convertSpotFleetRequestConfiguration(resourceProperties.spotFleetRequestConfigurations);
       const stringConfigs = JSON.stringify(convertedSpotFleetRequestConfigs);

packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/test/handler.test.ts

@@ -19,6 +19,7 @@ import {
   SpotFleetRequestType,
   SpotFleetResourceType,
 } from '../../../../deadline';
+import { CollectionType } from '../../lib/configure-spot-event-plugin';
 import { SEPConfiguratorResource } from '../handler';
 import {
   ConnectionOptions,
@@ -205,24 +206,48 @@ describe('SEPConfiguratorResource', () => {
     connection: validConnection,
   };
 
+  const deadlineGroups =  ['group_name'];
+  const deadlinePools =  ['pool_name'];
+
   const allConfigs: SEPConfiguratorResourceProps = {
     spotPluginConfigurations: validSpotEventPluginConfig,
     connection: validConnection,
     spotFleetRequestConfigurations: validSpotFleetRequestConfig,
+    deadlineGroups,
+    deadlinePools,
+  };
+
+  const noGroupsButPools: SEPConfiguratorResourceProps = {
+    ... allConfigs,
+    deadlineGroups: [],
   };
 
   const noConfigs: SEPConfiguratorResourceProps = {
     connection: validConnection,
   };
 
+  async function returnEmptyCollection(_v1: any): Promise<any> {
+    return {
+      Pools: [],
+      ObsoletePools: [],
+    };
+  }
+
   describe('doCreate', () => {
     let handler: SEPConfiguratorResource;
-    let mockSpotEventPluginClient: { saveServerData: jest.Mock<any, any>; configureSpotEventPlugin: jest.Mock<any, any>; };
+    let mockSpotEventPluginClient: {
+      saveServerData: jest.Mock<any, any>;
+      configureSpotEventPlugin: jest.Mock<any, any>;
+      getCollection: jest.Mock<any, any>;
+      saveCollection: jest.Mock<any, any>;
+    };
 
     beforeEach(() => {
       mockSpotEventPluginClient = {
         saveServerData: jest.fn(),
         configureSpotEventPlugin: jest.fn(),
+        getCollection: jest.fn(),
+        saveCollection: jest.fn(),
       };
 
       handler = new SEPConfiguratorResource(new AWS.SecretsManager());
@@ -250,6 +275,7 @@ describe('SEPConfiguratorResource', () => {
       const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin;
 
+
       // WHEN
       const result = await handler.doCreate('physicalId', noConfigs);
 
@@ -412,12 +438,19 @@ describe('SEPConfiguratorResource', () => {
       async function returnTrue(_v1: any): Promise<boolean> {
         return true;
       }
+
       const mockSaveServerData = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.saveServerData = mockSaveServerData;
 
       const mockConfigureSpotEventPlugin = jest.fn( (a) => returnTrue(a) );
       mockSpotEventPluginClient.configureSpotEventPlugin = mockConfigureSpotEventPlugin;
 
+      const mockGetCollection = jest.fn(async (a) => returnEmptyCollection(a));
+      mockSpotEventPluginClient.getCollection = mockGetCollection;
+
+      const mockSaveCollection = jest.fn( (a) => returnTrue(a) );
+      mockSpotEventPluginClient.saveCollection = mockSaveCollection;
+
       const configs: { Key: string, Value: any }[] = [];
       for (const [key, value] of Object.entries(validConvertedPluginConfig)) {
         configs.push({
@@ -445,6 +478,14 @@ describe('SEPConfiguratorResource', () => {
 
       expect(mockConfigureSpotEventPlugin.mock.calls.length).toBe(1);
       expect(mockConfigureSpotEventPlugin.mock.calls[0][0]).toEqual([...configs, ...securitySettings]);
+
+      expect(mockGetCollection.mock.calls.length).toBe(2);
+      expect(mockGetCollection.mock.calls[0][0]).toBe(CollectionType.Group);
+      expect(mockGetCollection.mock.calls[1][0]).toBe(CollectionType.Pool);
+
+      expect(mockSaveCollection.mock.calls.length).toBe(2);
+      expect(mockSaveCollection.mock.calls[0][0]).toEqual({ Pools: deadlineGroups, ObsoletePools: []});
+      expect(mockSaveCollection.mock.calls[1][0]).toEqual({ Pools: deadlinePools, ObsoletePools: []});
     });
 
     test('throw when cannot save spot fleet request configs', async () => {
@@ -480,6 +521,31 @@ describe('SEPConfiguratorResource', () => {
         .rejects
         .toThrowError(/Failed to save Spot Event Plugin Configurations/);
     });
+
+    test.each([
+      ['group', allConfigs],
+      ['pool', noGroupsButPools],
+    ])('throws when collection fails to save %s', async (collectionType: string, config: any) => {
+
+      // GIVEN
+      async function returnFalse(_v1: any): Promise<boolean> {
+        return false;
+      }
+
+      const mockGetCollection = jest.fn( async (a) => returnEmptyCollection(a) );
+      mockSpotEventPluginClient.getCollection = mockGetCollection;
+
+      const mockSaveCollection = jest.fn( (a) => returnFalse(a) );
+      mockSpotEventPluginClient.saveCollection = mockSaveCollection;
+
+      // WHEN
+      const promise = handler.doCreate('physicalId', config);
+
+      // THEN
+      await expect(promise)
+        .rejects
+        .toThrowError(`Failed to save ${collectionType} collection`);
+    });
   });
 
   test('doDelete does not do anything', async () => {

packages/aws-rfdk/lib/lambdas/nodejs/configure-spot-event-plugin/types.ts

@@ -23,6 +23,16 @@ export interface SEPConfiguratorResourceProps {
    * See https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#event-plugin-configuration-options
    */
   readonly spotPluginConfigurations?: PluginSettings;
+
+  /**
+   * Deadline groups are used by these fleets.
+   */
+  readonly deadlineGroups?: string[];
+
+  /**
+   * Deadline pools are used by these fleets.
+   */
+  readonly deadlinePools?: string[];
 }
 
 /**

packages/aws-rfdk/lib/lambdas/nodejs/lib/configure-spot-event-plugin/spot-event-plugin-client.ts

@@ -25,6 +25,30 @@ interface DescribeServerDataResponse {
   readonly ServerData: DescribedServerData[];
 }
 
+/**
+ * A response from get pool/group request
+ */
+export interface PoolCollections {
+  /**
+   * The collection of user-created Pools that are currently active
+   */
+  readonly Pools: string [];
+
+  /**
+   * The collection of Pools that are currently obsolete
+   */
+  readonly ObsoletePools: string [];
+}
+
+/**
+ * A type of collection to get/recive from Deadline.
+ */
+export enum CollectionType {
+  Pool = 'pool',
+
+  Group = 'group',
+}
+
 /**
  * Provides a simple interface to send requests to the Render Queue API related to the Deadline Spot Event Plugin.
  */
@@ -94,6 +118,45 @@ export class SpotEventPluginClient {
     }
   }
 
+  public async getCollection(type: CollectionType): Promise<PoolCollections> {
+    console.log(`Getting ${type} collection:`);
+    try {
+      const response = await this.deadlineClient.GetRequest(`/db/settings/collections/${type}s?invalidateCache=true`, {
+        headers: {
+          'Content-Type': 'application/json; charset=utf-8',
+        },
+      });
+      const deadlinePools: PoolCollections = response.data;
+      if (!deadlinePools.Pools || !Array.isArray(deadlinePools.Pools)) {
+        throw new Error(`Failed to receive a ${type} collection. Invalid response: ${JSON.stringify(response.data)}.`);
+      }
+      return deadlinePools;
+    } catch(e) {
+      throw new Error(`Failed to get ${type} collection. Reason: ${(<Error>e).message}`);
+    }
+  }
+
+  public async saveCollection(pools: PoolCollections, type: CollectionType): Promise<boolean> {
+    console.log(`Saving ${type} collection:`);
+    console.log(pools);
+
+    try {
+      await this.deadlineClient.PostRequest(`/db/settings/collections/${type}s/save`, {
+        Pools: pools.Pools,
+        ObsoletePools: pools.ObsoletePools,
+      },
+      {
+        headers: {
+          'Content-Type': 'application/json; charset=utf-8',
+        },
+      });
+      return true;
+    } catch(e) {
+      console.error(`Failed to save ${type} collection. Reason: ${(<Error>e).message}`);
+      return false;
+    }
+  }
+
   private async describeServerData(): Promise<Response> {
     return await this.deadlineClient.PostRequest('/rcs/v1/describeServerData', {
       ServerDataIds: [