Merge pull request #253 from aws/bump/0.21.0

chore(release): 0.21.0

CHANGELOG.md

@@ -2,6 +2,34 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [0.21.0](https://github.com/aws/aws-rfdk/compare/v0.20.0...v0.21.0) (2020-11-27)
+
+
+### Supported CDK Version
+
+* [1.75.0](https://github.com/aws/aws-cdk/releases/tag/v1.75.0)
+
+
+### Officially Supported Deadline Versions
+
+* [10.1.9.2 to 10.1.11.5](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/release-notes.html)
+
+
+### Features
+
+* **core:** Add configurable expiry to X.509 certificates ([#242](https://github.com/aws/aws-rfdk/issues/242)) ([ae7c153](https://github.com/aws/aws-rfdk/commit/ae7c1536c009909fe72e45385e56271d3b1cab0e))
+* **deadline:** add custom user data commands to Worker instance startup ([#239](https://github.com/aws/aws-rfdk/issues/239)) ([bdef391](https://github.com/aws/aws-rfdk/commit/bdef391df283864bbb0d05dea1e094785c333b33))
+* **examples:**  Added examples for Spot Event Plugin Deployment ([#180](https://github.com/aws/aws-rfdk/issues/180)) ([49e22bf](https://github.com/aws/aws-rfdk/commit/49e22bff5e89164e3f1daeeb24088e5112c7f8d8))
+* **integ:** use configurable version of Deadline for integration tests ([#160](https://github.com/aws/aws-rfdk/issues/160)) ([263021c](https://github.com/aws/aws-rfdk/commit/263021c1116ed81e091a7e9363122ace14e81e84))
+
+
+### Bug Fixes
+
+* **deadline:** bad file path for Repository when using VersionQuery ([#252](https://github.com/aws/aws-rfdk/issues/252)) ([84a20de](https://github.com/aws/aws-rfdk/commit/84a20de3f3e9fc49017626f2233929cf03d2e277))
+* **deadline:** Fix cyclic stack dependency when using UBL ([#246](https://github.com/aws/aws-rfdk/issues/246)) ([12f7db2](https://github.com/aws/aws-rfdk/commit/12f7db23cf18e71fb0fa4c7657fbe9f5455ac4f4))
+* **deadline:** lock down DocDC engine to version 3.6.0 ([#230](https://github.com/aws/aws-rfdk/issues/230)) ([2f46944](https://github.com/aws/aws-rfdk/commit/2f46944ff35123a828be30a7bdf9e7e0ca944b14))
+* **deadline:** Launcher restarts Workers reliably. RFDK assumed Workers connected to the Deadline Render Queue when configuring them ([#248](https://github.com/aws/aws-rfdk/issues/248)) ([dfdbda5](https://github.com/aws/aws-rfdk/commit/dfdbda518c43b981eb3835b23592896817b984cb))
+
 ## [0.20.0](https://github.com/aws/aws-rfdk/compare/v0.19.0...v0.20.0) (2020-11-10)
 
 

examples/deadline/All-In-AWS-Infrastructure-Basic/python/README.md

@@ -8,7 +8,7 @@
 ---
 **NOTE**
 
-These instructions assume that your working directory is `examples/deadline/All-In-AWS-Infrastructure-Basic-Tiered/python/` relative to the root of the AWS-RFDK package.
+These instructions assume that your working directory is `examples/deadline/All-In-AWS-Infrastructure-Basic/python/` relative to the root of the AWS-RFDK package.
 
 ---
 

examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/app.py

@@ -120,6 +120,8 @@ def main():
         render_queue=service.render_queue,
         worker_machine_image=deadline_client_image,
         key_pair_name=config.key_pair_name,
+        usage_based_licensing=service.ubl_licensing,
+        licenses=config.ubl_licenses
     )
     _compute = compute_tier.ComputeTier(app, 'ComputeTier', props=compute_props, env=env)
 

examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/lib/compute_tier.py

@@ -3,6 +3,7 @@
 
 from dataclasses import dataclass
 from typing import (
+    List,
     Optional
 )
 
@@ -17,15 +18,22 @@
     IVpc,
     Port
 )
+from aws_cdk.aws_s3_assets import (
+  Asset
+)
 
 from aws_rfdk import (
     HealthMonitor,
 )
 from aws_rfdk.deadline import (
+    InstanceUserDataProvider,
     IRenderQueue,
+    UsageBasedLicense,
+    UsageBasedLicensing,
     WorkerInstanceFleet,
 )
 
+import os
 
 @dataclass
 class ComputeTierProps(StackProps):
@@ -42,7 +50,35 @@ class ComputeTierProps(StackProps):
     key_pair_name: Optional[str]
     # The bastion host to  allow connection to Worker nodes.
     bastion: Optional[BastionHostLinux] = None
+    # Licensing source for UBL for worker nodes.
+    usage_based_licensing: Optional[UsageBasedLicensing] = None
+    # List of the usage-based liceses that the worker nodes will be served.
+    licenses: Optional[List[UsageBasedLicense]] = None
+
+class UserDataProvider(InstanceUserDataProvider):
+    def __init__(self, scope: Construct, stack_id: str):
+        super().__init__(scope, stack_id)
+        self.test_script=Asset(scope, "SampleAsset",
+            path=os.path.join(os.getcwd(), "..", "scripts", "configure_worker.sh")
+        )
+
+    def pre_cloud_watch_agent(self, host) -> None:
+        host.user_data.add_commands("echo preCloudWatchAgent")
 
+    def pre_render_queue_configuration(self, host) -> None:
+        host.user_data.add_commands("echo preRenderQueueConfiguration")
+
+    def pre_worker_configuration(self, host) -> None:
+        host.user_data.add_commands("echo preWorkerConfiguration")
+
+    def post_worker_launch(self, host) -> None:
+        host.user_data.add_commands("echo postWorkerLaunch")
+        self.test_script.grant_read(host)
+        local_path = host.user_data.add_s3_download_command(
+            bucket=self.test_script.bucket,
+            bucket_key=self.test_script.s3_object_key
+        )
+        host.user_data.add_execute_file_command(file_path=local_path)
 
 class ComputeTier(Stack):
     """
@@ -78,7 +114,11 @@ def __init__(self, scope: Construct, stack_id: str, *, props: ComputeTierProps,
             worker_machine_image=props.worker_machine_image,
             health_monitor=self.health_monitor,
             key_name=props.key_pair_name,
+            user_data_provider=UserDataProvider(self, 'UserDataProvider')
         )
 
+        if props.usage_based_licensing and props.licenses:
+            props.usage_based_licensing.grant_port_access(self.worker_fleet, props.licenses)
+
         if props.bastion:
             self.worker_fleet.connections.allow_from(props.bastion, Port.tcp(22))

examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/lib/service_tier.py

@@ -175,3 +175,5 @@ def __init__(self, scope: Construct, stack_id: str, *, props: ServiceTierProps,
                 render_queue=self.render_queue,
                 certificate_secret=ubl_cert_secret,
             )
+        else:
+            self.ubl_licensing = None

examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/lib/storage_tier.py

@@ -126,8 +126,11 @@ def __init__(self, scope: Construct, stack_id: str, *, props: StorageTierDocDBPr
             self,
             'DocDBCluster',
             instance_props=instance_props,
-            instances=len(self.availability_zones),
+            # TODO - For cost considerations this example only uses 1 Database instance. 
+            # It is recommended that when creating your render farm you use at least 2 instances for redundancy.
+            instances=1,
             master_user=Login(username='adminuser'),
+            engine_version='3.6.0',
             backup=BackupProps(
                 # We recommend setting the retention of your backups to 15 days
                 # for security reasons. The default retention is just one day.

examples/deadline/All-In-AWS-Infrastructure-Basic/python/setup.py

@@ -17,8 +17,8 @@
     packages=setuptools.find_packages(where="package"),
 
     install_requires=[
-        "aws-cdk.core==1.72.0",
-        "aws-rfdk==0.20.0"
+        "aws-cdk.core==1.75.0",
+        "aws-rfdk==0.21.0"
     ],
 
     python_requires=">=3.7",

examples/deadline/All-In-AWS-Infrastructure-Basic/scripts/configure_worker.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# This script is only here to illustrate how to add custom user data when using our WorkerInstanceFleet and WorkerInstanceConfiguration constructs
+mkdir test_dir 

examples/deadline/All-In-AWS-Infrastructure-Basic/ts/README.md

@@ -8,7 +8,7 @@
 ---
 **NOTE**
 
-These instructions assume that your working directory is `examples/deadline/All-In-AWS-Infrastructure-Basic-Tiered/ts/` relative to the root of the RFDK package.
+These instructions assume that your working directory is `examples/deadline/All-In-AWS-Infrastructure-Basic/ts/` relative to the root of the RFDK package.
 
 ---
 
@@ -27,6 +27,12 @@ These instructions assume that your working directory is `examples/deadline/All-
       // ...
       };
     ```
+
+    ---
+
+    **Note:** The next three steps are for setting up usage based licensing and are optional. You may skip these if you do not need to use licenses for rendering.
+
+    ---
 4.  Create a binary secret in [SecretsManager](https://aws.amazon.com/secrets-manager/) that contains your [Usage-Based Licensing](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/aws-portal/licensing-setup.html?highlight=usage%20based%20licensing) certificates in a `.zip` file:
 
     ```
@@ -53,7 +59,7 @@ These instructions assume that your working directory is `examples/deadline/All-
 
     ---
 
-    **Note:** The next two steps are optional. You may skip these if you do not need   SSH access into your render farm.
+    **Note:** The next two steps are for allowing SSH access to your render farm and are optional. You may skip these if you do not need SSH access into your render farm.
 
     ---
 7.  Create an EC2 key pair to give you SSH access to the render farm:

examples/deadline/All-In-AWS-Infrastructure-Basic/ts/bin/app.ts

@@ -29,12 +29,12 @@ import { ComputeTier } from '../lib/compute-tier';
   // --- Validate Config Values --- //
   // ------------------------------ //
 
-  if (!config.ublCertificatesSecretArn) {
-    throw new Error('UBL certificates secret ARN is required but was not specified.');
+  if (!config.ublCertificatesSecretArn && config.ublLicenses) {
+    throw new Error('UBL certificates secret ARN is required when using UBL but was not specified.');
   }
 
   if (!config.ublLicenses) {
-    throw new Error('At least one UBL license must be specified');
+    console.warn('No UBL licenses specified. UsageBasedLicensing will be skipped.');
   }
 
   if (!config.keyPairName) {
@@ -117,4 +117,6 @@ new ComputeTier(app, 'ComputeTier', {
   renderQueue: service.renderQueue,
   workerMachineImage: MachineImage.genericLinux(config.deadlineClientLinuxAmiMap),
   keyPairName: config.keyPairName ? config.keyPairName : undefined,
+  usageBasedLicensing: service.ublLicensing,
+  licenses: config.ublLicenses,
 });

examples/deadline/All-In-AWS-Infrastructure-Basic/ts/bin/config.ts

@@ -9,30 +9,32 @@ import { MongoDbSsplLicenseAcceptance } from 'aws-rfdk';
 
 /**
  * Configuration values for the sample app.
- * 
+ *
  * TODO: Fill these in with your own values.
  */
 class AppConfig {
 
   /**
-   * A map of regions to Deadline Client Linux AMIs.
+   * A map of regions to Deadline Client Linux AMIs. As an example, the Linux Deadline 10.1.11.5 AMI ID from us-west-2
+   * is filled in. It can be used as-is, added to, or replaced. Ideally the version here should match the one in
+   * package.json used for staging the render queue and usage based licensing recipes.
    */
-  public readonly deadlineClientLinuxAmiMap: Record<string, string> = {['region']: 'ami-id'};
+  public readonly deadlineClientLinuxAmiMap: Record<string, string> = {['us-west-2']: 'ami-0b12631d34ca9939f'};
 
   /**
-   * A secret (in binary form) in SecretsManager that stores the UBL certificates in a .zip file.
+   * (Optional) A secret (in binary form) in SecretsManager that stores the UBL certificates in a .zip file.
    */
-  public readonly ublCertificatesSecretArn: string = '';
+  public readonly ublCertificatesSecretArn?: string;
 
   /**
-   * The UBL licenses to use.
+   * (Optional) The UBL licenses to use.
    */
-  public readonly ublLicenses: UsageBasedLicense[] = [];
+  public readonly ublLicenses?: UsageBasedLicense[];
 
   /**
    * (Optional) The name of the EC2 keypair to associate with instances.
    */
-  public readonly keyPairName: string = '';
+  public readonly keyPairName?: string;
 
   /**
    * Whether to use MongoDB to back the render farm.

examples/deadline/All-In-AWS-Infrastructure-Basic/ts/lib/compute-tier.ts

@@ -11,14 +11,20 @@ import {
 } from '@aws-cdk/aws-ec2';
 import * as cdk from '@aws-cdk/core';
 import {
+  IHost,
+  InstanceUserDataProvider,  
   IRenderQueue,
   IWorkerFleet,
+  UsageBasedLicense,
+  UsageBasedLicensing,
   WorkerInstanceFleet,
 } from 'aws-rfdk/deadline';
 import {
   HealthMonitor,
   IHealthMonitor,
 } from 'aws-rfdk';
+import { Asset } from '@aws-cdk/aws-s3-assets';
+import * as path from 'path'
 
 /**
  * Properties for {@link ComputeTier}.
@@ -48,6 +54,46 @@ export interface ComputeTierProps extends cdk.StackProps {
    * The bastion host to allow connection to Worker nodes.
    */
   readonly bastion?: BastionHostLinux;
+
+  /**
+   * Licensing source for UBL for worker nodes.
+   */
+  readonly usageBasedLicensing?: UsageBasedLicensing;
+
+  /**
+   * List of the usage-based liceses that the worker nodes will be served.
+   */
+  readonly licenses?: UsageBasedLicense[];
+}
+
+class UserDataProvider extends InstanceUserDataProvider {
+  preCloudWatchAgent(host: IHost): void {
+    host.userData.addCommands('echo preCloudWatchAgent');
+  }
+  preRenderQueueConfiguration(host: IHost): void {
+    host.userData.addCommands('echo preRenderQueueConfiguration');
+  }  
+  preWorkerConfiguration(host: IHost): void {
+    host.userData.addCommands('echo preWorkerConfiguration');
+  }
+  postWorkerLaunch(host: IHost): void {
+    host.userData.addCommands('echo postWorkerLaunch');
+    if (host.node.scope != undefined) {
+      const testScript = new Asset(
+        host.node.scope as cdk.Construct, 
+        'SampleAsset',
+        {path: path.join(__dirname, '..', '..', 'scripts', 'configure_worker.sh')},
+      );
+      testScript.grantRead(host);
+      const localPath = host.userData.addS3DownloadCommand({
+        bucket: testScript.bucket,
+        bucketKey: testScript.s3ObjectKey,
+      });
+      host.userData.addExecuteFileCommand({
+        filePath: localPath,
+      })
+    }
+  }
 }
 
 /**
@@ -88,8 +134,13 @@ export class ComputeTier extends cdk.Stack {
       workerMachineImage: props.workerMachineImage,
       healthMonitor: this.healthMonitor,
       keyName: props.keyPairName,
+      userDataProvider: new UserDataProvider(this, 'UserDataProvider'),
     });
 
+    if (props.usageBasedLicensing && props.licenses) {
+      props.usageBasedLicensing.grantPortAccess(this.workerFleet, props.licenses);
+    }
+
     if (props.bastion) {
       this.workerFleet.connections.allowFrom(props.bastion, Port.tcp(22));
     }