diff --git a/examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/lib/storage_tier.py b/examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/lib/storage_tier.py index 63ccc96d6..b64abb8ea 100644 --- a/examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/lib/storage_tier.py +++ b/examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/lib/storage_tier.py @@ -44,6 +44,7 @@ MongoDbSsplLicenseAcceptance, MongoDbVersion, MountableEfs, + PadEfsStorage, X509CertificatePem, X509CertificatePkcs12 ) @@ -91,6 +92,34 @@ def __init__(self, scope: Construct, stack_id: str, *, props: StorageTierProps, removal_policy=RemovalPolicy.DESTROY ) + # Add padding files to the filesystem to increase baseline throughput. Deadline's Repository filesystem + # is small (initial size of about 1GB), which results in a very low baseline throughput for the Amazon + # EFS filesystem. We add files to the filesystem to increase this baseline throughput, while retaining the + # ability to burst throughput. See RFDK's PadEfsStorage documentation for additional details. + pad_access_point = AccessPoint( + self, + 'PaddingAccessPoint', + file_system=file_system, + path='/PaddingFiles', + # TODO - We set the padding files to be owned by root (uid/gid = 0) by default. You may wish to change this. + create_acl=Acl( + owner_gid='0', + owner_uid='0', + permissions='700', + ), + posix_user=PosixUser( + uid='0', + gid='0', + ), + ) + PadEfsStorage( + self, + 'PadEfsStorage', + vpc=props.vpc, + access_point=pad_access_point, + desired_padding_gb=40, # Provides 2 MB/s of baseline throughput. Costs $12/month. + ) + # Create an EFS access point that is used to grant the Repository and RenderQueue with write access to the # Deadline Repository directory in the EFS file-system. access_point = AccessPoint( diff --git a/examples/deadline/All-In-AWS-Infrastructure-Basic/ts/lib/storage-tier.ts b/examples/deadline/All-In-AWS-Infrastructure-Basic/ts/lib/storage-tier.ts index d7bd930e2..4b2b53510 100644 --- a/examples/deadline/All-In-AWS-Infrastructure-Basic/ts/lib/storage-tier.ts +++ b/examples/deadline/All-In-AWS-Infrastructure-Basic/ts/lib/storage-tier.ts @@ -22,6 +22,7 @@ import { MongoDbSsplLicenseAcceptance, MongoDbVersion, MountableEfs, + PadEfsStorage, X509CertificatePem, X509CertificatePkcs12, } from 'aws-rfdk'; @@ -74,6 +75,30 @@ export abstract class StorageTier extends cdk.Stack { removalPolicy: RemovalPolicy.DESTROY, }); + // Add padding files to the filesystem to increase baseline throughput. Deadline's Repository filesystem + // is small (initial size of about 1GB), which results in a very low baseline throughput for the Amazon + // EFS filesystem. We add files to the filesystem to increase this baseline throughput, while retaining the + // ability to burst throughput. See RFDK's PadEfsStorage documentation for additional details. + const padAccessPoint = new AccessPoint(this, 'PaddingAccessPoint', { + fileSystem, + path: '/PaddingFiles', + // TODO - We set the padding files to be owned by root (uid/gid = 0) by default. You may wish to change this. + createAcl: { + ownerGid: '0', + ownerUid: '0', + permissions: '700', + }, + posixUser: { + uid: '0', + gid: '0', + }, + }); + new PadEfsStorage(this, 'PadEfsStorage', { + vpc: props.vpc, + accessPoint: padAccessPoint, + desiredPaddingGB: 40, // Provides 2 MB/s of baseline throughput. Costs $12/month. + }); + // Create an EFS access point that is used to grant the Repository and RenderQueue with write access to the Deadline // Repository directory in the EFS file-system. const accessPoint = new AccessPoint(this, 'AccessPoint', { diff --git a/package.json b/package.json index 298d48818..c7f344032 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@aws-cdk/assets": "1.94.1", "@aws-cdk/aws-apigateway": "1.94.1", "@aws-cdk/aws-apigatewayv2": "1.94.1", + "@aws-cdk/aws-apigatewayv2-integrations": "1.94.1", "@aws-cdk/aws-applicationautoscaling": "1.94.1", "@aws-cdk/aws-autoscaling": "1.94.1", "@aws-cdk/aws-autoscaling-common": "1.94.1", @@ -44,6 +45,7 @@ "@aws-cdk/aws-codeguruprofiler": "1.94.1", "@aws-cdk/aws-codepipeline": "1.94.1", "@aws-cdk/aws-cognito": "1.94.1", + "@aws-cdk/aws-databrew": "1.94.1", "@aws-cdk/aws-docdb": "1.94.1", "@aws-cdk/aws-dynamodb": "1.94.1", "@aws-cdk/aws-ec2": "1.94.1", @@ -52,10 +54,13 @@ "@aws-cdk/aws-ecs": "1.94.1", "@aws-cdk/aws-ecs-patterns": "1.94.1", "@aws-cdk/aws-efs": "1.94.1", + "@aws-cdk/aws-eks": "1.94.1", "@aws-cdk/aws-elasticloadbalancing": "1.94.1", "@aws-cdk/aws-elasticloadbalancingv2": "1.94.1", "@aws-cdk/aws-events": "1.94.1", "@aws-cdk/aws-events-targets": "1.94.1", + "@aws-cdk/aws-globalaccelerator": "1.94.1", + "@aws-cdk/aws-glue": "1.94.1", "@aws-cdk/aws-iam": "1.94.1", "@aws-cdk/aws-kinesis": "1.94.1", "@aws-cdk/aws-kinesisfirehose": "1.94.1", @@ -69,15 +74,19 @@ "@aws-cdk/aws-sam": "1.94.1", "@aws-cdk/aws-secretsmanager": "1.94.1", "@aws-cdk/aws-servicediscovery": "1.94.1", + "@aws-cdk/aws-signer": "1.94.1", "@aws-cdk/aws-sns": "1.94.1", "@aws-cdk/aws-sns-subscriptions": "1.94.1", "@aws-cdk/aws-sqs": "1.94.1", "@aws-cdk/aws-ssm": "1.94.1", "@aws-cdk/aws-stepfunctions": "1.94.1", + "@aws-cdk/aws-stepfunctions-tasks": "1.94.1", "@aws-cdk/cloud-assembly-schema": "1.94.1", "@aws-cdk/core": "1.94.1", "@aws-cdk/custom-resources": "1.94.1", "@aws-cdk/cx-api": "1.94.1", + "@aws-cdk/lambda-layer-awscli": "1.94.1", + "@aws-cdk/lambda-layer-kubectl": "1.94.1", "@aws-cdk/region-info": "1.94.1", "@types/jest": "^26.0.21", "@types/node": "^14.14.35", diff --git a/packages/aws-rfdk/lib/core/lib/index.ts b/packages/aws-rfdk/lib/core/lib/index.ts index c8138fc97..94ec110e8 100644 --- a/packages/aws-rfdk/lib/core/lib/index.ts +++ b/packages/aws-rfdk/lib/core/lib/index.ts @@ -16,6 +16,7 @@ export * from './mongodb-post-install'; export * from './mountable-ebs'; export * from './mountable-efs'; export * from './mountable-filesystem'; +export * from './pad-efs-storage'; export { RFDK_VERSION } from './runtime-info'; export * from './script-assets'; export * from './session-manager-helper'; diff --git a/packages/aws-rfdk/lib/core/lib/pad-efs-storage.ts b/packages/aws-rfdk/lib/core/lib/pad-efs-storage.ts new file mode 100644 index 000000000..29b74e7f3 --- /dev/null +++ b/packages/aws-rfdk/lib/core/lib/pad-efs-storage.ts @@ -0,0 +1,322 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as path from 'path'; +import { + ISecurityGroup, + IVpc, + SecurityGroup, + SubnetSelection, + SubnetType, +} from '@aws-cdk/aws-ec2'; +import { + IAccessPoint, +} from '@aws-cdk/aws-efs'; +import { + Code, + FileSystem as LambdaFilesystem, + Function as LambdaFunction, + Runtime, +} from '@aws-cdk/aws-lambda'; +import { + RetentionDays, +} from '@aws-cdk/aws-logs'; +import { + Choice, + Condition, + Fail, + InputType, + StateMachine, + Succeed, +}from '@aws-cdk/aws-stepfunctions'; +import { + LambdaInvoke, +} from '@aws-cdk/aws-stepfunctions-tasks'; +import { + Annotations, + Construct, + Duration, + Size, + SizeRoundingBehavior, + Stack, +} from '@aws-cdk/core'; +import { + AwsSdkCall, + AwsCustomResource, + AwsCustomResourcePolicy, + PhysicalResourceId, +} from '@aws-cdk/custom-resources'; +import { + tagConstruct, +} from './runtime-info'; + +/** + * Input properties for PadEfsStorage. + */ +export interface PadEfsStorageProps { + /** + * VPC in which the given access point is deployed. + */ + readonly vpc: IVpc; + + /** + * PadEfsStorage deploys AWS Lambda Functions that need to contact your Amazon EFS mount target(s). + * To do this, AWS Lambda creates network interfaces in these given subnets in your VPC. + * These can be any subnet(s) in your VPC that can route traffic to the EFS mount target(s). + * + * @default All private subnets + */ + readonly vpcSubnets?: SubnetSelection; + + /** + * Amazon EFS Access Point into which the filesystem padding files will be added. Files will + * be added/removed from the root directory of the given access point. + * We strongly recommend that you provide an access point that is for a dedicated padding-files + * directory in your EFS filesystem, rather than the root directory or some other in-use directory + * of the filesystem. + */ + readonly accessPoint: IAccessPoint; + /** + * Security group for the AWS Lambdas created by this construct. + * + * @default Security group with no egress or ingress will be automatically created for each Lambda. + */ + readonly securityGroup?: ISecurityGroup; + + /** + * The desired total size, in GiB, of files stored in the access point directory. + */ + readonly desiredPadding: Size; +} + +/** + * This construct provides a mechanism that adds 1GB-sized files containing only zero-bytes + * to an Amazon EFS filesystem through a given Access Point to that filesystem. This is being + * provided to give you a way to increase the baseline throughput of an Amazon EFS filesystem + * that has been deployed in bursting throughput mode (see: https://docs.aws.amazon.com/efs/latest/ug/performance.html#throughput-modes). + * This is most useful for your Amazon EFS filesystems that contain a very small amount of data and + * have a baseline throughput that exceeds the throughput provided by the size of the filesystem. + * + * When deployed in bursting throughput mode, an Amazon EFS filesystem provides you with a baseline + * throughput that is proportional to the amount of data stored in that filesystem. However, usage + * of that filesystem is allowed to burst above that throughput; doing so consumes burst credits that + * are associated with the filesystem. When all burst credits have been expended, then your filesystem + * is no longer allowed to burst throughput and you will be limited in throughput to the greater of 1MiB/s + * or the throughput dictated by the amount of data stored in your filesystem; the filesystem will be able + * to burst again if it is able to accrue burst credits by staying below its baseline throughput for a time. + * + * Customers that deploy the Deadline Repository Filesystem on an Amazon EFS filesystem may find that + * the filesystem does not contain sufficient data to meet the throughput needs of Deadline; evidenced by + * a downward trend in EFS bursting credits over time. When bursting credits are expended, then the render + * farm may begin to exhibit failure mode behaviors such as the RenderQueue dropping or refusing connections, + * or becoming unresponsive. + * + * Warning: The implementation of this construct creates and starts an AWS Step Function to add the files + * to the filesystem. The execution of this Step Function occurs asynchronously from your deployment. We recommend + * verifying that the step function completed successfully via your Step Functions console. + * + * Resources Deployed + * -------------------------- + * - Two AWS Lambda Functions, with roles, with full access to the given EFS Access Point. + * - An Elastic Network Interface (ENI) for each Lambda Function in each of the selected VPC Subnets, so + * that the Lambda Functions can connect to the given EFS Access Point. + * - An AWS Step Function to coordinate execution of the two Lambda Functions. + * - Security Groups for each AWS Lambda Function. + * - A CloudFormation custom resource that executes StepFunctions.startExecution on the Step Function + * whenever the stack containing this construct is created or updated. + * + * Security Considerations + * --------------------------- + * - The AWS Lambdas that are deployed through this construct will be created from a deployment package + * that is uploaded to your CDK bootstrap bucket during deployment. You must limit write access to + * your CDK bootstrap bucket to prevent an attacker from modifying the actions performed by these Lambdas. + * We strongly recommend that you either enable Amazon S3 server access logging on your CDK bootstrap bucket, + * or enable AWS CloudTrail on your account to assist in post-incident analysis of compromised production + * environments. + * - By default, the network interfaces created by this construct's AWS Lambda Functions have Security Groups + * that restrict egress access from the Lambda Function into your VPC such that the Lambda Functions can + * access only the given EFS Access Point. + */ +export class PadEfsStorage extends Construct { + constructor(scope: Construct, id: string, props: PadEfsStorageProps) { + super(scope, id); + + /* + Implementation: + This is implemented as an AWS Step Function that implements the following + algorithm: + try { + du = diskUsage() + while (du != desiredPadding) { + if (du < desiredPadding) { + + } else if (du > desiredPadding) { + + // Note: We break here to prevent two separate invocations of the step function (e.g. accidental manual + // invocations) from looping indefinitely. Without a break, one invocation trying to grow while another + // tries to shrink will infinitely loop both -- the diskUsage will never settle on the value that either + // invocation wants. + break; + } + du = diskUsage() + } + return success + } catch (error) { + return failure + } + */ + + const diskUsageTimeout = Duration.minutes(5); + const paddingTimeout = Duration.minutes(15); + // Location in the lambda environment where the EFS will be mounted. + const efsMountPoint = '/mnt/efs'; + + let desiredSize; + try { + desiredSize = props.desiredPadding.toGibibytes({rounding: SizeRoundingBehavior.FAIL}); + } catch (err) { + Annotations.of(this).addError('Failed to round desiredSize to an integer number of GiB. The size must be in GiB.'); + } + + const securityGroup = props.securityGroup ?? new SecurityGroup(this, 'LambdaSecurityGroup', { + vpc: props.vpc, + allowAllOutbound: false, + }); + + const lambdaProps: any = { + code: Code.fromAsset(path.join(__dirname, '..', '..', 'lambdas', 'nodejs')), + runtime: Runtime.NODEJS_14_X, + logRetention: RetentionDays.ONE_WEEK, + // Required for access point... + vpc: props.vpc, + vpcSubnets: props.vpcSubnets ?? { + subnetType: SubnetType.PRIVATE, + }, + securityGroups: [ securityGroup ], + filesystem: LambdaFilesystem.fromEfsAccessPoint(props.accessPoint, efsMountPoint), + }; + + const diskUsage = new LambdaFunction(this, 'DiskUsage', { + description: 'Used by RFDK PadEfsStorage to calculate disk usage of an EFS access point', + handler: 'pad-efs-storage.getDiskUsage', + timeout: diskUsageTimeout, + memorySize: 128, + ...lambdaProps, + }); + + const doPadding = new LambdaFunction(this, 'PadFilesystem', { + description: 'Used by RFDK PadEfsStorage to add or remove numbered 1GB files in an EFS access point', + handler: 'pad-efs-storage.padFilesystem', + timeout: paddingTimeout, + // Execution requires about 70MB for just the lambda, but the filesystem driver will use every available byte. + // Larger sizes do not seem to make a difference on filesystem write performance. + // Set to 256MB just to give a buffer. + memorySize: 256, + ...lambdaProps, + }); + + + // Build the step function's state machine. + const fail = new Fail(this, 'Fail'); + const succeed = new Succeed(this, 'Succeed'); + + const diskUsageTask = new LambdaInvoke(this, 'QueryDiskUsage', { + lambdaFunction: diskUsage, + comment: 'Determine the number of GB currently stored in the EFS access point', + timeout: diskUsageTimeout, + payload: { + type: InputType.OBJECT, + value: { + 'desiredPadding.$': '$.desiredPadding', + 'mountPoint': efsMountPoint, + }, + }, + resultPath: '$.diskUsage', + }); + + const growTask = new LambdaInvoke(this, 'GrowTask', { + lambdaFunction: doPadding, + comment: 'Add up to 20 numbered 1GB files to the EFS access point', + timeout: paddingTimeout, + payload: { + type: InputType.OBJECT, + value: { + 'desiredPadding.$': '$.desiredPadding', + 'mountPoint': efsMountPoint, + }, + }, + resultPath: '$.null', + }); + + const shrinkTask = new LambdaInvoke(this, 'ShrinkTask', { + lambdaFunction: doPadding, + comment: 'Remove 1GB numbered files from the EFS access point to shrink the padding', + timeout: paddingTimeout, + payload: { + type: InputType.OBJECT, + value: { + 'desiredPadding.$': '$.desiredPadding', + 'mountPoint': efsMountPoint, + }, + }, + resultPath: '$.null', + }); + + const choice = new Choice(this, 'BranchOnDiskUsage') + .when(Condition.numberLessThanJsonPath('$.diskUsage.Payload', '$.desiredPadding'), growTask) + .when(Condition.numberGreaterThanJsonPath('$.diskUsage.Payload', '$.desiredPadding'), shrinkTask) + .otherwise(succeed); + + diskUsageTask.next(choice); + diskUsageTask.addCatch(fail, { + // See: https://docs.aws.amazon.com/step-functions/latest/dg/concepts-error-handling.html + errors: ['States.ALL'], + }); + + growTask.next(diskUsageTask); + growTask.addCatch(fail, { + errors: [ 'States.ALL' ], + }); + + shrinkTask.next(succeed); + shrinkTask.addCatch(fail, { + errors: [ 'States.ALL' ], + }); + + const statemachine = new StateMachine(this, 'StateMachine', { + definition: diskUsageTask, + }); + + // ========== + // Invoke the step function on stack create & update. + const invokeCall: AwsSdkCall = { + action: 'startExecution', + service: 'StepFunctions', + apiVersion: '2016-11-23', + region: Stack.of(this).region, + physicalResourceId: PhysicalResourceId.fromResponse('executionArn'), + parameters: { + stateMachineArn: statemachine.stateMachineArn, + input: JSON.stringify({ + desiredPadding: desiredSize, + }), + }, + }; + + const resource = new AwsCustomResource(this, 'Default', { + installLatestAwsSdk: true, + logRetention: RetentionDays.ONE_WEEK, + onCreate: invokeCall, + onUpdate: invokeCall, + policy: AwsCustomResourcePolicy.fromSdkCalls({ + resources: [ statemachine.stateMachineArn ], + }), + }); + resource.node.addDependency(statemachine); + + // Add RFDK tags to the construct tree. + tagConstruct(this); + } +} diff --git a/packages/aws-rfdk/lib/core/test/pad-efs-storage.test.ts b/packages/aws-rfdk/lib/core/test/pad-efs-storage.test.ts new file mode 100644 index 000000000..efb46769f --- /dev/null +++ b/packages/aws-rfdk/lib/core/test/pad-efs-storage.test.ts @@ -0,0 +1,324 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + arrayWith, + expect as cdkExpect, + haveResourceLike, +} from '@aws-cdk/assert'; +import { + SecurityGroup, + Vpc, +} from '@aws-cdk/aws-ec2'; +import { + AccessPoint, + FileSystem as EfsFileSystem, +} from '@aws-cdk/aws-efs'; +import { + Function as LambdaFunction, +} from '@aws-cdk/aws-lambda'; +import { + App, + Size, + Stack, +} from '@aws-cdk/core'; +import { + PadEfsStorage, +} from '../lib/pad-efs-storage'; + +describe('Test PadEfsStorage', () => { + let app: App; + let stack: Stack; + let vpc: Vpc; + let efsFS: EfsFileSystem; + let accessPoint: AccessPoint; + + + beforeEach(() => { + app = new App(); + stack = new Stack(app); + vpc = new Vpc(stack, 'Vpc'); + efsFS = new EfsFileSystem(stack, 'EFS', { vpc }); + accessPoint = new AccessPoint(stack, 'AccessPoint', { + fileSystem: efsFS, + createAcl: { + ownerGid: '1000', + ownerUid: '1000', + permissions: '755', + }, + path: '/SpaceFillingFiles', + posixUser: { + uid: '1000', + gid: '1000', + }, + }); + }); + + test('Create with defaults', () => { + // WHEN + const pad = new PadEfsStorage(stack, 'PadEfs', { + vpc, + accessPoint, + desiredPadding: Size.gibibytes(20), + }); + const sg = pad.node.findChild('LambdaSecurityGroup') as SecurityGroup; + const diskUsage = pad.node.findChild('DiskUsage') as LambdaFunction; + const padFilesystem = pad.node.findChild('PadFilesystem') as LambdaFunction; + + // THEN + cdkExpect(stack).to(haveResourceLike('AWS::Lambda::Function', { + FileSystemConfigs: [ + { + Arn: stack.resolve(accessPoint.accessPointArn), + LocalMountPath: '/mnt/efs', + }, + ], + Handler: 'pad-efs-storage.getDiskUsage', + Runtime: 'nodejs14.x', + Timeout: 300, + VpcConfig: { + SecurityGroupIds: [ stack.resolve(sg.securityGroupId) ], + SubnetIds: [ + { + Ref: 'VpcPrivateSubnet1Subnet536B997A', + }, + { + Ref: 'VpcPrivateSubnet2Subnet3788AAA1', + }, + ], + }, + })); + cdkExpect(stack).to(haveResourceLike('AWS::Lambda::Function', { + FileSystemConfigs: [ + { + Arn: stack.resolve(accessPoint.accessPointArn), + LocalMountPath: '/mnt/efs', + }, + ], + Handler: 'pad-efs-storage.padFilesystem', + Runtime: 'nodejs14.x', + Timeout: 900, + VpcConfig: { + SecurityGroupIds: [ stack.resolve(sg.securityGroupId) ], + SubnetIds: [ + { + Ref: 'VpcPrivateSubnet1Subnet536B997A', + }, + { + Ref: 'VpcPrivateSubnet2Subnet3788AAA1', + }, + ], + }, + })); + + const lambdaRetryCatch = { + Retry: [ + { + ErrorEquals: [ + 'Lambda.ServiceException', + 'Lambda.AWSLambdaException', + 'Lambda.SdkClientException', + ], + IntervalSeconds: 2, + MaxAttempts: 6, + BackoffRate: 2, + }, + ], + Catch: [ + { + ErrorEquals: ['States.ALL'], + Next: 'Fail', + }, + ], + }; + cdkExpect(stack).to(haveResourceLike('AWS::StepFunctions::StateMachine', { + DefinitionString: stack.resolve(JSON.stringify({ + StartAt: 'QueryDiskUsage', + States: { + QueryDiskUsage: { + Next: 'BranchOnDiskUsage', + ...lambdaRetryCatch, + Type: 'Task', + Comment: 'Determine the number of GB currently stored in the EFS access point', + TimeoutSeconds: 300, + ResultPath: '$.diskUsage', + Resource: `arn:${stack.partition}:states:::lambda:invoke`, + Parameters: { + FunctionName: `${diskUsage.functionArn}`, + Payload: { + 'desiredPadding.$': '$.desiredPadding', + 'mountPoint': '/mnt/efs', + }, + }, + }, + GrowTask: { + Next: 'QueryDiskUsage', + ...lambdaRetryCatch, + Type: 'Task', + Comment: 'Add up to 20 numbered 1GB files to the EFS access point', + TimeoutSeconds: 900, + ResultPath: '$.null', + Resource: `arn:${stack.partition}:states:::lambda:invoke`, + Parameters: { + FunctionName: `${padFilesystem.functionArn}`, + Payload: { + 'desiredPadding.$': '$.desiredPadding', + 'mountPoint': '/mnt/efs', + }, + }, + }, + BranchOnDiskUsage: { + Type: 'Choice', + Choices: [ + { + Variable: '$.diskUsage.Payload', + NumericLessThanPath: '$.desiredPadding', + Next: 'GrowTask', + }, + { + Variable: '$.diskUsage.Payload', + NumericGreaterThanPath: '$.desiredPadding', + Next: 'ShrinkTask', + }, + ], + Default: 'Succeed', + }, + Succeed: { + Type: 'Succeed', + }, + ShrinkTask: { + Next: 'Succeed', + ...lambdaRetryCatch, + Type: 'Task', + Comment: 'Remove 1GB numbered files from the EFS access point to shrink the padding', + TimeoutSeconds: 900, + ResultPath: '$.null', + Resource: `arn:${stack.partition}:states:::lambda:invoke`, + Parameters: { + FunctionName: `${padFilesystem.functionArn}`, + Payload: { + 'desiredPadding.$': '$.desiredPadding', + 'mountPoint': '/mnt/efs', + }, + }, + }, + Fail: { + Type: 'Fail', + }, + }, + })), + })); + + cdkExpect(stack).to(haveResourceLike('Custom::AWS', { + Create: { + 'Fn::Join': [ + '', + [ + '{"action":"startExecution","service":"StepFunctions","apiVersion":"2016-11-23","region":"', + { + Ref: 'AWS::Region', + }, + '","physicalResourceId":{"responsePath":"executionArn"},"parameters":{"stateMachineArn":"', + { + Ref: 'PadEfsStateMachineDA538E87', + }, + '","input":"{\\"desiredPadding\\":20}"}}', + ], + ], + }, + Update: { + 'Fn::Join': [ + '', + [ + '{"action":"startExecution","service":"StepFunctions","apiVersion":"2016-11-23","region":"', + { + Ref: 'AWS::Region', + }, + '","physicalResourceId":{"responsePath":"executionArn"},"parameters":{"stateMachineArn":"', + { + Ref: 'PadEfsStateMachineDA538E87', + }, + '","input":"{\\"desiredPadding\\":20}"}}', + ], + ], + }, + })); + }); + + test('Set desiredPadding', () => { + // WHEN + const desiredPadding = 200; + new PadEfsStorage(stack, 'PadEfs', { + vpc, + accessPoint, + desiredPadding: Size.gibibytes(desiredPadding), + }); + + // THEN + cdkExpect(stack).to(haveResourceLike('Custom::AWS', { + Create: { + 'Fn::Join': [ + '', + arrayWith(`","input":"{\\"desiredPadding\\":${desiredPadding}}"}}`), + ], + }, + Update: { + 'Fn::Join': [ + '', + arrayWith(`","input":"{\\"desiredPadding\\":${desiredPadding}}"}}`), + ], + }, + })); + }); + + test('Throws on bad desiredPadding', () => { + // GIVEN + const pad = new PadEfsStorage(stack, 'PadEfs', { + vpc, + accessPoint, + desiredPadding: Size.mebibytes(100), // not rounable to GiB + }); + + // THEN + expect(pad.node.metadata).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'aws:cdk:error', + data: 'Failed to round desiredSize to an integer number of GiB. The size must be in GiB.', + }), + ]), + ); + }); + + test('Provide SecurityGroup', () => { + // GIVEN + const sg = new SecurityGroup(stack, 'TestSG', { + vpc, + }); + + // WHEN + new PadEfsStorage(stack, 'PadEfs', { + vpc, + accessPoint, + desiredPadding: Size.gibibytes(20), + securityGroup: sg, + }); + + // THEN + cdkExpect(stack).to(haveResourceLike('AWS::Lambda::Function', { + Handler: 'pad-efs-storage.getDiskUsage', + VpcConfig: { + SecurityGroupIds: [ stack.resolve(sg.securityGroupId) ], + }, + })); + cdkExpect(stack).to(haveResourceLike('AWS::Lambda::Function', { + Handler: 'pad-efs-storage.padFilesystem', + VpcConfig: { + SecurityGroupIds: [ stack.resolve(sg.securityGroupId) ], + }, + })); + }); + +}); diff --git a/packages/aws-rfdk/lib/deadline/lib/repository.ts b/packages/aws-rfdk/lib/deadline/lib/repository.ts index 5950a3cae..aca9b6004 100644 --- a/packages/aws-rfdk/lib/deadline/lib/repository.ts +++ b/packages/aws-rfdk/lib/deadline/lib/repository.ts @@ -46,6 +46,7 @@ import { Duration, IConstruct, RemovalPolicy, + Size, Stack, Tags, Token, @@ -57,6 +58,7 @@ import { LogGroupFactory, LogGroupFactoryProps, MountableEfs, + PadEfsStorage, ScriptAsset, } from '../../core'; import { @@ -298,7 +300,13 @@ export interface RepositoryProps { /** * Specify the file system where the deadline repository needs to be initialized. * - * @default An Encrypted EFS File System and Access Point will be created + * If not providing a filesystem, then we will provision an Amazon EFS filesystem for you. + * This filesystem will contain files for the Deadline Repository filesystem. It will also + * contain 40GB of additional padding files (see RFDK's PadEfsStorage for details) to increase + * the baseline throughput of the filesystem; these files will be added to the /RFDK_PaddingFiles directory + * in the filesystem. + * + * @default An Encrypted EFS File System and Access Point will be created. */ readonly fileSystem?: IMountableLinuxFilesystem; @@ -383,11 +391,12 @@ export interface RepositoryProps { * Resources Deployed * ------------------------ * - Encrypted Amazon Elastic File System (EFS) - If no file system is provided. - * - An Amazon EFS Point - If no filesystem is provided + * - An Amazon EFS Point - If no filesystem is provided. * - An Amazon DocumentDB - If no database connection is provided. * - Auto Scaling Group (ASG) with min & max capacity of 1 instance. * - Instance Role and corresponding IAM Policy. * - An Amazon CloudWatch log group that contains the Deadline Repository installation logs. + * - An RFDK PadEfsStorage - If no filesystem is provided. * * Security Considerations * ------------------------ @@ -506,6 +515,22 @@ export class Repository extends Construct implements IRepository { securityGroup: props.securityGroupsOptions?.fileSystem, }); + const paddingAccess = fs.addAccessPoint('PaddingAccessPoint', { + createAcl: { + ownerGid: '0', + ownerUid: '0', + permissions: '744', + }, + path: '/RFDK_PaddingFiles', + }); + + new PadEfsStorage(this, 'PadEfsStorage', { + vpc: props.vpc, + vpcSubnets: props.vpcSubnets ?? { subnetType: SubnetType.PRIVATE }, + accessPoint: paddingAccess, + desiredPadding: Size.gibibytes(40), + }); + const accessPoint = fs.addAccessPoint('AccessPoint', { posixUser: { uid: '0', diff --git a/packages/aws-rfdk/lib/deadline/test/repository.test.ts b/packages/aws-rfdk/lib/deadline/test/repository.test.ts index 6f6a93a48..c388f3900 100644 --- a/packages/aws-rfdk/lib/deadline/test/repository.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/repository.test.ts @@ -78,7 +78,22 @@ function escapeTokenRegex(s: string): string { beforeEach(() => { app = new App(); stack = new Stack(app, 'Stack'); - vpc = new Vpc(stack, 'VPC'); + vpc = new Vpc(stack, 'VPC', { + subnetConfiguration: [ + { + name: 'Public', + subnetType: SubnetType.PUBLIC, + }, + { + name: 'Private', + subnetType: SubnetType.PRIVATE, + }, + { + name: 'Isolated', + subnetType: SubnetType.ISOLATED, + }, + ], + }); class MockVersion extends Version implements IVersion { readonly linuxInstallers = { @@ -182,21 +197,21 @@ test('repository installer honors vpcSubnet', () => { // private subnets. // WHEN - const publicSubnetIds = [ 'PublicSubnet1', 'PublicSubnet2' ]; + const isolatedSubnetIds = [ 'IsolatedSubnet1', 'IsolatedSubnet2' ]; const attrVpc = Vpc.fromVpcAttributes(stack, 'TestVpc', { availabilityZones: ['us-east-1a', 'us-east-1b'], vpcId: 'vpcid', - publicSubnetIds, + isolatedSubnetIds, }); new Repository(stack, 'repositoryInstaller', { vpc: attrVpc, version, - vpcSubnets: { subnetType: SubnetType.PUBLIC }, + vpcSubnets: { subnetType: SubnetType.ISOLATED }, }); // THEN expectCDK(stack).to(haveResourceLike('AWS::AutoScaling::AutoScalingGroup', { - VPCZoneIdentifier: publicSubnetIds, + VPCZoneIdentifier: isolatedSubnetIds, })); }); @@ -798,7 +813,7 @@ test('repository creates filesystem if none provided', () => { } // WHEN - new Repository(stack, 'repositoryInstaller', { + const repo = new Repository(stack, 'repositoryInstaller', { vpc, database: DatabaseConnection.forDocDB({ database: fsDatabase, login: fsDatabase.secret }), version, @@ -807,6 +822,8 @@ test('repository creates filesystem if none provided', () => { // THEN expectCDK(stack).to(haveResource('AWS::EFS::FileSystem')); expectCDK(stack).to(haveResource('AWS::EFS::MountTarget')); + expect(repo.node.tryFindChild('PadEfsStorage')).toBeDefined(); + expect(repo.node.findChild('FileSystem').node.tryFindChild('PaddingAccessPoint')).toBeDefined(); }); test('default repository instance is created using user defined installation path prefix', () => { diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/filesystem-ops.ts b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/filesystem-ops.ts new file mode 100644 index 000000000..cb2b1806f --- /dev/null +++ b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/filesystem-ops.ts @@ -0,0 +1,106 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console */ + +import { + exec, +} from 'child_process'; +import { + promises as fsp, +} from 'fs'; +import { + promisify, +} from 'util'; + +/** + * Verifies that the path given in the argument exists and is a directory. + * @param location + */ +export async function ensureIsDirectory(location: string): Promise { + try { + const stat = await fsp.stat(location); + if (!stat.isDirectory()) { + throw Error(`Must be a directory: ${location}`); + } + } catch (err) { + throw Error(`No such file or directory: ${location}`); + } +} + +/** + * Given a filename that is assumed to be numeric, return the next numeric + * filename in increasing order padded out to 5 digits. + * @param filename + * @returns + */ +export function nextSequentialFile(filename: string): string { + const value = parseInt(filename); + return (value+1).toString().padStart(5, '0'); +} + +/** + * List all of the names in the given directory that are numeric. + * @param location Path of the directory to list. Assumed to be a directory. + * @returns Array of the names of numeric contents in the directory, sorted into increasing order. + */ +export async function listNumberedFiles(location: string): Promise { + const dirContents = await fsp.readdir(location); + const digitsRegex = /^\d+$/; + const numericFiles = dirContents.filter(name => digitsRegex.test(name)).sort(); + return numericFiles; +} + +/** + * Invoke "du -sh -BM" on the given location, to determine the total size in MB stored + * in the filesystem location. + * @param location Directory location. + * @returns Filesystem size under the location, in MiB. + */ +export async function diskUsage(location: string): Promise { + await ensureIsDirectory(location); + + const execPromise = promisify(exec); + const { stdout, stderr } = await execPromise(`/usr/bin/du -sh -BM ${location}`); + if (stderr) { + throw Error(stderr); + } + // stdout will be: M\t\n + const size = parseInt(stdout); + if (isNaN(size)) { + throw Error(`Unexpected error. Could not parse size of directory ${location}: ${stdout}`); + } + return size; +} + +/** + * Inspect the filenames in the given directory location, and return the next highest numeric + * filename that does not already exist. + * e.g. + * i) Empty dir -> 00000 + * ii) '00000', '00002' -> '00003' + * @param location Directory to inspect. + * @returns + */ +export async function determineNextSequentialFilename(location: string): Promise { + const numericFiles = await listNumberedFiles(location); + if (numericFiles.length == 0) { + return '00000'; + } + return nextSequentialFile(numericFiles[numericFiles.length-1]); +} + +/** + * Writes a file of zeroes to the given location. + * @param filename Name of the file to create. + * @param filesize Size of the file in MiB. Must be a multiple of 10. + */ +export async function writePaddingFile(filename: string, filesize: number): Promise { + const execPromise = promisify(exec); + const command = `/usr/bin/dd if=/dev/zero of=${filename} bs=10M count=${filesize/10}`; + console.log(`Writing ${filesize}MiB to ${filename}: ${command}`); + const { stderr } = await execPromise(command); + console.log(stderr); +} diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/handlers.ts b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/handlers.ts new file mode 100644 index 000000000..d947b6f91 --- /dev/null +++ b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/handlers.ts @@ -0,0 +1,195 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console */ + +import { + promises as fsp, +} from 'fs'; +import { + join, +} from 'path'; +import { + LambdaContext, +} from '../lib/aws-lambda'; +import { + determineNextSequentialFilename, + diskUsage, + ensureIsDirectory, + listNumberedFiles, + nextSequentialFile, + writePaddingFile, +} from './filesystem-ops'; + +/** + * Default filesize for lambda operation, in MiB. + * External code that calls this assumes that this is exactly 1GiB=1024MiB + */ +var FILESIZE: number = 1024; + +/** + * Provided solely for the purpose of testing to shrink the default file size from 1GiB + * to something smaller. + * @param filesize Desired filesize in MiB. Must be a multiple of 10 + */ +export function setDefaultFilesize(filesize: number) { + FILESIZE = filesize; +} + +/** + * Local helper to extract the desiredPadding field from the input event, and validate the input. + * @param event Event object passed to the lambda handler. + * @returns Value of desiredPadding or throws if invalid. + */ +function getDesiredPadding(event: { [key: string]: string }): number { + const desiredPadding = parseInt(event.desiredPadding); + if (isNaN(desiredPadding)) { + throw new Error(`Could not parse 'desiredPadding' field of the given event: ${event.desiredPadding}`); + } + return desiredPadding; +} + +/** + * Local helper to extract the mountPoint field from the input event, and validate the input. + * @param event Event object passed to the lambda handler. + * @returns Value of mountPoint or throws if invalid. + */ +function getMountPoint(event: { [key: string]: string }): string { + const mountPoint = event.mountPoint; + if (!mountPoint) { + throw new Error(`Invalid mount point given in event: ${mountPoint}`); + } + return mountPoint; +} + +/** + * Add numbered files (e.g. 00000, 00001) of a given size to a filesystem. + * Note: exported so that we can test it. + * @param numFilesToAdd How many numbered files to add. + * @param filesize Size, in MiB, of the files to create. + * @param mountPoint Directory in which to create the directory. + */ +export async function growFilesystem(numFilesToAdd: number, filesize: number, mountPoint: string): Promise { + // We find the highest numbered file created thus far, and start adding numbered files after it. + let filename: string = await determineNextSequentialFilename(mountPoint); + for (let i=0; i { + // Find all of the numbered "files" in the directory, and then delete the highest numbered ones until + // we've deleted as many as we need to. + const numberedFiles = await listNumberedFiles(mountPoint); + let numFilesDeleted = 0; + let index = numberedFiles.length - 1; + while (numFilesDeleted < numFilesToRemove && index >= 0) { + const filename = join(mountPoint, numberedFiles[index]); + try { + const stat = await fsp.stat(filename); + if (stat.isFile()) { + console.log(`Deleting: ${filename}`); + try { + await fsp.unlink(filename); + numFilesDeleted += 1; + } catch (err) { + console.error(`Unable to delete: ${filename} -- Error: ${err.message}`); + } + } + } catch (err) { + console.warn(`Warning: Unable to stat file '${filename}'`); + } + index -= 1; + } + console.log(`Deleted ${numFilesDeleted} numbered files`); + if (numFilesToRemove !== numFilesDeleted) { + throw Error('Could not delete the desired number of files.'); + } +} + +/** + * Lambda handler. Expected to be invoked from a step function. + * Calculates the disk size under the given directory. This is equivalent to calling: + * du -sh -BG + * @param event { "mountPoint": } + * @param context + * @returns Disk usage in GiB + */ +export async function getDiskUsage(event: { [key: string]: string }, context: LambdaContext): Promise { + console.log(`Executing event: ${JSON.stringify(event)}`); + console.log(`Context: ${JSON.stringify(context)}`); + + const mountPoint = getMountPoint(event); + + try { + // Make sure that the given directory has been mounted before continuing. + await ensureIsDirectory(mountPoint); + } catch (err) { + throw new Error(`Mount point '${mountPoint}' is not a directory. Please ensure that the EFS is mounted to this directory.`); + } + + const scaledUsage = Math.floor(await diskUsage(mountPoint) / FILESIZE); + + return scaledUsage; +} + +/** + * Lambda handler. Expected to be invoked from a step function. + * Adds or removes files from a given EFS filesystem to pad the filesystem with data. + * + * If the filesystem is padded to less than the number of desired GiB then files are added as numbered + * files 1GiB in size to the given 'mountPoint'; adding at most 20 on each invocation. + * + * If the filesystem is padded with more than the desired GiB then numbered files are removed from the + * given filesystem. Each numbered file is assumed to be exactly 1GiB in size. + * @param event { + * "desiredPadding": "", + * "mountPoint": "" + * } + * @param context + * @returns Nothing + */ +export async function padFilesystem(event: { [key: string]: string }, context: LambdaContext): Promise { + console.log(`Executing event: ${JSON.stringify(event)}`); + console.log(`Context: ${JSON.stringify(context)}`); + + const desiredPadding = getDesiredPadding(event); + const mountPoint = getMountPoint(event); + + try { + // Make sure that the given directory has been mounted before continuing. + await ensureIsDirectory(mountPoint); + } catch (err) { + throw new Error(`Mount point '${mountPoint}' is not a directory. Please ensure that the EFS is mounted to this directory.`); + } + + const scaledUsage = Math.floor(await diskUsage(mountPoint) / FILESIZE); + console.log(`Access point contains ${scaledUsage}GiB (rounded down) of data. Desired size is ${desiredPadding}GiB.`); + if (scaledUsage < desiredPadding) { + // Create files. + + // We'll be running this in a loop driven by a step function. Limit to 20GiB written per invocation, + // just to avoid any chance of hitting a lambda timeout. + const delta = Math.min(desiredPadding - scaledUsage, 20); + + console.log(`Adding ${delta}GiB of files to the filesystem to grow size.`); + await growFilesystem(delta, FILESIZE, mountPoint); + } else if (scaledUsage > desiredPadding) { + // Remove files + const delta = scaledUsage - desiredPadding; + console.log(`Removing ${delta}GiB of files from the filesystem`); + await shrinkFilesystem(delta, mountPoint); + } else { + console.log('No change to filesystem required.'); + } +} diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/index.ts b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/index.ts new file mode 100644 index 000000000..6cc85c419 --- /dev/null +++ b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/index.ts @@ -0,0 +1,9 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +export { + getDiskUsage, + padFilesystem, +} from './handlers'; diff --git a/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/test/handlers.test.ts b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/test/handlers.test.ts new file mode 100644 index 000000000..1c5263313 --- /dev/null +++ b/packages/aws-rfdk/lib/lambdas/nodejs/pad-efs-storage/test/handlers.test.ts @@ -0,0 +1,351 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console */ + +import { + exec, +} from 'child_process'; +import { + promises as fsp, +} from 'fs'; +import { + tmpdir, +} from 'os'; +import { + join, +} from 'path'; +import { + promisify, +} from 'util'; +import { + getDiskUsage, + growFilesystem, + padFilesystem, + setDefaultFilesize, + shrinkFilesystem, +} from '../handlers'; + +// Prevent log from printing during the test +console.log = jest.fn(); + +jest.setTimeout(20000); // 20s timeout, to give time to files. + +async function recursiveDeleteDirectory(location: string): Promise { + if (!location) return; + const contents: string[] = await fsp.readdir(location); + const stats = await Promise.all(contents.map(async (loc) => { + return await fsp.stat(join(location, loc)); + })); + const files = contents.filter((_, i) => stats[i].isFile()); + const directories = contents.filter((_, i) => stats[i].isDirectory()); + + await Promise.all(files.map(async(loc) => fsp.unlink(join(location, loc)))); + await Promise.all(directories.map(async(loc) => recursiveDeleteDirectory(join(location, loc)))); + await fsp.rmdir(location); +} + +describe('Testing filesystem modifications', () => { + + var tempDirectory: string; + + beforeEach(async () => { + // Create a temp directory for putting files. + tempDirectory = await fsp.mkdtemp(join(tmpdir(), 'tmp.')); + }); + afterEach(async () => { + await recursiveDeleteDirectory(tempDirectory); + tempDirectory = ''; + }); + + test('Add to empty directory', async () => { + // WHEN + // Add 5 10MB files to the temp directory. + await growFilesystem(5, 10, tempDirectory); + + // THEN + const dirContents = (await fsp.readdir(tempDirectory)).sort(); + expect(dirContents).toEqual(['00000', '00001', '00002', '00003', '00004']); + for (var file of dirContents) { + const stat = await fsp.stat(join(tempDirectory, file)); + expect(stat.size).toBe(10485760); + } + }); + + test('Append to directory', async () => { + // GIVEN + for (var i=4; i<8; i++) { + const filename = join(tempDirectory, i.toString()); + await fsp.writeFile(filename, 'Some data'); + } + + // WHEN + // Add 2 10MB files to the temp directory. + await growFilesystem(2, 10, tempDirectory); + + // THEN + // Make sure that the files that we added started numbering at 8 + const dirContents = (await fsp.readdir(tempDirectory)).sort(); + expect(dirContents).toEqual(['00008', '00009', '4', '5', '6', '7']); + }); + + test('Delete from directory', async () => { + // GIVEN + for (var i=1; i<8; i++) { + const filename = join(tempDirectory, i.toString().padStart(3, '0')); + await fsp.writeFile(filename, 'Some data'); + } + const preDirContents = (await fsp.readdir(tempDirectory)).sort(); + + // WHEN + // Remove two files from the filesystem + await shrinkFilesystem(2, tempDirectory); + + // THEN + const dirContents = (await fsp.readdir(tempDirectory)).sort(); + expect(preDirContents).toEqual(['001', '002', '003', '004', '005', '006', '007']); + expect(dirContents).toEqual(['001', '002', '003', '004', '005']); + }); +}); + +describe('Testing getDiskUsage behavior', () => { + var tempDirectory: string; + + beforeEach(async () => { + // Create a temp directory for putting files. + tempDirectory = await fsp.mkdtemp(join(tmpdir(), 'tmp.')); + }); + afterEach(async () => { + await recursiveDeleteDirectory(tempDirectory); + tempDirectory = ''; + }); + + test('Throws when no mountPoint', async () => { + await expect(() => getDiskUsage({ + }, + { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Throws mountPoint does not exist', async () => { + await expect(() => getDiskUsage({ + mountPoint: join(tempDirectory, 'does_not_exist'), + }, + { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Throws when mountPoint not a directory', async () => { + // WHEN + const filename = join(tempDirectory, '001'); + await fsp.writeFile(filename, 'Some data'); + + // THEN + await expect(() => getDiskUsage({ + mountPoint: filename, + }, + { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Correctly calculates disk usage', async () => { + // GIVEN + + // This overrides the default padding file size to 10MB from 1000MB. Keep this in mind when interpreting the test. + // All of the interface points are phrased in terms of 1GB files, but this little hack changes the semantics of those + // to be phrased in terms of 10MB files. + setDefaultFilesize(10); + + const execPromise = promisify(exec); + await execPromise(`/usr/bin/dd if=/dev/zero of=${join(tempDirectory, 'file1.tmp')} bs=10MB count=1`); + await fsp.mkdir(join(tempDirectory, 'subdir')); + await execPromise(`/usr/bin/dd if=/dev/zero of=${join(tempDirectory, 'subdir', 'file2.tmp')} bs=10MB count=1`); + + // WHEN + const usage = await getDiskUsage({ + mountPoint: tempDirectory, + }, + { + logGroupName: '', + logStreamName: '', + }); + + // THEN + expect(usage).toBe(2); + }); + +}); + +describe('Testing padFilesystem macro behavior', () => { + + var tempDirectory: string; + + beforeEach(async () => { + // Create a temp directory for putting files. + tempDirectory = await fsp.mkdtemp(join(tmpdir(), 'tmp.')); + }); + afterEach(async () => { + await recursiveDeleteDirectory(tempDirectory); + tempDirectory = ''; + }); + + test('Throws when no desiredPadding', async () => { + await expect(() => padFilesystem({ + mountPoint: tempDirectory, + }, { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Throws desiredPadding not number', async () => { + await expect(() => padFilesystem({ + desiredPadding: 'one hundred', + mountPoint: tempDirectory, + }, { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Throws when no mountPoint', async () => { + await expect(() => padFilesystem({ + desiredPadding: '2', + }, { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Throws mountPoint does not exist', async () => { + await expect(() => padFilesystem({ + desiredPadding: '2', + mountPoint: join(tempDirectory, 'does_not_exist'), + }, { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Throws when mountPoint not a directory', async () => { + // WHEN + const filename = join(tempDirectory, '001'); + await fsp.writeFile(filename, 'Some data'); + + // THEN + await expect(() => padFilesystem({ + desiredPadding: '2', + mountPoint: filename, + }, { + logGroupName: '', + logStreamName: '', + })).rejects.toThrow(); + }); + + test('Adds file if needed', async () => { + // GIVEN + // Empty directory: tempDirectory + + // This overrides the default padding file size to 10MB from 1000MB. Keep this in mind when interpreting the test. + // All of the interface points are phrased in terms of 1GB files, but this little hack changes the semantics of those + // to be phrased in terms of 10MB files. + setDefaultFilesize(10); + + // WHEN + await padFilesystem({ + desiredPadding: '1', + mountPoint: tempDirectory, + }, { + logGroupName: '', + logStreamName: '', + }); + + // THEN + const dirContents = (await fsp.readdir(tempDirectory)).sort(); + expect(dirContents).toEqual(['00000']); + for (var file of dirContents) { + const stat = await fsp.stat(join(tempDirectory, file)); + expect(stat.size).toBe(10485760); + } + }); + + test('Removes file if needed', async () => { + // GIVEN + // This overrides the default padding file size to 10MB from 1000MB. Keep this in mind when interpreting the test. + // All of the interface points are phrased in terms of 1GB files, but this little hack changes the semantics of those + // to be phrased in terms of 10MB files. + setDefaultFilesize(10); + // tempDirectory with 2 10MB files in it + await padFilesystem({ + desiredPadding: '2', + mountPoint: tempDirectory, + }, { + logGroupName: '', + logStreamName: '', + }); + + // WHEN + const preDirContents = (await fsp.readdir(tempDirectory)).sort(); + // Desire to shrink down to 1 file + await padFilesystem({ + desiredPadding: '1', + mountPoint: tempDirectory, + }, { + logGroupName: '', + logStreamName: '', + }); + + // THEN + const dirContents = (await fsp.readdir(tempDirectory)).sort(); + expect(preDirContents).toEqual(['00000', '00001']); + expect(dirContents).toEqual(['00000']); + for (var file of dirContents) { + const stat = await fsp.stat(join(tempDirectory, file)); + expect(stat.size).toBe(10485760); + } + }); + + test('No change to filesystem', async () => { + // GIVEN + // This overrides the default padding file size to 10MB from 1000MB. Keep this in mind when interpreting the test. + // All of the interface points are phrased in terms of 1GB files, but this little hack changes the semantics of those + // to be phrased in terms of 10MB files. + setDefaultFilesize(10); + // tempDirectory with a 10MB file in it + await padFilesystem({ + desiredPadding: '1', + mountPoint: tempDirectory, + }, { + logGroupName: '', + logStreamName: '', + }); + + // WHEN + const preDirContents = (await fsp.readdir(tempDirectory)).sort(); + // Desire for 10MB of files + await padFilesystem({ + desiredPadding: '1', + mountPoint: tempDirectory, + }, { + logGroupName: '', + logStreamName: '', + }); + + // THEN + const dirContents = (await fsp.readdir(tempDirectory)).sort(); + expect(preDirContents).toEqual(['00000']); + expect(dirContents).toEqual(preDirContents); + for (var file of dirContents) { + const stat = await fsp.stat(join(tempDirectory, file)); + expect(stat.size).toBe(10485760); + } + }); +}); diff --git a/packages/aws-rfdk/package.json b/packages/aws-rfdk/package.json index 94a8d5888..0f1c6ea60 100644 --- a/packages/aws-rfdk/package.json +++ b/packages/aws-rfdk/package.json @@ -78,6 +78,8 @@ "dependencies": { "@aws-cdk/assets": "1.94.1", "@aws-cdk/aws-apigateway": "1.94.1", + "@aws-cdk/aws-apigatewayv2": "1.94.1", + "@aws-cdk/aws-apigatewayv2-integrations": "1.94.1", "@aws-cdk/aws-applicationautoscaling": "1.94.1", "@aws-cdk/aws-autoscaling": "1.94.1", "@aws-cdk/aws-autoscaling-common": "1.94.1", @@ -93,6 +95,7 @@ "@aws-cdk/aws-codeguruprofiler": "1.94.1", "@aws-cdk/aws-codepipeline": "1.94.1", "@aws-cdk/aws-cognito": "1.94.1", + "@aws-cdk/aws-databrew": "1.94.1", "@aws-cdk/aws-docdb": "1.94.1", "@aws-cdk/aws-dynamodb": "1.94.1", "@aws-cdk/aws-ec2": "1.94.1", @@ -101,12 +104,16 @@ "@aws-cdk/aws-ecs": "1.94.1", "@aws-cdk/aws-ecs-patterns": "1.94.1", "@aws-cdk/aws-efs": "1.94.1", + "@aws-cdk/aws-eks": "1.94.1", "@aws-cdk/aws-elasticloadbalancing": "1.94.1", "@aws-cdk/aws-elasticloadbalancingv2": "1.94.1", "@aws-cdk/aws-events": "1.94.1", "@aws-cdk/aws-events-targets": "1.94.1", + "@aws-cdk/aws-globalaccelerator": "1.94.1", + "@aws-cdk/aws-glue": "1.94.1", "@aws-cdk/aws-iam": "1.94.1", "@aws-cdk/aws-kinesis": "1.94.1", + "@aws-cdk/aws-kinesisfirehose": "1.94.1", "@aws-cdk/aws-kms": "1.94.1", "@aws-cdk/aws-lambda": "1.94.1", "@aws-cdk/aws-logs": "1.94.1", @@ -117,21 +124,27 @@ "@aws-cdk/aws-sam": "1.94.1", "@aws-cdk/aws-secretsmanager": "1.94.1", "@aws-cdk/aws-servicediscovery": "1.94.1", + "@aws-cdk/aws-signer": "1.94.1", "@aws-cdk/aws-sns": "1.94.1", "@aws-cdk/aws-sns-subscriptions": "1.94.1", "@aws-cdk/aws-sqs": "1.94.1", "@aws-cdk/aws-ssm": "1.94.1", "@aws-cdk/aws-stepfunctions": "1.94.1", + "@aws-cdk/aws-stepfunctions-tasks": "1.94.1", "@aws-cdk/cloud-assembly-schema": "1.94.1", "@aws-cdk/core": "1.94.1", "@aws-cdk/custom-resources": "1.94.1", "@aws-cdk/cx-api": "1.94.1", + "@aws-cdk/lambda-layer-awscli": "1.94.1", + "@aws-cdk/lambda-layer-kubectl": "1.94.1", "@aws-cdk/region-info": "1.94.1", "constructs": "^3.2.0" }, "peerDependencies": { "@aws-cdk/assets": "1.94.1", "@aws-cdk/aws-apigateway": "1.94.1", + "@aws-cdk/aws-apigatewayv2": "1.94.1", + "@aws-cdk/aws-apigatewayv2-integrations": "1.94.1", "@aws-cdk/aws-applicationautoscaling": "1.94.1", "@aws-cdk/aws-autoscaling": "1.94.1", "@aws-cdk/aws-autoscaling-common": "1.94.1", @@ -147,6 +160,7 @@ "@aws-cdk/aws-codeguruprofiler": "1.94.1", "@aws-cdk/aws-codepipeline": "1.94.1", "@aws-cdk/aws-cognito": "1.94.1", + "@aws-cdk/aws-databrew": "1.94.1", "@aws-cdk/aws-docdb": "1.94.1", "@aws-cdk/aws-dynamodb": "1.94.1", "@aws-cdk/aws-ec2": "1.94.1", @@ -155,12 +169,16 @@ "@aws-cdk/aws-ecs": "1.94.1", "@aws-cdk/aws-ecs-patterns": "1.94.1", "@aws-cdk/aws-efs": "1.94.1", + "@aws-cdk/aws-eks": "1.94.1", "@aws-cdk/aws-elasticloadbalancing": "1.94.1", "@aws-cdk/aws-elasticloadbalancingv2": "1.94.1", "@aws-cdk/aws-events": "1.94.1", "@aws-cdk/aws-events-targets": "1.94.1", + "@aws-cdk/aws-globalaccelerator": "1.94.1", + "@aws-cdk/aws-glue": "1.94.1", "@aws-cdk/aws-iam": "1.94.1", "@aws-cdk/aws-kinesis": "1.94.1", + "@aws-cdk/aws-kinesisfirehose": "1.94.1", "@aws-cdk/aws-kms": "1.94.1", "@aws-cdk/aws-lambda": "1.94.1", "@aws-cdk/aws-logs": "1.94.1", @@ -171,15 +189,19 @@ "@aws-cdk/aws-sam": "1.94.1", "@aws-cdk/aws-secretsmanager": "1.94.1", "@aws-cdk/aws-servicediscovery": "1.94.1", + "@aws-cdk/aws-signer": "1.94.1", "@aws-cdk/aws-sns": "1.94.1", "@aws-cdk/aws-sns-subscriptions": "1.94.1", "@aws-cdk/aws-sqs": "1.94.1", "@aws-cdk/aws-ssm": "1.94.1", "@aws-cdk/aws-stepfunctions": "1.94.1", + "@aws-cdk/aws-stepfunctions-tasks": "1.94.1", "@aws-cdk/cloud-assembly-schema": "1.94.1", "@aws-cdk/core": "1.94.1", "@aws-cdk/custom-resources": "1.94.1", "@aws-cdk/cx-api": "1.94.1", + "@aws-cdk/lambda-layer-awscli": "1.94.1", + "@aws-cdk/lambda-layer-kubectl": "1.94.1", "@aws-cdk/region-info": "1.94.1", "constructs": "^3.2.0" }, diff --git a/yarn.lock b/yarn.lock index be90853ef..58c7529fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -41,6 +41,20 @@ "@aws-cdk/cx-api" "1.94.1" constructs "^3.2.0" +"@aws-cdk/aws-apigatewayv2-integrations@1.94.1": + version "1.94.1" + resolved "https://registry.yarnpkg.com/@aws-cdk/aws-apigatewayv2-integrations/-/aws-apigatewayv2-integrations-1.94.1.tgz#deddd4024be8689586c369d8cd0666243b35da8b" + integrity sha512-ITgawQAA78jQxyS1sqVYyLmV6uF/RbSoQE33ExN/Qsvj6ielcWta5UnBOEHkBBXgtJr8EBqY8EMMEkk3el3dFA== + dependencies: + "@aws-cdk/aws-apigatewayv2" "1.94.1" + "@aws-cdk/aws-ec2" "1.94.1" + "@aws-cdk/aws-elasticloadbalancingv2" "1.94.1" + "@aws-cdk/aws-iam" "1.94.1" + "@aws-cdk/aws-lambda" "1.94.1" + "@aws-cdk/aws-servicediscovery" "1.94.1" + "@aws-cdk/core" "1.94.1" + constructs "^3.2.0" + "@aws-cdk/aws-apigatewayv2@1.94.1": version "1.94.1" resolved "https://registry.yarnpkg.com/@aws-cdk/aws-apigatewayv2/-/aws-apigatewayv2-1.94.1.tgz#e1172898fc65ddaaa9add2aa595ef4ab6803030e" @@ -244,6 +258,13 @@ constructs "^3.2.0" punycode "^2.1.1" +"@aws-cdk/aws-databrew@1.94.1": + version "1.94.1" + resolved "https://registry.yarnpkg.com/@aws-cdk/aws-databrew/-/aws-databrew-1.94.1.tgz#00170b4dfb028a06b20547c49efd52352a4938f9" + integrity sha512-WKN17Wraywr4DkR95QA7UAkMZYm6s6DP8HOofn/W0LPcVSvMF4tdU/9U8Shkd7cv/8WimjLJ/dR1PrYefPxJCw== + dependencies: + "@aws-cdk/core" "1.94.1" + "@aws-cdk/aws-docdb@1.94.1": version "1.94.1" resolved "https://registry.yarnpkg.com/@aws-cdk/aws-docdb/-/aws-docdb-1.94.1.tgz#f74f739a5e4b2ec9d6cbf58e649ae197e4592195" @@ -377,6 +398,24 @@ "@aws-cdk/cx-api" "1.94.1" constructs "^3.2.0" +"@aws-cdk/aws-eks@1.94.1": + version "1.94.1" + resolved "https://registry.yarnpkg.com/@aws-cdk/aws-eks/-/aws-eks-1.94.1.tgz#efaf3908a0f97a60d2ae789debd6f3ef7823588e" + integrity sha512-u+becNyCM3+km0II/XB/xpR9CX0E7v1etPafyItgdsSS7Pvt+NGlFKPHj5T7BCVKkwGEKtcbPl7MWfVtzX/aQQ== + dependencies: + "@aws-cdk/aws-autoscaling" "1.94.1" + "@aws-cdk/aws-ec2" "1.94.1" + "@aws-cdk/aws-iam" "1.94.1" + "@aws-cdk/aws-kms" "1.94.1" + "@aws-cdk/aws-lambda" "1.94.1" + "@aws-cdk/aws-ssm" "1.94.1" + "@aws-cdk/core" "1.94.1" + "@aws-cdk/custom-resources" "1.94.1" + "@aws-cdk/lambda-layer-awscli" "1.94.1" + "@aws-cdk/lambda-layer-kubectl" "1.94.1" + constructs "^3.2.0" + yaml "1.10.0" + "@aws-cdk/aws-elasticloadbalancing@1.94.1": version "1.94.1" resolved "https://registry.yarnpkg.com/@aws-cdk/aws-elasticloadbalancing/-/aws-elasticloadbalancing-1.94.1.tgz#2e694a971c8c7c6710dbe1abb30df0ad92aea52f" @@ -446,6 +485,18 @@ "@aws-cdk/custom-resources" "1.94.1" constructs "^3.2.0" +"@aws-cdk/aws-glue@1.94.1": + version "1.94.1" + resolved "https://registry.yarnpkg.com/@aws-cdk/aws-glue/-/aws-glue-1.94.1.tgz#a575c32dad4af591d6cb69fad4075e62c4622f7c" + integrity sha512-dKAKInhO5mutWoTyYQ7meYyVDXLPhJQJJ2JR2gHvdfFISID3S0mmIgNrjsVgl6RZE90NAZzFGwdB+a1XUN9Rpw== + dependencies: + "@aws-cdk/aws-ec2" "1.94.1" + "@aws-cdk/aws-iam" "1.94.1" + "@aws-cdk/aws-kms" "1.94.1" + "@aws-cdk/aws-s3" "1.94.1" + "@aws-cdk/core" "1.94.1" + constructs "^3.2.0" + "@aws-cdk/aws-iam@1.94.1": version "1.94.1" resolved "https://registry.yarnpkg.com/@aws-cdk/aws-iam/-/aws-iam-1.94.1.tgz#c3e2fc3d3e61b0630627e5dee97393f9036d26db" @@ -673,6 +724,35 @@ "@aws-cdk/core" "1.94.1" constructs "^3.2.0" +"@aws-cdk/aws-stepfunctions-tasks@1.94.1": + version "1.94.1" + resolved "https://registry.yarnpkg.com/@aws-cdk/aws-stepfunctions-tasks/-/aws-stepfunctions-tasks-1.94.1.tgz#e638904f2fbf62a06129244651d7b3077603e4a3" + integrity sha512-NoVURt0yUfpHOiC/immtKFdWO5K2KYw/ygNDu5QF5pJ1mT+hQUzByUY5S4UZbJj07YTF0TaBV2YeSi5h0F9VMw== + dependencies: + "@aws-cdk/aws-apigateway" "1.94.1" + "@aws-cdk/aws-apigatewayv2" "1.94.1" + "@aws-cdk/aws-apigatewayv2-integrations" "1.94.1" + "@aws-cdk/aws-batch" "1.94.1" + "@aws-cdk/aws-cloudwatch" "1.94.1" + "@aws-cdk/aws-codebuild" "1.94.1" + "@aws-cdk/aws-databrew" "1.94.1" + "@aws-cdk/aws-dynamodb" "1.94.1" + "@aws-cdk/aws-ec2" "1.94.1" + "@aws-cdk/aws-ecr" "1.94.1" + "@aws-cdk/aws-ecr-assets" "1.94.1" + "@aws-cdk/aws-ecs" "1.94.1" + "@aws-cdk/aws-eks" "1.94.1" + "@aws-cdk/aws-glue" "1.94.1" + "@aws-cdk/aws-iam" "1.94.1" + "@aws-cdk/aws-kms" "1.94.1" + "@aws-cdk/aws-lambda" "1.94.1" + "@aws-cdk/aws-s3" "1.94.1" + "@aws-cdk/aws-sns" "1.94.1" + "@aws-cdk/aws-sqs" "1.94.1" + "@aws-cdk/aws-stepfunctions" "1.94.1" + "@aws-cdk/core" "1.94.1" + constructs "^3.2.0" + "@aws-cdk/aws-stepfunctions@1.94.1": version "1.94.1" resolved "https://registry.yarnpkg.com/@aws-cdk/aws-stepfunctions/-/aws-stepfunctions-1.94.1.tgz#d366f6f284f0efe4b4b4f2689092aa8bd4715a9d" @@ -749,6 +829,24 @@ "@aws-cdk/cloud-assembly-schema" "1.94.1" semver "^7.3.4" +"@aws-cdk/lambda-layer-awscli@1.94.1": + version "1.94.1" + resolved "https://registry.yarnpkg.com/@aws-cdk/lambda-layer-awscli/-/lambda-layer-awscli-1.94.1.tgz#1bcd876e7f2896fba1c1e2e5cace3d8fe2c366f5" + integrity sha512-rgTnYPeOzsbPX23EgoBPIDA3yqBLIinlcLK2tSOWElEn0D4iWKlFgoRNOAYtGsG6cBbKJpq/tc9w+DOuNjEY/g== + dependencies: + "@aws-cdk/aws-lambda" "1.94.1" + "@aws-cdk/core" "1.94.1" + constructs "^3.2.0" + +"@aws-cdk/lambda-layer-kubectl@1.94.1": + version "1.94.1" + resolved "https://registry.yarnpkg.com/@aws-cdk/lambda-layer-kubectl/-/lambda-layer-kubectl-1.94.1.tgz#7e9e946f3e48025c7f843df4d316f776cb72f61f" + integrity sha512-iq7XyykV0zhIPXQBI5HU01Kfyvzea45Vxo0xVsaZNurDF82AinvNlitYSucSNCptcbL+PoMAr5y9a20yQuMF5A== + dependencies: + "@aws-cdk/aws-lambda" "1.94.1" + "@aws-cdk/core" "1.94.1" + constructs "^3.2.0" + "@aws-cdk/region-info@1.94.1": version "1.94.1" resolved "https://registry.yarnpkg.com/@aws-cdk/region-info/-/region-info-1.94.1.tgz#d142a87be0d84ad4824f3191cd5359fb6f7d79fe"