Skip to content

Commit

Permalink
Allow the bundler to re-use assets aws#8882
Browse files Browse the repository at this point in the history
  • Loading branch information
@misterjoshua
misterjoshua committed Jul 6, 2020
1 parent 755648a commit cee16cf
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 17 deletions.
37 changes: 29 additions & 8 deletions packages/@aws-cdk/core/lib/asset-staging.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,15 @@ export class AssetStaging extends Construct {
this.fingerprintOptions = props;

if (props.bundling) {
this.bundleDir = this.bundle(props.bundling);
}
const assetHashIfKnownInAdvance = props.assetHashType !== AssetHashType.BUNDLE
? this.calculateHash(props)
: undefined;

this.assetHash = this.calculateHash(props);
this.bundleDir = this.bundle(props.bundling, assetHashIfKnownInAdvance);
this.assetHash = assetHashIfKnownInAdvance || this.calculateHash(props);
} else {
this.assetHash = this.calculateHash(props);
}

const stagingDisabled = this.node.tryGetContext(cxapi.DISABLE_ASSET_STAGING_CONTEXT);
if (stagingDisabled) {
Expand Down Expand Up @@ -137,15 +142,31 @@ export class AssetStaging extends Construct {
}
}

private bundle(options: BundlingOptions): string {
private bundle(options: BundlingOptions, assetHash?: string): string {
// Temp staging directory in the working directory
const stagingTmp = path.join('.', STAGING_TMP);
fs.ensureDirSync(stagingTmp);

// Create temp directory for bundling inside the temp staging directory
const bundleDir = path.resolve(fs.mkdtempSync(path.join(stagingTmp, 'asset-bundle-')));
// Chmod the bundleDir to full access.
fs.chmodSync(bundleDir, 0o777);
let bundleDir = '';
if (assetHash !== undefined) {
// When an asset hash is known in advance of bundling, bundling is done into a dedicated staging directory.
bundleDir = path.join(stagingTmp, 'asset-bundle-hash-' + assetHash);

if (fs.existsSync(bundleDir)) {
// Pre-existing bundle directory. The bundle has already been generated once before, so lets provide it
// as-is to the caller.
return bundleDir;
}

fs.ensureDirSync(bundleDir);
} else {
// When the asset hash isn't known in advance, bundling is done into a temporary staging directory.

// Create temp directory for bundling inside the temp staging directory
bundleDir = path.resolve(fs.mkdtempSync(path.join(stagingTmp, 'asset-bundle-temp-')));
// Chmod the bundleDir to full access.
fs.chmodSync(bundleDir, 0o777);
}

let user: string;
if (options.user) {
Expand Down
1 change: 1 addition & 0 deletions packages/@aws-cdk/core/test/docker-stub.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ set -euo pipefail
# `/tmp/docker-stub.input` and accepts one of 3 commands that impact it's
# behavior.

echo "$@" >> /tmp/docker-stub.input.concat
echo "$@" > /tmp/docker-stub.input

if echo "$@" | grep "DOCKER_STUB_SUCCESS_NO_OUTPUT"; then
Expand Down
99 changes: 90 additions & 9 deletions packages/@aws-cdk/core/test/test.staging.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import * as sinon from 'sinon';
import { App, AssetHashType, AssetStaging, BundlingDockerImage, Stack } from '../lib';

const STUB_INPUT_FILE = '/tmp/docker-stub.input';
const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat';
const STAGING_TMP_DIRECTORY = path.join('.', '.cdk.staging');

enum DockerStubCommand {
SUCCESS = 'DOCKER_STUB_SUCCESS',
Expand All @@ -26,6 +28,12 @@ export = {
if (fs.existsSync(STUB_INPUT_FILE)) {
fs.unlinkSync(STUB_INPUT_FILE);
}
if (fs.existsSync(STUB_INPUT_CONCAT_FILE)) {
fs.unlinkSync(STUB_INPUT_CONCAT_FILE);
}
if (fs.existsSync(STAGING_TMP_DIRECTORY)) {
fs.removeSync(STAGING_TMP_DIRECTORY);
}
cb();
sinon.restore();
},
Expand Down Expand Up @@ -106,8 +114,6 @@ export = {
const stack = new Stack(app, 'stack');
const directory = path.join(__dirname, 'fs', 'fixtures', 'test1');
const ensureDirSyncSpy = sinon.spy(fs, 'ensureDirSync');
const mkdtempSyncSpy = sinon.spy(fs, 'mkdtempSync');
const chmodSyncSpy = sinon.spy(fs, 'chmodSync');

// WHEN
new AssetStaging(stack, 'Asset', {
Expand All @@ -133,10 +139,73 @@ export = {
]);

// asset is bundled in a directory inside .cdk.staging
const stagingTmp = path.join('.', '.cdk.staging');
test.ok(ensureDirSyncSpy.calledWith(stagingTmp));
test.ok(mkdtempSyncSpy.calledWith(sinon.match(path.join(stagingTmp, 'asset-bundle-'))));
test.ok(chmodSyncSpy.calledWith(sinon.match(path.join(stagingTmp, 'asset-bundle-')), 0o777));
test.ok(ensureDirSyncSpy.calledWith(STAGING_TMP_DIRECTORY));
test.ok(ensureDirSyncSpy.calledWith(path.join(STAGING_TMP_DIRECTORY, 'asset-bundle-hash-2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00')));

test.done();
},

'with bundling and non-bundle asset hash type'(test: Test) {
// GIVEN
const app = new App();
const stack = new Stack(app, 'stack');
const directory = path.join(__dirname, 'fs', 'fixtures', 'test1');
const ensureDirSyncSpy = sinon.spy(fs, 'ensureDirSync');

// WHEN
new AssetStaging(stack, 'Asset', {
sourcePath: directory,
bundling: {
image: BundlingDockerImage.fromRegistry('alpine'),
command: [ DockerStubCommand.SUCCESS ],
},
});

new AssetStaging(stack, 'AssetDuplicate', {
sourcePath: directory,
bundling: {
image: BundlingDockerImage.fromRegistry('alpine'),
command: [ DockerStubCommand.SUCCESS ],
},
});

// THEN
app.synth();

// We're testing that docker was run exactly once even though there are two bundling assets.
test.deepEqual(
readDockerStubInputConcat(),
`run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated -w /asset-input alpine DOCKER_STUB_SUCCESS`,
);

// asset is bundled in a directory inside .cdk.staging
test.ok(ensureDirSyncSpy.calledWith(STAGING_TMP_DIRECTORY));
test.ok(ensureDirSyncSpy.calledWith(path.join(STAGING_TMP_DIRECTORY, 'asset-bundle-hash-2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00')));

test.done();
},

'with bundling and bundle asset hash type'(test: Test) {
// GIVEN
const app = new App();
const stack = new Stack(app, 'stack');
const directory = path.join(__dirname, 'fs', 'fixtures', 'test1');
const mkdtempSyncSpy = sinon.spy(fs, 'mkdtempSync');
const chmodSyncSpy = sinon.spy(fs, 'chmodSync');

// WHEN
new AssetStaging(stack, 'Asset', {
sourcePath: directory,
assetHashType: AssetHashType.BUNDLE,
bundling: {
image: BundlingDockerImage.fromRegistry('alpine'),
command: [ DockerStubCommand.SUCCESS ],
},
});

// THEN
test.ok(mkdtempSyncSpy.calledWith(sinon.match(path.join(STAGING_TMP_DIRECTORY, 'asset-bundle-temp-'))));
test.ok(chmodSyncSpy.calledWith(sinon.match(path.join(STAGING_TMP_DIRECTORY, 'asset-bundle-temp-')), 0o777));

test.done();
},
Expand Down Expand Up @@ -273,6 +342,7 @@ export = {
// THEN
test.throws(() => new AssetStaging(stack, 'Asset', {
sourcePath: directory,
assetHashType: AssetHashType.BUNDLE,
bundling: {
image: BundlingDockerImage.fromRegistry('this-is-an-invalid-docker-image'),
command: [ DockerStubCommand.FAIL ],
Expand All @@ -287,9 +357,20 @@ export = {
},
};

function readDockerStubInput() {
const out = fs.readFileSync(STUB_INPUT_FILE, 'utf-8').trim();
return out
// Reads a docker stub and cleans the volume paths out of the stub.
function readAndCleanDockerStubInput(file: string) {
return fs
.readFileSync(file, 'utf-8')
.trim()
.replace(/-v ([^:]+):\/asset-input/, '-v /input:/asset-input')
.replace(/-v ([^:]+):\/asset-output/, '-v /output:/asset-output');
}

// Last docker input since last teardown
function readDockerStubInput() {
return readAndCleanDockerStubInput(STUB_INPUT_FILE);
}
// Concatenated docker inputs since last teardown
function readDockerStubInputConcat() {
return readAndCleanDockerStubInput(STUB_INPUT_CONCAT_FILE);
}

0 comments on commit cee16cf

Please sign in to comment.