Failing to build lambda image in CI

[broom9]

Hi there,

We're using this lib to copy an image from the CDK ECR repo into a repo we control as suggested here https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecr_assets-readme.html#publishing-images-to-ecr-repositories

The codes look like below:

 const image = new DockerImageAsset(this, 'FmApiImage', {
        directory: path.join(__dirname, '../..'),
        buildArgs: {
          BUILD_STAMP: buildId,
        },
      });
      new ecrdeploy.ECRDeployment(this, 'FmApiImageCopy', {
        src: new ecrdeploy.DockerImageName(image.imageUri),
        dest: new ecrdeploy.DockerImageName(`${this.account}.dkr.ecr.us-east-1.amazonaws.com/fm-api:${buildId}`),
      });

We started seeing intermittent failures on CI pipelines in recent days as below. This library builds the Docker image from scratch when the CI env var is present. Looks like it also runs some tests when building the Docker image, and one of the tests failed.

Is it possible to allow using a pre-built image in a CI environment? The env var CI affects lots of other things so I don't want to turn it off in a CI environment.

I'm also not sure if it's necessary to run these tests when building the docker image, or if the tests are flaky for some reason.

Thanks.

 ---> Running in 737dc92d52a2
CGO_ENABLED=0 CGO_CFLAGS="" CGO_LDFLAGS="-L/usr/lib64 -lgpgme" go test -v -tags "exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp" ./...
=== RUN   TestMain
    main_test.go:19: 
--- SKIP: TestMain (0.00s)
=== RUN   TestGetECRRegion
--- PASS: TestGetECRRegion (0.00s)
=== RUN   TestGetCredsType
--- PASS: TestGetCredsType (0.00s)
PASS
ok  	cdk-ecr-deployment-handler	0.019s
=== RUN   TestReadAtMost
--- PASS: TestReadAtMost (0.00s)
PASS
ok  	cdk-ecr-deployment-handler/internal/iolimits	0.005s
=== RUN   TestNewS3FileReader
    reader_test.go:16: 
--- SKIP: TestNewS3FileReader (0.00s)
=== RUN   TestNewS3File
    s3file_test.go:20: 
--- SKIP: TestNewS3File (0.00s)
=== RUN   TestBlockAddressTranslation
--- PASS: TestBlockAddressTranslation (0.00s)
=== RUN   TestBlockCache
--- PASS: TestBlockCache (0.06s)
=== RUN   TestLRUBlockPool
    s3file_test.go:115: 
        	Error Trace:	s3file_test.go:115
        	Error:      	Not equal: 
        	            	expected: 0x41
        	            	actual  : 0x0
        	Test:       	TestLRUBlockPool
--- FAIL: TestLRUBlockPool (0.01s)
FAIL
FAIL	cdk-ecr-deployment-handler/internal/tarfile	0.073s
=== RUN   TestTransportName
--- PASS: TestTransportName (0.00s)
=== RUN   TestTransportParseReference
--- PASS: TestTransportParseReference (0.00s)
=== RUN   TestTransportValidatePolicyConfigurationScope
--- PASS: TestTransportValidatePolicyConfigurationScope (0.00s)
=== RUN   TestParseReference
--- PASS: TestParseReference (0.00s)
=== RUN   TestReferenceTransport
--- PASS: TestReferenceTransport (0.00s)
PASS
ok  	cdk-ecr-deployment-handler/s3	0.008s
FAIL
make: *** [test] Error 1
The command '/bin/sh -c mkdir -p /asset/ &&     make OUTPUT=/asset/main &&     file /asset/main &&     ls -lh /asset/main' returned a non-zero code: 2
/opt/atlassian/pipelines/agent/build/cdk/node_modules/aws-cdk-lib/core/lib/bundling.js:1
"use strict";var _a,_b;Object.defineProperty(exports,"__esModule",{value:!0}),exports.DockerVolumeConsistency=exports.DockerImage=exports.BundlingDockerImage=exports.BundlingOutput=void 0;const jsiiDeprecationWarnings=require("../../.warnings.jsii.js"),JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti"),child_process_1=require("child_process"),crypto=require("crypto"),path_1=require("path"),fs_1=require("./fs"),jsii_deprecated_1=require("./private/jsii-deprecated");var BundlingOutput;(function(BundlingOutput2){BundlingOutput2.ARCHIVED="archived",BundlingOutput2.NOT_ARCHIVED="not-archived",BundlingOutput2.AUTO_DISCOVER="auto-discover"})(BundlingOutput=exports.BundlingOutput||(exports.BundlingOutput={}));class BundlingDockerImage{constructor(image,_imageHash){this.image=image,this._imageHash=_imageHash}static fromRegistry(image){return new DockerImage(image)}static fromAsset(path,options={}){return DockerImage.fromBuild(path,options)}toJSON(){var _c;return(_c=this._imageHash)!==null&&_c!==void 0?_c:this.image}run(options={}){var _c,_d;const volumes=options.volumes||[],environment=options.environment||{},entrypoint=((_c=options.entrypoint)===null||_c===void 0?void 0:_c[0])||null,command=[...!((_d=options.entrypoint)===null||_d===void 0)&&_d[1]?[...options.entrypoint.slice(1)]:[],...options.command?[...options.command]:[]],dockerArgs=["run","--rm",...options.securityOpt?["--security-opt",options.securityOpt]:[],...options.user?["-u",options.user]:[],...flatten(volumes.map(v=>{var _c2;return["-v",`${v.hostPath}:${v.containerPath}:${isSeLinux()?"z,":""}${(_c2=v.consistency)!==null&&_c2!==void 0?_c2:DockerVolumeConsistency.DELEGATED}`]})),...flatten(Object.entries(environment).map(([k,v])=>["--env",`${k}=${v}`])),...options.workingDirectory?["-w",options.workingDirectory]:[],...entrypoint?["--entrypoint",entrypoint]:[],this.image,...command];dockerExec(dockerArgs)}cp(imagePath,outputPath){const{stdout}=dockerExec(["create",this.image],{}),match=stdout.toString().match(/([0-9a-f]{16,})/);if(!match)throw new Error("Failed to extract container ID from Docker create output");const containerId=match[1],containerPath=`${containerId}:${imagePath}`,destPath=outputPath!=null?outputPath:fs_1.FileSystem.mkdtemp("cdk-docker-cp-");try{return dockerExec(["cp",containerPath,destPath]),destPath}catch(err){throw new Error(`Failed to copy files from ${containerPath} to ${destPath}: ${err}`)}finally{dockerExec(["rm","-v",containerId])}}}exports.BundlingDockerImage=BundlingDockerImage,_a=JSII_RTTI_SYMBOL_1,BundlingDockerImage[_a]={fqn:"aws-cdk-lib.BundlingDockerImage",version:"2.19.0"};class DockerImage extends BundlingDockerImage{constructor(image,_imageHash){const deprecated=jsii_deprecated_1.quiet();super(image,_imageHash);jsii_deprecated_1.reset(deprecated),this.image=image}static fromBuild(path,options={}){jsiiDeprecationWarnings.aws_cdk_lib_DockerBuildOptions(options);const buildArgs=options.buildArgs||{};if(options.file&&path_1.isAbsolute(options.file))throw new Error(`"file" must be relative to the docker build directory. Got ${options.file}`);const input=JSON.stringify({path,...options}),tag=`cdk-${crypto.createHash("sha256").update(input).digest("hex")}`,dockerArgs=["build","-t",tag,...options.file?["-f",path_1.join(path,options.file)]:[],...options.platform?["--platform",options.platform]:[],...flatten(Object.entries(buildArgs).map(([k,v])=>["--build-arg",`${k}=${v}`])),path];dockerExec(dockerArgs);const hash=fs_1.FileSystem.fingerprint(path,{extraHash:JSON.stringify(options)});return new DockerImage(tag,hash)}static fromRegistry(image){return new DockerImage(image)}toJSON(){const deprecated=jsii_deprecated_1.quiet(),json=super.toJSON();return jsii_deprecated_1.reset(deprecated),json}run(options={}){jsiiDeprecationWarnings.aws_cdk_lib_DockerRunOptions(options);const deprecated=jsii_deprecated_1.quiet(),result=super.run(options);return jsii_deprecated_1.reset(deprecated),result}cp(imagePath,outputPath){const deprecated=jsii_deprecated_1.quiet(),result=super.cp(imagePath,outputPath);return jsii_deprecated_1.reset(deprecated),result}}exports.DockerImage=DockerImage,_b=JSII_RTTI_SYMBOL_1,DockerImage[_b]={fqn:"aws-cdk-lib.DockerImage",version:"2.19.0"};var DockerVolumeConsistency;(function(DockerVolumeConsistency2){DockerVolumeConsistency2.CONSISTENT="consistent",DockerVolumeConsistency2.DELEGATED="delegated",DockerVolumeConsistency2.CACHED="cached"})(DockerVolumeConsistency=exports.DockerVolumeConsistency||(exports.DockerVolumeConsistency={}));function flatten(x){return Array.prototype.concat([],...x)}function dockerExec(args,options){var _c,_d,_e;const prog=(_c=process.env.CDK_DOCKER)!==null&&_c!==void 0?_c:"docker",proc=child_process_1.spawnSync(prog,args,options!=null?options:{stdio:["ignore",process.stderr,"inherit"]});if(proc.error)throw proc.error;if(proc.status!==0)throw proc.stdout||proc.stderr?new Error(`[Status ${proc.status}] stdout: ${(_d=proc.stdout)===null||_d===void 0?void 0:_d.toString().trim()}
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            
Error: docker exited with status 2
    at dockerExec (/opt/atlassian/pipelines/agent/build/cdk/node_modules/aws-cdk-lib/core/lib/bundling.js:4:79)
    at Function.fromBuild (/opt/atlassian/pipelines/agent/build/cdk/node_modules/aws-cdk-lib/core/lib/bundling.js:1:3415)
    at Function.fromDockerBuild (/opt/atlassian/pipelines/agent/build/cdk/node_modules/aws-cdk-lib/aws-lambda/lib/code.js:1:1304)
    at getCode (/opt/atlassian/pipelines/agent/build/cdk/node_modules/cdk-ecr-deployment/lib/index.js:30:42)
    at new ECRDeployment (/opt/atlassian/pipelines/agent/build/cdk/node_modules/cdk-ecr-deployment/lib/index.js:98:19)
    at new FmApiStack (/opt/atlassian/pipelines/agent/build/cdk/lib/fm-api-stack.js:68:7)
    at /opt/atlassian/pipelines/agent/build/cdk/bin/main.js:31:38
    at Array.forEach (<anonymous>)
    at /opt/atlassian/pipelines/agent/build/cdk/bin/main.js:30:25
    at Array.forEach (<anonymous>)
Subprocess exited with error 1

[wchaws]

Is it possible to allow using a pre-built image in a CI environment? The env var CI affects lots of other things so I don't want to turn it off in a CI environment.

That's a good point. Let me add another ENV variable to control that.

[wchaws]

@broom9 Try FORCE_PREBUILT_LAMBDA=1 with the latest version.