From 67d01c5c02df62682194907b14a1f3c71bb13b6a Mon Sep 17 00:00:00 2001
From: penghaoh <penghaoh@amazon.com>
Date: Wed, 4 Nov 2020 14:25:38 -0800
Subject: [PATCH 1/2] chore: add retry to EnvController waitFor requests

---
 cf-custom-resources/lib/env-controller.js     | 63 +++++++++++++------
 .../test/env-controller-test.js               |  5 +-
 2 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/cf-custom-resources/lib/env-controller.js b/cf-custom-resources/lib/env-controller.js
index ee3362d14a9..2da09a19279 100644
--- a/cf-custom-resources/lib/env-controller.js
+++ b/cf-custom-resources/lib/env-controller.js
@@ -136,6 +136,10 @@ const controlEnv = async function (
       await cfn
         .waitFor("stackUpdateComplete", {
           StackName: stackName,
+          $waiter: {
+            delay: 30,
+            maxAttempts: 29,
+          },
         })
         .promise();
       continue;
@@ -144,6 +148,10 @@ const controlEnv = async function (
     await cfn
       .waitFor("stackUpdateComplete", {
         StackName: stackName,
+        $waiter: {
+          delay: 30,
+          maxAttempts: 29,
+        },
       })
       .promise();
     describeStackResp = await cfn
@@ -169,30 +177,39 @@ exports.handler = async function (event, context) {
   try {
     switch (event.RequestType) {
       case "Create":
-        responseData = await controlEnv(
-          "Create",
-          props.EnvStack,
-          props.Workload,
-          props.Parameters
-        );
+        responseData = await Promise.race([
+          exports.deadlineExpired(),
+          controlEnv(
+            "Create",
+            props.EnvStack,
+            props.Workload,
+            props.Parameters
+          ),
+        ]);
         physicalResourceId = `envcontoller/${props.EnvStack}/${props.Workload}`;
         break;
       case "Update":
-        responseData = await controlEnv(
-          "Update",
-          props.EnvStack,
-          props.Workload,
-          props.Parameters
-        );
+        responseData = await Promise.race([
+          exports.deadlineExpired(),
+          controlEnv(
+            "Update",
+            props.EnvStack,
+            props.Workload,
+            props.Parameters
+          ),
+        ]);
         physicalResourceId = event.PhysicalResourceId;
         break;
       case "Delete":
-        await controlEnv(
-          "Delete",
-          props.EnvStack,
-          props.Workload,
-          props.Parameters
-        );
+        responseData = await Promise.race([
+          exports.deadlineExpired(),
+          controlEnv(
+            "Delete",
+            props.EnvStack,
+            props.Workload,
+            props.Parameters
+          ),
+        ]);
         physicalResourceId = event.PhysicalResourceId;
         break;
       default:
@@ -253,6 +270,16 @@ const updateParameter = function (requestType, workload, paramValue) {
   return [updatedParamValue, updatedParamValue !== paramValue];
 };
 
+exports.deadlineExpired = function () {
+  return new Promise(function (resolve, reject) {
+    setTimeout(
+      reject,
+      14 * 60 * 1000 + 30 * 1000 /* 14.5 minutes*/,
+      new Error("Lambda took longer than 14.5 minutes to update environment")
+    );
+  });
+};
+
 /**
  * @private
  */
diff --git a/cf-custom-resources/test/env-controller-test.js b/cf-custom-resources/test/env-controller-test.js
index ba6fc167296..d5af301953f 100644
--- a/cf-custom-resources/test/env-controller-test.js
+++ b/cf-custom-resources/test/env-controller-test.js
@@ -30,6 +30,9 @@ describe("Env Controller Handler", () => {
 
   beforeEach(() => {
     EnvController.withDefaultResponseURL(ResponseURL);
+    EnvController.deadlineExpired = function () {
+      return new Promise(function (resolve, reject) {});
+    };
     // Prevent logging.
     console.log = function () {};
   });
@@ -208,7 +211,7 @@ describe("Env Controller Handler", () => {
         {
           StackName: "mockEnvStack",
           Parameters: testParams,
-          Outputs: []
+          Outputs: [],
         },
       ],
     });

From e65ffdc0d9759fd7f93bab48b44f1043cf96ea0a Mon Sep 17 00:00:00 2001
From: penghaoh <penghaoh@amazon.com>
Date: Wed, 4 Nov 2020 14:59:56 -0800
Subject: [PATCH 2/2] Address feedback

---
 cf-custom-resources/lib/env-controller.js | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/cf-custom-resources/lib/env-controller.js b/cf-custom-resources/lib/env-controller.js
index 2da09a19279..a66ffda6af9 100644
--- a/cf-custom-resources/lib/env-controller.js
+++ b/cf-custom-resources/lib/env-controller.js
@@ -7,6 +7,11 @@ const aws = require("aws-sdk");
 // These are used for test purposes only
 let defaultResponseURL;
 
+const updateStackWaiter = {
+  delay: 30,
+  maxAttempts: 29,
+};
+
 /**
  * Upload a CloudFormation response object to S3.
  *
@@ -136,10 +141,7 @@ const controlEnv = async function (
       await cfn
         .waitFor("stackUpdateComplete", {
           StackName: stackName,
-          $waiter: {
-            delay: 30,
-            maxAttempts: 29,
-          },
+          $waiter: updateStackWaiter,
         })
         .promise();
       continue;
@@ -148,10 +150,7 @@ const controlEnv = async function (
     await cfn
       .waitFor("stackUpdateComplete", {
         StackName: stackName,
-        $waiter: {
-          delay: 30,
-          maxAttempts: 29,
-        },
+        $waiter: updateStackWaiter,
       })
       .promise();
     describeStackResp = await cfn