chore: add retry to EnvController waitFor requests (#1632)

<!-- Provide summary of changes -->
Address this [comment](#1508 (comment)) left in #1508. Add retry to EnvController when we call `cfn.waitFor()`, so that it won't immediately error out when the request gets throttled.
<!-- Issue number, if available. E.g. "Fixes #31", "Addresses #42, 77" -->

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

cf-custom-resources/lib/env-controller.js

@@ -7,6 +7,11 @@ const aws = require("aws-sdk");
 // These are used for test purposes only
 let defaultResponseURL;
 
+const updateStackWaiter = {
+  delay: 30,
+  maxAttempts: 29,
+};
+
 /**
  * Upload a CloudFormation response object to S3.
  *
@@ -136,6 +141,7 @@ const controlEnv = async function (
       await cfn
         .waitFor("stackUpdateComplete", {
           StackName: stackName,
+          $waiter: updateStackWaiter,
         })
         .promise();
       continue;
@@ -144,6 +150,7 @@ const controlEnv = async function (
     await cfn
       .waitFor("stackUpdateComplete", {
         StackName: stackName,
+        $waiter: updateStackWaiter,
       })
       .promise();
     describeStackResp = await cfn
@@ -169,30 +176,39 @@ exports.handler = async function (event, context) {
   try {
     switch (event.RequestType) {
       case "Create":
-        responseData = await controlEnv(
-          "Create",
-          props.EnvStack,
-          props.Workload,
-          props.Parameters
-        );
+        responseData = await Promise.race([
+          exports.deadlineExpired(),
+          controlEnv(
+            "Create",
+            props.EnvStack,
+            props.Workload,
+            props.Parameters
+          ),
+        ]);
         physicalResourceId = `envcontoller/${props.EnvStack}/${props.Workload}`;
         break;
       case "Update":
-        responseData = await controlEnv(
-          "Update",
-          props.EnvStack,
-          props.Workload,
-          props.Parameters
-        );
+        responseData = await Promise.race([
+          exports.deadlineExpired(),
+          controlEnv(
+            "Update",
+            props.EnvStack,
+            props.Workload,
+            props.Parameters
+          ),
+        ]);
         physicalResourceId = event.PhysicalResourceId;
         break;
       case "Delete":
-        await controlEnv(
-          "Delete",
-          props.EnvStack,
-          props.Workload,
-          props.Parameters
-        );
+        responseData = await Promise.race([
+          exports.deadlineExpired(),
+          controlEnv(
+            "Delete",
+            props.EnvStack,
+            props.Workload,
+            props.Parameters
+          ),
+        ]);
         physicalResourceId = event.PhysicalResourceId;
         break;
       default:
@@ -253,6 +269,16 @@ const updateParameter = function (requestType, workload, paramValue) {
   return [updatedParamValue, updatedParamValue !== paramValue];
 };
 
+exports.deadlineExpired = function () {
+  return new Promise(function (resolve, reject) {
+    setTimeout(
+      reject,
+      14 * 60 * 1000 + 30 * 1000 /* 14.5 minutes*/,
+      new Error("Lambda took longer than 14.5 minutes to update environment")
+    );
+  });
+};
+
 /**
  * @private
  */

cf-custom-resources/test/env-controller-test.js

@@ -30,6 +30,9 @@ describe("Env Controller Handler", () => {
 
   beforeEach(() => {
     EnvController.withDefaultResponseURL(ResponseURL);
+    EnvController.deadlineExpired = function () {
+      return new Promise(function (resolve, reject) {});
+    };
     // Prevent logging.
     console.log = function () {};
   });
@@ -208,7 +211,7 @@ describe("Env Controller Handler", () => {
         {
           StackName: "mockEnvStack",
           Parameters: testParams,
-          Outputs: []
+          Outputs: [],
         },
       ],
     });