diff --git a/code-build.js b/code-build.js index c9d8704..99bc81b 100644 --- a/code-build.js +++ b/code-build.js @@ -35,13 +35,20 @@ async function build(sdk, params) { } async function waitForBuildEndTime(sdk, { id, logs }, nextToken) { - const { codeBuild, cloudWatchLogs, wait = 1000 * 5 } = sdk; + const { + codeBuild, + cloudWatchLogs, + wait = 1000 * 30, + backOff = 1000 * 15 + } = sdk; // Get the CloudWatchLog info const startFromHead = true; const { cloudWatchLogsArn } = logs; const { logGroupName, logStreamName } = logName(cloudWatchLogsArn); + let errObject = false; + // Check the state const [batch, cloudWatch = {}] = await Promise.all([ codeBuild.batchGetBuilds({ ids: [id] }).promise(), @@ -50,7 +57,37 @@ async function waitForBuildEndTime(sdk, { id, logs }, nextToken) { cloudWatchLogs .getLogEvents({ logGroupName, logStreamName, startFromHead, nextToken }) .promise() - ]); + ]).catch(err => { + errObject = err; +/* Returning [] here so that the assignment above + * does not throw `TypeError: undefined is not iterable`. + * The error is handled below, + * since it might be a rate limit. + */ + return []; + }); + + if (errObject) { + //We caught an error in trying to make the AWS api call, and are now checking to see if it was just a rate limiting error + if (errObject.message && errObject.message.search("Rate exceeded") !== -1) { + //We were rate-limited, so add `backOff` seconds to the wait time + let newWait = wait + backOff; + + //Sleep before trying again + await new Promise(resolve => setTimeout(resolve, newWait)); + + // Try again from the same token position + return waitForBuildEndTime( + { ...sdk, wait: newWait }, + { id, logs }, + nextToken + ); + } else { + //The error returned from the API wasn't about rate limiting, so throw it as an actual error and fail the job + throw errObject; + } + } + // Pluck off the relevant state const [current] = batch.builds; const { nextForwardToken, events = [] } = cloudWatch; @@ -64,7 +101,7 @@ async function waitForBuildEndTime(sdk, { id, logs }, nextToken) { // We did it! We can stop looking! if (current.endTime && !events.length) return current; - // More to do: Sleep for 5 seconds :) + // More to do: Sleep for a few seconds to avoid rate limiting await new Promise(resolve => setTimeout(resolve, wait)); // Try again diff --git a/test/code-build-test.js b/test/code-build-test.js index 4a4d21f..a4f6873 100644 --- a/test/code-build-test.js +++ b/test/code-build-test.js @@ -318,6 +318,109 @@ describe("waitForBuildEndTime", () => { }); expect(test).to.equal(buildReplies.pop().builds[0]); }); + + it("waits after being rate limited and tries again", async function() { + const buildID = "buildID"; + const nullArn = + "arn:aws:logs:us-west-2:111122223333:log-group:null:log-stream:null"; + const cloudWatchLogsArn = + "arn:aws:logs:us-west-2:111122223333:log-group:/aws/codebuild/CloudWatchLogGroup:log-stream:1234abcd-12ab-34cd-56ef-1234567890ab"; + + const buildReplies = [ + () => { + throw { message: "Rate exceeded" }; + }, + { builds: [{ id: buildID, logs: { cloudWatchLogsArn } }] }, + { + builds: [ + { id: buildID, logs: { cloudWatchLogsArn }, endTime: "endTime" } + ] + } + ]; + + const sdk = help( + () => { + //similar to the ret function in the helper, allows me to throw an error in a function or return a more standard reply + let reply = buildReplies.shift(); + + if (typeof reply === "function") return reply(); + return reply; + }, + () => { + if (!buildReplies.length) { + return { events: [] }; + } + + return { events: [{ message: "got one" }] }; + } + ); + + const test = await waitForBuildEndTime( + { ...sdk, wait: 1, backOff: 1 }, + { + id: buildID, + logs: { cloudWatchLogsArn: nullArn } + } + ); + + expect(test.id).to.equal(buildID); + }); + + it("dies after getting an error from the aws sdk that isn't rate limiting", async function() { + const buildID = "buildID"; + const nullArn = + "arn:aws:logs:us-west-2:111122223333:log-group:null:log-stream:null"; + const cloudWatchLogsArn = + "arn:aws:logs:us-west-2:111122223333:log-group:/aws/codebuild/CloudWatchLogGroup:log-stream:1234abcd-12ab-34cd-56ef-1234567890ab"; + + const buildReplies = [ + () => { + throw { message: "Some AWS error" }; + }, + { builds: [{ id: buildID, logs: { cloudWatchLogsArn } }] }, + { + builds: [ + { id: buildID, logs: { cloudWatchLogsArn }, endTime: "endTime" } + ] + } + ]; + + const sdk = help( + () => { + //similar to the ret function in the helper + //allows me to throw an error in a function or return a more standard reply + let reply = buildReplies.shift(); + + if (typeof reply === "function") return reply(); + return reply; + }, + () => { + if (!buildReplies.length) { + return { events: [] }; + } + + return { events: [{ message: "got one" }] }; + } + ); + + //run the thing and it should fail + let didFail = false; + + try { + await waitForBuildEndTime( + { ...sdk, wait: 1, backOff: 1 }, + { + id: buildID, + logs: { cloudWatchLogsArn: nullArn } + } + ); + } catch (err) { + didFail = true; + expect(err.message).to.equal("Some AWS error"); + } + + expect(didFail).to.equal(true); + }); }); function help(builds, logs) {