[NoQA] Add sleep cooldowns to e2e tests

.github/workflows/e2ePerformanceTests.yml

@@ -183,6 +183,7 @@ jobs:
           file_artifacts: Customer Artifacts.zip
           log_artifacts: debug.log
           cleanup: true
+          timeout: 5400
 
       - name: Print logs if run failed
         if: failure()
@@ -213,6 +214,7 @@ jobs:
           remote_src: false
           file_artifacts: Customer Artifacts.zip
           cleanup: true
+          timeout: 5400
 
       - name: Unzip AWS Device Farm delta results
         run: unzip "Customer Artifacts.zip" -d deltaResults

tests/e2e/config.js

@@ -31,7 +31,7 @@ module.exports = {
     SERVER_PORT: 4723,
 
     // The amount of times a test should be executed for average performance metrics
-    RUNS: 80,
+    RUNS: 60,
 
     DEFAULT_BASELINE_BRANCH: 'main',
 
@@ -47,7 +47,10 @@ module.exports = {
     INTERACTION_TIMEOUT: 300000,
 
     // Period we wait between each test runs, to let the device cool down
-    COOL_DOWN: 90 * 1000,
+    BOOT_COOL_DOWN: 90 * 1000,
+
+    // Period we wait between each test runs, to let the device cool down
+    SUITE_COOL_DOWN: 10 * 1000,
 
     TEST_NAMES,
 

tests/e2e/testRunner.js

@@ -27,6 +27,7 @@ const math = require('./measure/math');
 const writeTestStats = require('./measure/writeTestStats');
 const withFailTimeout = require('./utils/withFailTimeout');
 const reversePort = require('./utils/androidReversePort');
+const sleep = require('./utils/sleep');
 
 // VARIABLE CONFIGURATION
 const args = process.argv.slice(2);
@@ -202,6 +203,15 @@ const runTests = async () => {
             }
         }
 
+        const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`);
+        coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`);
+
+        // Having the cooldown right at the beginning should hopefully lower the chances of heat
+        // throttling from the previous run (which we have no control over and will be a
+        // completely different AWS DF customer/app). It also gives the time to cool down between test suites.
+        await sleep(config.BOOT_COOL_DOWN);
+        coolDownLogs.done();
+
         server.setTestConfig(testConfig);
 
         const warmupLogs = Logger.progressInfo(`Running warmup '${testConfig.name}'`);
@@ -229,7 +239,17 @@ const runTests = async () => {
             progressText = `Suite '${testConfig.name}' [${testIndex + 1}/${numOfTests}], iteration [${i + 1}/${config.RUNS}]\n`;
             testLog.updateText(progressText);
 
-            await restartApp();
+            Logger.log('Killing app...');
+            await killApp('android', config.APP_PACKAGE);
+
+            testLog.updateText(`Coolin down phone 🧊 ${config.SUITE_COOL_DOWN / 1000}s\n`);
+
+            // Adding the cool down between booting the app again, had the side-effect of actually causing a cold boot,
+            // which increased TTI/bundle load JS times significantly but also stabilized standard deviation.
+            await sleep(config.SUITE_COOL_DOWN);
+
+            Logger.log('Starting app...');
+            await launchApp('android', config.APP_PACKAGE);
 
             // Wait for a test to finish by waiting on its done call to the http server
             try {
@@ -250,15 +270,6 @@ const runTests = async () => {
             }
         }
         testLog.done();
-
-        // If we still have tests add a cool down period
-        if (testIndex < numOfTests - 1) {
-            const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`);
-            coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`);
-            // eslint-disable-next-line no-loop-func
-            await new Promise((resolve) => setTimeout(resolve, config.COOL_DOWN));
-            coolDownLogs.done();
-        }
     }
 
     // Calculate statistics and write them to our work file

tests/e2e/utils/sleep.js

@@ -0,0 +1,5 @@
+module.exports = function sleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+};