Prune hanging instances longer than 3 hours

samuel40791765 · Dec 17, 2024 · d335f89 · d335f89
1 parent 57133c0
commit d335f89
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 3 deletions.
diff --git a/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml b/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml
@@ -10,9 +10,6 @@ mainSteps:
     inputs:
       timeoutSeconds: '7200'
       runCommand:
-        # TODO (P131897680): Parallelize the FIPS and sanitizer tests. The instance timeout can be lowered
-        #       once we do so.
-        #
         # Fallback plan to shut down the ec2 instance in 90 minutes in case it's not terminated.
         # Codebuild just "stops" the instance calling the script, so "trap cleanup" is not executed.
         - shutdown -P +90
@@ -28,6 +25,7 @@ mainSteps:
         # install aws-cli
         - killall apt apt-get
         - apt-get update
+        - apt-get -y remove needrestart
         - apt-get -y install unzip
         - curl "https://awscli.amazonaws.com/awscli-exe-linux-${AWS_CLI_PREFIX}64.zip" -o "awscliv2.zip"
         - unzip awscliv2.zip

diff --git a/tests/ci/lambda/src/bin/purge-stale-builds.rs b/tests/ci/lambda/src/bin/purge-stale-builds.rs
@@ -4,6 +4,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 
 use aws_sdk_codebuild::types::BuildBatchFilter;
 use aws_sdk_ec2::operation::describe_instances::DescribeInstancesOutput;
+use aws_sdk_ec2::primitives::DateTime;
 use aws_sdk_ec2::types::Filter;
 use aws_sdk_ssm::types::DocumentKeyValuesFilter;
 use lambda_runtime::{service_fn, Error, LambdaEvent};
@@ -115,6 +116,8 @@ async fn handle(_event: LambdaEvent<Value>) -> Result<(), Error> {
     let mut ec2_terminated_instances: Vec<String> = vec![];
     let mut stopped_builds: u64 = 0;
 
+    let now_as_secs = DateTime::from(SystemTime::now()).secs();
+
     for (k, v) in &pull_requests {
         if v.len() <= 1 {
             continue;
@@ -158,6 +161,11 @@ async fn handle(_event: LambdaEvent<Value>) -> Result<(), Error> {
                                     .push(instance.instance_id().unwrap().to_string());
                             }
                         }
+                        let launch_elapsed_time = now_as_secs - instance.launch_time().unwrap().secs();
+                        log::info!("Launch time: {:?}", launch_elapsed_time);
+                        if launch_elapsed_time > 30 {
+                            log::info!("Longer than 30 seconds");
+                        }
                     }
                 }
             }
@@ -168,6 +176,8 @@ async fn handle(_event: LambdaEvent<Value>) -> Result<(), Error> {
         }
     }
 
+    log::info!("time {:?}", now_as_secs);
+
     log::info!("Terminating instances {:?}", ec2_terminated_instances);
     if let Some(ref ec2_client) = ec2_client_optional {
         if !ec2_terminated_instances.is_empty() {