Allow sonobuoy test retries

agent/test-agent/examples/example_test_agent/main.rs

@@ -82,13 +82,13 @@ impl test_agent::Runner for ExampleTestRunner {
             println!("Nested Data:\n {:?}", nested.data);
         }
         Ok(TestResults {
-            outcome: Outcome::Fail,
+            outcome: Outcome::Pass,
             ..TestResults::default()
         })
     }
 
     async fn rerun_failed(&mut self, _prev_result: &TestResults) -> Result<TestResults, Self::E> {
-        println!("ExampleTestRunner::run");
+        println!("ExampleTestRunner::rerun");
         for i in 1..=self.config.hello_count {
             println!("hello #{} to {}", i, self.config.person);
             sleep(Duration::from_millis(

bottlerocket/agents/src/bin/sonobuoy-test-agent/main.rs

@@ -34,7 +34,7 @@ spec:
 
 use async_trait::async_trait;
 use bottlerocket_agents::error::Error;
-use bottlerocket_agents::sonobuoy::{delete_sonobuoy, run_sonobuoy};
+use bottlerocket_agents::sonobuoy::{delete_sonobuoy, rerun_failed_sonobuoy, run_sonobuoy};
 use bottlerocket_agents::wireguard::setup_wireguard;
 use bottlerocket_agents::{
     decode_write_kubeconfig, error, init_agent_logger, setup_test_env, TEST_CLUSTER_KUBECONFIG_PATH,
@@ -95,6 +95,33 @@ impl test_agent::Runner for SonobuoyTestRunner {
         .await
     }
 
+    async fn rerun_failed(&mut self, _prev_results: &TestResults) -> Result<TestResults, Self::E> {
+        // Set up the aws credentials if they were provided.
+        if let Some(aws_secret_name) = &self.aws_secret_name {
+            setup_test_env(self, aws_secret_name).await?;
+        }
+
+        if let Some(wireguard_secret_name) = &self.wireguard_secret_name {
+            // If a wireguard secret is specified, try to set up an wireguard connection with the
+            // wireguard configuration stored in the secret.
+            let wireguard_secret = self
+                .get_secret(wireguard_secret_name)
+                .context(error::SecretMissingSnafu)?;
+            setup_wireguard(&wireguard_secret).await?;
+        }
+
+        delete_sonobuoy(TEST_CLUSTER_KUBECONFIG_PATH).await?;
+
+        decode_write_kubeconfig(&self.config.kubeconfig_base64, TEST_CLUSTER_KUBECONFIG_PATH)
+            .await?;
+        rerun_failed_sonobuoy(
+            TEST_CLUSTER_KUBECONFIG_PATH,
+            &self.config,
+            &self.results_dir,
+        )
+        .await
+    }
+
     async fn terminate(&mut self) -> Result<(), Self::E> {
         delete_sonobuoy(TEST_CLUSTER_KUBECONFIG_PATH).await
     }

bottlerocket/agents/src/sonobuoy.rs

@@ -46,6 +46,42 @@ pub async fn run_sonobuoy(
     // TODO - log something or check what happened?
     ensure!(status.success(), error::SonobuoyRunSnafu);
 
+    results_sonobuoy(kubeconfig_path, sonobuoy_config, results_dir)
+}
+
+/// Reruns the the failed tests from sonobuoy conformance that has already run in this agent.
+pub async fn rerun_failed_sonobuoy(
+    kubeconfig_path: &str,
+    sonobuoy_config: &SonobuoyConfig,
+    results_dir: &Path,
+) -> Result<TestResults, error::Error> {
+    let kubeconfig_arg = vec!["--kubeconfig", kubeconfig_path];
+    let results_filepath = results_dir.join(SONOBUOY_RESULTS_FILENAME);
+    info!("Rerunning sonobuoy");
+    let status = Command::new("/usr/bin/sonobuoy")
+        .args(kubeconfig_arg.to_owned())
+        .arg("e2e")
+        .arg("--wait")
+        .arg("--rerun-failed")
+        .arg(results_filepath.as_os_str())
+        .status()
+        .context(error::SonobuoyProcessSnafu)?;
+    info!("Sonobuoy testing has completed, checking results");
+
+    // TODO - log something or check what happened?
+    ensure!(status.success(), error::SonobuoyRunSnafu);
+
+    results_sonobuoy(kubeconfig_path, sonobuoy_config, results_dir)
+}
+
+/// Retrieve the results from a sonobuoy test and convert them into `TestResults`.
+pub fn results_sonobuoy(
+    kubeconfig_path: &str,
+    sonobuoy_config: &SonobuoyConfig,
+    results_dir: &Path,
+) -> Result<TestResults, error::Error> {
+    let kubeconfig_arg = vec!["--kubeconfig", kubeconfig_path];
+
     info!("Running sonobuoy retrieve");
     let results_filepath = results_dir.join(SONOBUOY_RESULTS_FILENAME);
     let status = Command::new("/usr/bin/sonobuoy")

testsys/src/run_aws_k8s.rs

@@ -129,6 +129,10 @@ pub(crate) struct RunAwsK8s {
     #[structopt(long)]
     keep_instance_provider_running: bool,
 
+    /// Allow the sonobuoy test agent to rerun failed test.
+    #[structopt(long)]
+    allow_retries: Option<u32>,
+
     /// Perform an upgrade downgrade test.
     #[structopt(long, requires_all(&["starting-version", "upgrade-version", "migration-agent-image"]))]
     upgrade_downgrade: bool,
@@ -425,7 +429,7 @@ impl RunAwsK8s {
                     cluster_resource_name.to_string(),
                 ],
                 depends_on,
-                retries: None,
+                retries: self.allow_retries.clone(),
                 agent: Agent {
                     name: "sonobuoy-test-agent".to_string(),
                     image: self.test_agent_image.clone(),

testsys/src/run_vmware.rs

@@ -142,6 +142,10 @@ pub(crate) struct RunVmware {
     #[structopt(long)]
     keep_instance_provider_running: bool,
 
+    /// Allow the sonobuoy test agent to rerun failed test.
+    #[structopt(long)]
+    allow_retries: Option<u32>,
+
     /// Perform an upgrade downgrade test.
     #[structopt(long, requires_all(&["starting-version", "upgrade-version"]))]
     upgrade_downgrade: bool,
@@ -373,7 +377,7 @@ impl RunVmware {
             spec: TestSpec {
                 resources: vec![vm_resource_name.to_string()],
                 depends_on,
-                retries: None,
+                retries: self.allow_retries,
                 agent: Agent {
                     name: "vmware-sonobuoy-test-agent".to_string(),
                     image: self.test_agent_image.clone(),