testsys: Provide more config control

tools/testsys-config/src/lib.rs

@@ -4,7 +4,7 @@ use handlebars::Handlebars;
 use log::{debug, trace, warn};
 use maplit::btreemap;
 use model::constants::TESTSYS_VERSION;
-use model::SecretName;
+use model::{DestructionPolicy, SecretName};
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 use std::collections::{BTreeMap, HashMap};
@@ -261,6 +261,8 @@ pub struct GenericVariantConfig {
     pub control_plane_endpoint: Option<String>,
     /// The path to userdata that should be used for Bottlerocket launch
     pub userdata: Option<String>,
+    #[serde(default)]
+    pub dev: DeveloperConfig,
 }
 
 impl GenericVariantConfig {
@@ -287,6 +289,40 @@ impl GenericVariantConfig {
             conformance_registry: self.conformance_registry.or(other.conformance_registry),
             control_plane_endpoint: self.control_plane_endpoint.or(other.control_plane_endpoint),
             userdata: self.userdata.or(other.userdata),
+            dev: self.dev.merge(other.dev),
+        }
+    }
+}
+
+/// The configuration for a specific config level (<PLATFORM>-<FLAVOR>). This may or may not be arch
+/// specific depending on it's location in `GenericConfig`.
+/// The configurable fields here add refined control to TestSys objects.
+#[derive(Debug, Default, Deserialize, Serialize, PartialEq, Eq, Clone)]
+#[serde(deny_unknown_fields)]
+#[serde(rename_all = "kebab-case")]
+pub struct DeveloperConfig {
+    /// Control the destruction behavior of cluster CRDs
+    pub cluster_destruction_policy: Option<DestructionPolicy>,
+    /// Control the destruction behavior of Bottlerocket CRDs
+    pub bottlerocket_destruction_policy: Option<DestructionPolicy>,
+    /// Keep test pods running on completion
+    pub keep_tests_running: Option<bool>,
+    /// Use an alternate account for image lookup
+    pub image_account_id: Option<String>,
+}
+
+impl DeveloperConfig {
+    /// Overwrite the unset values of `self` with the set values of `other`
+    fn merge(self, other: Self) -> Self {
+        Self {
+            cluster_destruction_policy: self
+                .cluster_destruction_policy
+                .or(other.cluster_destruction_policy),
+            bottlerocket_destruction_policy: self
+                .bottlerocket_destruction_policy
+                .or(other.bottlerocket_destruction_policy),
+            keep_tests_running: self.keep_tests_running.or(other.keep_tests_running),
+            image_account_id: self.image_account_id.or(other.image_account_id),
         }
     }
 }

tools/testsys/Test.toml.example

@@ -78,6 +78,9 @@ eks-resource-agent-image = "public.ecr.aws/bottlerocket-test-system/eks_resource
 # Note: values passed by command line argument will take precedence over those passed by environment
 # variable, and both take precedence over values set by `Test.toml`.
 
+# Additional fields are configurable with the `dev` table.
+# See `DeveloperConfig` for individual fields.
+
 # Example Configurations
 
 # Configuration for all variants with the `aws` platform.

tools/testsys/src/aws_ecs.rs

@@ -39,6 +39,7 @@ impl CrdCreator for AwsEcsCreator {
             })?)
            , &crd_input.arch,
            & self.region,
+           crd_input.config.dev.image_account_id.as_deref(),
         )
         .await
     }
@@ -73,7 +74,15 @@ impl CrdCreator for AwsEcsCreator {
             .cluster_name(cluster_input.cluster_name)
             .region(Some(self.region.to_owned()))
             .assume_role(cluster_input.crd_input.config.agent_role.clone())
-            .destruction_policy(DestructionPolicy::OnTestSuccess)
+            .destruction_policy(
+                cluster_input
+                    .crd_input
+                    .config
+                    .dev
+                    .cluster_destruction_policy
+                    .to_owned()
+                    .unwrap_or(DestructionPolicy::OnTestSuccess),
+            )
             .image(
                 cluster_input
                     .crd_input
@@ -160,7 +169,14 @@ impl CrdCreator for AwsEcsCreator {
                     .testsys_agent_pull_secret
                     .to_owned(),
             )
-            .keep_running(true)
+            .keep_running(
+                test_input
+                    .crd_input
+                    .config
+                    .dev
+                    .keep_tests_running
+                    .unwrap_or(false),
+            )
             .set_secrets(Some(test_input.crd_input.config.secrets.to_owned()))
             .set_labels(Some(labels))
             .build(format!(

tools/testsys/src/aws_k8s.rs

@@ -43,6 +43,7 @@ impl CrdCreator for AwsK8sCreator {
             })?)
            , &crd_input.arch,
            & self.region,
+           crd_input.config.dev.image_account_id.as_deref(),
         )
         .await
     }
@@ -123,7 +124,15 @@ impl CrdCreator for AwsK8sCreator {
             )
             .set_labels(Some(labels))
             .set_secrets(Some(cluster_input.crd_input.config.secrets.clone()))
-            .destruction_policy(DestructionPolicy::Never)
+            .destruction_policy(
+                cluster_input
+                    .crd_input
+                    .config
+                    .dev
+                    .cluster_destruction_policy
+                    .to_owned()
+                    .unwrap_or(DestructionPolicy::Never),
+            )
             .build(cluster_name)
             .context(error::BuildSnafu {
                 what: "EKS cluster CRD",

tools/testsys/src/aws_resources.rs

@@ -38,7 +38,12 @@ pub(crate) fn ami(ami_input: &str, region: &str) -> Result<String> {
 }
 
 /// Queries EC2 for the given AMI name. If found, returns Ok(Some(id)), if not returns Ok(None).
-pub(crate) async fn get_ami_id<S1, S2, S3>(name: S1, arch: S2, region: S3) -> Result<String>
+pub(crate) async fn get_ami_id<S1, S2, S3>(
+    name: S1,
+    arch: S2,
+    region: S3,
+    account: Option<&str>,
+) -> Result<String>
 where
     S1: Into<String>,
     S2: Into<String>,
@@ -54,7 +59,7 @@ where
     // Find all images named `name` on `arch` in the `region`.
     let describe_images = ec2_client
         .describe_images()
-        .owners("self")
+        .owners(account.unwrap_or("self"))
         .filters(Filter::builder().name("name").values(name).build())
         .filters(
             Filter::builder()
@@ -176,7 +181,15 @@ pub(crate) async fn ec2_crd<'a>(
         .set_labels(Some(labels))
         .set_conflicts_with(conflicting_resources.into())
         .set_secrets(Some(bottlerocket_input.crd_input.config.secrets.clone()))
-        .destruction_policy(DestructionPolicy::OnTestSuccess);
+        .destruction_policy(
+            bottlerocket_input
+                .crd_input
+                .config
+                .dev
+                .bottlerocket_destruction_policy
+                .to_owned()
+                .unwrap_or(DestructionPolicy::OnTestSuccess),
+        );
 
     // Add in the EKS specific configuration.
     if cluster_type == ClusterType::Eks {

tools/testsys/src/migration.rs

@@ -80,7 +80,14 @@ pub(crate) fn migration_crd(
                 .testsys_agent_pull_secret
                 .to_owned(),
         )
-        .keep_running(true)
+        .keep_running(
+            migration_input
+                .crd_input
+                .config
+                .dev
+                .keep_tests_running
+                .unwrap_or(false),
+        )
         .set_secrets(Some(migration_input.crd_input.config.secrets.to_owned()))
         .set_labels(Some(labels))
         .build(format!(

tools/testsys/src/run.rs

@@ -164,6 +164,7 @@ impl From<CliConfig> for GenericVariantConfig {
             conformance_registry: val.conformance_registry,
             control_plane_endpoint: val.control_plane_endpoint,
             userdata: val.userdata,
+            dev: Default::default(),
         }
     }
 }

tools/testsys/src/sonobuoy.rs

@@ -48,7 +48,14 @@ pub(crate) fn sonobuoy_crd(test_input: TestInput) -> Result<Test> {
                 .testsys_agent_pull_secret
                 .to_owned(),
         )
-        .keep_running(true)
+        .keep_running(
+            test_input
+                .crd_input
+                .config
+                .dev
+                .keep_tests_running
+                .unwrap_or(false),
+        )
         .kubeconfig_base64_template(cluster_resource_name, "encodedKubeconfig")
         .plugin("e2e")
         .mode(sonobuoy_mode)

tools/testsys/src/vmware_k8s.rs

@@ -121,7 +121,15 @@ impl CrdCreator for VmwareK8sCreator {
             .vcenter_workload_folder(&self.datacenter.folder)
             .mgmt_cluster_kubeconfig_base64(&self.encoded_mgmt_cluster_kubeconfig)
             .set_conflicts_with(Some(existing_clusters))
-            .destruction_policy(DestructionPolicy::OnTestSuccess)
+            .destruction_policy(
+                cluster_input
+                    .crd_input
+                    .config
+                    .dev
+                    .cluster_destruction_policy
+                    .to_owned()
+                    .unwrap_or(DestructionPolicy::OnTestSuccess),
+            )
             .image(
                 cluster_input
                     .crd_input
@@ -208,7 +216,15 @@ impl CrdCreator for VmwareK8sCreator {
             .assume_role(bottlerocket_input.crd_input.config.agent_role.clone())
             .set_labels(Some(labels))
             .set_conflicts_with(Some(existing_clusters))
-            .destruction_policy(DestructionPolicy::OnTestSuccess)
+            .destruction_policy(
+                bottlerocket_input
+                    .crd_input
+                    .config
+                    .dev
+                    .bottlerocket_destruction_policy
+                    .to_owned()
+                    .unwrap_or(DestructionPolicy::OnTestSuccess),
+            )
             .image(
                 bottlerocket_input
                     .crd_input