From e0b94fffd12a836de2f776e92dcac27753151509 Mon Sep 17 00:00:00 2001 From: ecpullen Date: Thu, 21 Jul 2022 21:12:47 +0000 Subject: [PATCH] testsys: migration support for aws-k8s variants --- Makefile.toml | 11 + TESTING.md | 74 +++++- tools/Cargo.lock | 414 ++++++++++++++++++++++++++++- tools/testsys/Cargo.toml | 2 + tools/testsys/src/aws_resources.rs | 281 ++++++++++++++++++-- tools/testsys/src/run.rs | 74 +++++- 6 files changed, 825 insertions(+), 31 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 58636cf8995..05e11842e9f 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -119,9 +119,20 @@ AMI_DATA_FILE_SUFFIX = "amis.json" # The type of testsys test that should be run. # `quick` will run a quick test which usually tests that the instances are reachable. # `conformance` will run a certified conformance test, these tests may take up to 3 hrs. +# `migration` will run an upgrade downgrade test including: +# 1: an initial `quick` test +# 2: a migration from TESTSYS_STARTING_VERSION to BUILDSYS_FULL_VERSION +# 3: a `quick` test on the migrated instances +# 4: a migration from BUILDSYS_FULL_VERSION back to TESTSYS_STARTING_VERSION +# 5: a final `quick` test on the downgraded instances +# TESTSYS_STARTING_IMAGE_ID can be used to provide the correct starting image for migration tests. TESTSYS_TEST = "quick" # The path to the testsys cluster's kubeconfig file. This is used for all testsys calls. TESTSYS_KUBECONFIG_PATH = "${BUILDSYS_ROOT_DIR}/testsys.kubeconfig" +# The last released version of bottlerocket. +TESTSYS_STARTING_VERSION = { script = ["git tag --list --sort=version:refname 'v*' | tail -1"] } +# The commit for the last release of bottlerocket. +TESTSYS_STARTING_COMMIT = { script = ["git describe --tag ${TESTSYS_STARTING_VERSION} --always --exclude '*' || echo 00000000"] } [env.development] # Certain variables are defined here to allow us to override a component value diff --git a/TESTING.md b/TESTING.md index b3959ba2eef..955536fdbbb 100644 --- a/TESTING.md +++ b/TESTING.md @@ -169,8 +169,78 @@ cargo make \ test ``` -(`-r` tells testsys to also show the status of resources like the cluster and instances in addition to tests): - ```shell cargo make watch-test ``` + +## Migration Testing + +Migration testing is used to ensure Bottlerocket can update from one version to a new version and back. +This involves launching Bottlerocket instances, upgrading them, and downgrading them. + +Migration testing launches instances of a starting Bottlerocket version, or a provided initial AMI and migrates instances to the target version. +In order to accomplish this a few artifacts need to be created: +* A publicly accessible TUF repository +* A previous release of Bottlerocket signed with available keys +* The AMI ID for the previous release +* Image artifacts and local TUF repos of said artifacts for current changes + +### The setup + +#### Prepare `Infra.toml` + +We need the URL of an accessible TUF repo so the Bottlerocket instances know where to retrieve the update metadata and targets. +Follow our (publishing guide)[PUBLISHING.md#repo-location] to set up TUF repos. +`Infra.toml` is used by testsys to determine TUF repo locations, so `metadata_base_url` and `targets_base_url` need to be set based on the repo that was just created. +The examples below also assume that the default repo is being used in `Infra.toml`, but any repo can be used by setting the `PUBLISH_REPO` environment variable. + +#### Starting Bottlerocket images + +In this example we will use `v1.9.0` as our starting Bottlerocket version, but any tag from Bottlerocket will work. +The following bash script will checkout the proper branch from git and create the build images and TUF repos for testing. + +```shell +git checkout "v1.9.0" +cargo make +cargo make ami +cargo make repo +``` + +Remember to sync your TUF repos with the new metadata and targets. + +#### Target Bottlerocket images + +Now, it's time to create the Bottlerocket artifacts that need to be upgraded to. + +Switch to the working git branch that should be built from. + +```shell +WORKING_BRANCH="develop" +git checkout "${WORKING_BRANCH}" +``` + +Next, build Bottlerocket images and repos and sync TUF repos. + +```shell +cargo make +cargo make ami +cargo make repo +``` + +Now, sync your TUF repos with the new metadata and targets. + +This completes the setup and it is time to test migrations! + +### Testing Migrations + +The previous steps set up the artifacts necessary to perform migration testing using `testsys`. +Ensure all environment variables are still set and set them if they aren't. + +To run the migration test set `TESTSYS_TEST=migration` in the `cargo make test` call. +This will automatically determine the ami that should be used by finding the latest released version of bottlerocket and checking the user's AMIs to find the correct starting AMI ID. + +```shell +cargo make -e TESTSYS_TEST=migration test +``` + +To see the state of the tests as they run use `cargo make watch-test`. diff --git a/tools/Cargo.lock b/tools/Cargo.lock index a6ce1c5b3c5..c807e4d8267 100644 --- a/tools/Cargo.lock +++ b/tools/Cargo.lock @@ -119,6 +119,290 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11a8c971b0cb0484fc9436a291a44503b95141edc36ce7a6af6b6d7a06a02ab0" +dependencies = [ + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "hex", + "http", + "hyper", + "ring", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bc956f415dda77215372e5bc751a2463d1f9a1ec34edf3edc6c0ff67e5c8e43" +dependencies = [ + "aws-smithy-http", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0d98a1d606aa24554e604f220878db4aa3b525b72f88798524497cc3867fc6" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body", + "lazy_static", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ec2" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b6c04720f5846edb80aa8c4dda848b77efdf99597f1ae48e12ea6b1ad1d3ce" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa0c66fab12976065403cf4cafacffe76afa91d0da335d195af379d4223d235" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048037cdfd7f42fb29b5f969c7f639b4b7eac00e8f911e4eac4f89fb7b3a0500" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tower", +] + +[[package]] +name = "aws-sig-auth" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8386fc0d218dbf2011f65bd8300d21ba98603fd150b962f61239be8b02d1fc6" +dependencies = [ + "aws-sigv4", + "aws-smithy-http", + "aws-types", + "http", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd866926c2c4978210bcb01d7d1b431c794f0c23ca9ee1e420204b018836b5fb" +dependencies = [ + "aws-smithy-http", + "form_urlencoded", + "hex", + "http", + "once_cell", + "percent-encoding", + "regex", + "ring", + "time 0.3.11", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb59cfdd21143006c01b9ca4dc4a9190b8c50c2ef831f9eb36f54f69efa42f1" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-client" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44243329ba8618474c3b7f396de281f175ae172dd515b3d35648671a3cf51871" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls 0.22.1", + "lazy_static", + "pin-project-lite", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba78f69a5bbe7ac1826389304c67b789032d813574e78f9a2d450634277f833" +dependencies = [ + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8a512d68350561e901626baa08af9491cfbd54596201b84b4da846a59e4da3" +dependencies = [ + "aws-smithy-http", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b7633698853aae80bd8b26866531420138eca91ea4620735d20b0537c93c2e" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a94b5a8cc94a85ccbff89eb7bc80dc135ede02847a73d68c04ac2a3e4cf6b7" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d230d281653de22fb0e9c7c74d18d724a39d7148e2165b1e760060064c4967c0" +dependencies = [ + "itoa", + "num-integer", + "ryu", + "time 0.3.11", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aacaf6c0fa549ebe5d9daa96233b8635965721367ee7c69effc8d8078842df3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb54f097516352475a0159c9355f8b4737c54044538a4d9aca4d376ef2361ccc" +dependencies = [ + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http", + "rustc_version", + "tracing", + "zeroize", +] + [[package]] name = "backtrace" version = "0.3.66" @@ -230,6 +514,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0b3de4a0c5e67e16066a0715723abd91edc2f9001d09c46e1dca929351e130e" +[[package]] +name = "bytes-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cargo-readme" version = "3.2.0" @@ -461,6 +755,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct 0.6.1", +] + [[package]] name = "darling" version = "0.14.1" @@ -915,6 +1218,23 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls 0.19.1", + "rustls-native-certs 0.5.0", + "tokio", + "tokio-rustls 0.22.0", + "webpki 0.21.4", +] + [[package]] name = "hyper-rustls" version = "0.23.0" @@ -924,10 +1244,10 @@ dependencies = [ "http", "hyper", "log", - "rustls", - "rustls-native-certs", + "rustls 0.20.6", + "rustls-native-certs 0.6.2", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -1836,7 +2156,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.0", "ipnet", "js-sys", "lazy_static", @@ -1844,13 +2164,13 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.20.6", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", "tower-service", "url", "wasm-bindgen", @@ -1902,7 +2222,7 @@ dependencies = [ "futures", "http", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.0", "lazy_static", "log", "rusoto_credential", @@ -2058,6 +2378,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct 0.6.1", + "webpki 0.21.4", +] + [[package]] name = "rustls" version = "0.20.6" @@ -2066,8 +2399,20 @@ checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ "log", "ring", - "sct", - "webpki", + "sct 0.7.0", + "webpki 0.22.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls 0.19.1", + "schannel", + "security-framework", ] [[package]] @@ -2146,6 +2491,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sct" version = "0.7.0" @@ -2527,6 +2882,8 @@ name = "testsys" version = "0.1.0" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-ec2", "bottlerocket-types", "bottlerocket-variant", "clap 3.2.15", @@ -2686,15 +3043,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.1", + "tokio", + "webpki 0.21.4", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.6", "tokio", - "webpki", + "webpki 0.22.0", ] [[package]] @@ -3003,6 +3371,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" + [[package]] name = "utf-8" version = "0.7.6" @@ -3136,6 +3510,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki" version = "0.22.0" @@ -3152,7 +3536,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" dependencies = [ - "webpki", + "webpki 0.22.0", ] [[package]] @@ -3244,6 +3628,12 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" +[[package]] +name = "xmlparser" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/tools/testsys/Cargo.toml b/tools/testsys/Cargo.toml index 84701cf91a0..653e59c0f7a 100644 --- a/tools/testsys/Cargo.toml +++ b/tools/testsys/Cargo.toml @@ -8,6 +8,8 @@ publish = false [dependencies] anyhow = "1.0" +aws-config = "0.46" +aws-sdk-ec2 = "0.16" bottlerocket-types = { git = "https://github.com/bottlerocket-os/bottlerocket-test-system", rev = "021e8d6", version = "0.1"} bottlerocket-variant = { version = "0.1", path = "../../sources/bottlerocket-variant" } clap = { version = "3", features = ["derive", "env"] } diff --git a/tools/testsys/src/aws_resources.rs b/tools/testsys/src/aws_resources.rs index 4537cb29698..5234250920d 100644 --- a/tools/testsys/src/aws_resources.rs +++ b/tools/testsys/src/aws_resources.rs @@ -1,10 +1,12 @@ use crate::run::{TestType, TestsysImages}; use anyhow::{anyhow, Context, Result}; use bottlerocket_types::agent_config::{ - ClusterType, CreationPolicy, Ec2Config, EksClusterConfig, K8sVersion, SonobuoyConfig, - SonobuoyMode, + ClusterType, CreationPolicy, Ec2Config, EksClusterConfig, K8sVersion, MigrationConfig, + SonobuoyConfig, SonobuoyMode, TufRepoConfig, }; +use aws_sdk_ec2::model::{Filter, Image}; +use aws_sdk_ec2::Region; use bottlerocket_variant::Variant; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use k8s_openapi::serde_json::Value; @@ -15,6 +17,7 @@ use model::{ TestSpec, }; use std::collections::BTreeMap; +use std::convert::identity; pub(crate) struct AwsK8s { pub(crate) arch: String, @@ -26,11 +29,17 @@ pub(crate) struct AwsK8s { pub(crate) secrets: Option>, pub(crate) kube_conformance_image: Option, pub(crate) target_cluster_name: Option, + pub(crate) tuf_repo: Option, + pub(crate) starting_version: Option, + pub(crate) migrate_starting_commit: Option, + pub(crate) starting_image_id: Option, + pub(crate) migrate_to_version: Option, + pub(crate) capabilities: Option>, } impl AwsK8s { /// Create the necessary test and resource crds for the specified test type. - pub(crate) fn create_crds( + pub(crate) async fn create_crds( &self, test: TestType, testsys_images: &TestsysImages, @@ -40,6 +49,7 @@ impl AwsK8s { self.sonobuoy_test_crds(testsys_images, SonobuoyMode::CertifiedConformance) } TestType::Quick => self.sonobuoy_test_crds(testsys_images, SonobuoyMode::Quick), + TestType::Migration => self.migration_test_crds(testsys_images).await, } } @@ -49,13 +59,77 @@ impl AwsK8s { sonobuoy_mode: SonobuoyMode, ) -> Result> { let crds = vec![ - self.eks_crd("", testsys_images)?, - self.ec2_crd("", testsys_images)?, - self.sonobuoy_crd("", "-test", sonobuoy_mode, None, testsys_images)?, + self.eks_crd(testsys_images)?, + self.ec2_crd(testsys_images, None)?, + self.sonobuoy_crd("-test", sonobuoy_mode, None, testsys_images)?, ]; Ok(crds) } + /// Creates `Test` crds for migration testing. + async fn migration_test_crds(&self, testsys_images: &TestsysImages) -> Result> { + let ami = self + .starting_image_id + .as_ref() + .unwrap_or( + &get_ami_id( + format!( + "bottlerocket-{}-{}-{}-{}", + self.variant, self.arch, self.starting_version.as_ref().context("The starting version must be provided for migration testing")?, self.migrate_starting_commit.as_ref().context("The commit for the starting version must be provided if the starting image id is not")? + ), & self.arch, + self.region.to_string(), + ) + .await?, + ) + .to_string(); + let eks = self.eks_crd(testsys_images)?; + let ec2 = self.ec2_crd(testsys_images, Some(ami))?; + let mut depends_on = Vec::new(); + // Start with a `quick` test to make sure instances launched properly + let initial = self.sonobuoy_crd("-1-initial", SonobuoyMode::Quick, None, testsys_images)?; + depends_on.push(initial.name().context("Crd missing name")?); + // Migrate instances to the target version + let start_migrate = self.migration_crd( + format!("{}-2-migrate", self.cluster_name()), + MigrationVersion::Migrated, + Some(depends_on.clone()), + testsys_images, + )?; + // A `quick` test to validate the migration + depends_on.push(start_migrate.name().context("Crd missing name")?); + let migrated = self.sonobuoy_crd( + "-3-migrated", + SonobuoyMode::Quick, + Some(depends_on.clone()), + testsys_images, + )?; + // Migrate instances to the starting version + depends_on.push(migrated.name().context("Crd missing name")?); + let end_migrate = self.migration_crd( + format!("{}-4-migrate", self.cluster_name()), + MigrationVersion::Starting, + Some(depends_on.clone()), + testsys_images, + )?; + // A final quick test to validate the migration back to the starting version + depends_on.push(end_migrate.name().context("Crd missing name")?); + let last = self.sonobuoy_crd( + "-5-final", + SonobuoyMode::Quick, + Some(depends_on.clone()), + testsys_images, + )?; + Ok(vec![ + eks, + ec2, + initial, + start_migrate, + migrated, + end_migrate, + last, + ]) + } + /// Labels help filter test results with `testsys status`. fn labels(&self) -> BTreeMap { btreemap! { @@ -73,13 +147,13 @@ impl AwsK8s { } /// Bottlerocket cluster naming convention. - fn cluster_name(&self, suffix: &str) -> String { + fn cluster_name(&self) -> String { self.target_cluster_name .clone() - .unwrap_or_else(|| format!("{}-{}{}", self.kube_arch(), self.kube_variant(), suffix)) + .unwrap_or_else(|| format!("{}-{}", self.kube_arch(), self.kube_variant())) } - fn eks_crd(&self, cluster_suffix: &str, testsys_images: &TestsysImages) -> Result { + fn eks_crd(&self, testsys_images: &TestsysImages) -> Result { let cluster_version = K8sVersion::parse( Variant::new(&self.variant) .context("The provided variant cannot be interpreted.")? @@ -87,7 +161,7 @@ impl AwsK8s { .context("aws-k8s variant is missing k8s version")?, ) .map_err(|e| anyhow!(e))?; - let cluster_name = self.cluster_name(cluster_suffix); + let cluster_name = self.cluster_name(); let eks_crd = Resource { metadata: ObjectMeta { name: Some(cluster_name.clone()), @@ -125,10 +199,10 @@ impl AwsK8s { Ok(Crd::Resource(eks_crd)) } - fn ec2_crd(&self, cluster_suffix: &str, testsys_images: &TestsysImages) -> Result { - let cluster_name = self.cluster_name(cluster_suffix); + fn ec2_crd(&self, testsys_images: &TestsysImages, override_ami: Option) -> Result { + let cluster_name = self.cluster_name(); let mut ec2_config = Ec2Config { - node_ami: self.ami.clone(), + node_ami: override_ami.unwrap_or_else(|| self.ami.clone()), instance_count: Some(2), instance_type: self.instance_type.clone(), cluster_name: format!("${{{}.clusterName}}", cluster_name), @@ -179,13 +253,12 @@ impl AwsK8s { fn sonobuoy_crd( &self, - cluster_suffix: &str, test_name_suffix: &str, sonobuoy_mode: SonobuoyMode, depends_on: Option>, testsys_images: &TestsysImages, ) -> Result { - let cluster_name = self.cluster_name(cluster_suffix); + let cluster_name = self.cluster_name(); let ec2_resource_name = format!("{}-instances", cluster_name); let test_name = format!("{}{}", cluster_name, test_name_suffix); let sonobuoy = Test { @@ -227,3 +300,181 @@ impl AwsK8s { Ok(Crd::Test(sonobuoy)) } } + +/// In order to easily create migration tests for `aws-k8s` variants we need to implement +/// `Migration` for it. +impl Migration for AwsK8s { + fn migration_config(&self) -> Result { + Ok(MigrationsConfig { + tuf_repo: self + .tuf_repo + .as_ref() + .context("Tuf repo metadata is required for upgrade downgrade testing.")? + .clone(), + starting_version: self + .starting_version + .as_ref() + .context("You must provide a starting version for upgrade downgrade testing.")? + .clone(), + migrate_to_version: self + .migrate_to_version + .as_ref() + .context("You must provide a target version for upgrade downgrade testing.")? + .clone(), + region: self.region.to_string(), + secrets: self.secrets.clone(), + capabilities: self.capabilities.clone(), + assume_role: self.assume_role.clone(), + }) + } + + fn instance_provider(&self) -> String { + let cluster_name = self.cluster_name(); + format!("{}-instances", cluster_name) + } + + fn migration_labels(&self) -> BTreeMap { + btreemap! { + "testsys/arch".to_string() => self.arch.to_string(), + "testsys/variant".to_string() => self.variant.to_string(), + "testsys/flavor".to_string() => "updown".to_string(), + } + } +} + +/// An enum to differentiate between upgrade and downgrade tests. +enum MigrationVersion { + ///`MigrationVersion::Starting` will create a migration to the starting version. + Starting, + ///`MigrationVersion::Migrated` will create a migration to the target version. + Migrated, +} + +/// A configuration containing all information needed to create a migration test for a given +/// variant. +struct MigrationsConfig { + tuf_repo: TufRepoConfig, + starting_version: String, + migrate_to_version: String, + region: String, + secrets: Option>, + capabilities: Option>, + assume_role: Option, +} + +/// Migration is a trait that should be implemented for all traits that use upgrade/downgrade +/// testing. It provides the infrastructure to easily create migration tests. +trait Migration { + /// Create a migration config that is used to create migration tests. + fn migration_config(&self) -> Result; + + /// Create the labels that should be used for the migration tests. + fn migration_labels(&self) -> BTreeMap; + + /// Return the name of the instance provider that the migration agents should use to get the + /// instance ids. + fn instance_provider(&self) -> String; + + /// Create a migration test for a given arch/variant. + fn migration_crd( + &self, + test_name: String, + migration_version: MigrationVersion, + depends_on: Option>, + testsys_images: &TestsysImages, + ) -> Result { + // Get the migration configuration for the given type. + let migration = self.migration_config()?; + + // Determine which version we are migrating to. + let version = match migration_version { + MigrationVersion::Starting => migration.starting_version, + MigrationVersion::Migrated => migration.migrate_to_version, + }; + + // Create the migration test crd. + let mut migration_config = MigrationConfig { + aws_region: migration.region, + instance_ids: Default::default(), + migrate_to_version: version, + tuf_repo: Some(migration.tuf_repo.clone()), + assume_role: migration.assume_role.clone(), + } + .into_map() + .context("Unable to convert migration config to map")?; + migration_config.insert( + "instanceIds".to_string(), + Value::String(format!("${{{}.ids}}", self.instance_provider())), + ); + Ok(Crd::Test(Test { + metadata: ObjectMeta { + name: Some(test_name), + namespace: Some(NAMESPACE.into()), + labels: Some(self.migration_labels()), + ..Default::default() + }, + spec: TestSpec { + resources: vec![self.instance_provider()], + depends_on, + retries: None, + agent: Agent { + name: "migration-test-agent".to_string(), + image: testsys_images.migration_test.to_string(), + pull_secret: testsys_images.secret.clone(), + keep_running: true, + timeout: None, + configuration: Some(migration_config), + secrets: migration.secrets.clone(), + capabilities: migration.capabilities, + }, + }, + status: None, + })) + } +} + +/// Queries EC2 for the given AMI name. If found, returns Ok(Some(id)), if not returns Ok(None). +pub(crate) async fn get_ami_id(name: S1, arch: S2, region: S3) -> Result +where + S1: Into, + S2: Into, + S3: Into, +{ + let config = aws_config::from_env() + .region(Region::new(region.into())) + .load() + .await; + let ec2_client = aws_sdk_ec2::Client::new(&config); + let describe_images = ec2_client + .describe_images() + .owners("self") + .filters(Filter::builder().name("name").values(name).build()) + .filters( + Filter::builder() + .name("image-type") + .values("machine") + .build(), + ) + .filters(Filter::builder().name("architecture").values(arch).build()) + .filters( + Filter::builder() + .name("virtualization-type") + .values("hvm") + .build(), + ) + .send() + .await? + .images; + let images: Vec<&Image> = describe_images + .iter() + .flat_map(|image| identity(image)) + .collect(); + if images.len() > 1 { + return Err(anyhow!("Multiple images were found")); + }; + if let Some(image) = images.last().as_ref() { + Ok(image.image_id().context("No image id for AMI")?.to_string()) + } else { + Err(anyhow!("No images were found")) + } +} diff --git a/tools/testsys/src/run.rs b/tools/testsys/src/run.rs index cc881670316..043846ac044 100644 --- a/tools/testsys/src/run.rs +++ b/tools/testsys/src/run.rs @@ -1,5 +1,6 @@ use crate::aws_resources::AwsK8s; use anyhow::{anyhow, ensure, Context, Result}; +use bottlerocket_types::agent_config::TufRepoConfig; use bottlerocket_variant::Variant; use clap::Parser; use log::{debug, info}; @@ -30,6 +31,10 @@ pub(crate) struct Run { #[clap(long, env = "PUBLISH_INFRA_CONFIG_PATH", parse(from_os_str))] infra_config_path: PathBuf, + /// Use this named repo infrastructure from Infra.toml for upgrade/downgrade testing. + #[clap(long, env = "PUBLISH_REPO", default_value = "default")] + repo: String, + /// The path to `amis.json` #[clap(long, env = "AMI_INPUT")] ami_input: String, @@ -68,6 +73,32 @@ pub(crate) struct Run { #[clap(flatten)] agent_images: TestsysImages, + + // Migrations + /// Override the starting image used for migrations. The image will be pulled from available + /// amis in the users account if no override is provided. + #[clap(long, env = "TESTSYS_STARTING_IMAGE_ID")] + starting_image_id: Option, + + /// The starting version for migrations. This is required for all migrations tests. + /// This is the version that will be created and migrated to `migration-target-version`. + #[clap(long, env = "TESTSYS_STARTING_VERSION")] + migration_starting_version: Option, + + /// The commit id of the starting version for migrations. This is required for all migrations + /// tests unless `starting-image-id` is provided. This is the version that will be created and + /// migrated to `migration-target-version`. + #[clap( + long, + env = "TESTSYS_STARTING_COMMIT", + conflicts_with = "starting-image-id" + )] + migration_starting_commit: Option, + + /// The target version for migrations. This is required for all migration tests. This is the + /// version that will be migrated to. + #[clap(long, env = "BUILDSYS_VERSION_IMAGE")] + migration_target_version: Option, } impl Run { @@ -99,6 +130,26 @@ impl Run { .context("No region was provided and no regions found in infra config")? }; + let repo_config = infra_config + .repo + .unwrap_or_default() + .get(&self.repo) + .and_then(|repo| { + if let (Some(metadata_base_url), Some(targets_url)) = + (&repo.metadata_base_url, &repo.targets_url) + { + Some(TufRepoConfig { + metadata_url: format!( + "{}{}/{}", + metadata_base_url, &self.variant, &self.arch + ), + targets_url: targets_url.to_string(), + }) + } else { + None + } + }); + match variant.family() { "aws-k8s" => { debug!("Variant is in 'aws-k8s' family"); @@ -114,9 +165,17 @@ impl Run { secrets, kube_conformance_image: self.kube_conformance_image, target_cluster_name: self.target_cluster_name, + tuf_repo: repo_config, + starting_version: self.migration_starting_version, + starting_image_id: self.starting_image_id, + migrate_to_version: self.migration_target_version, + capabilities: None, + migrate_starting_commit: self.migration_starting_commit, }; debug!("Creating crds for aws-k8s testing"); - let crds = aws_k8s.create_crds(self.test_flavor, &self.agent_images)?; + let crds = aws_k8s + .create_crds(self.test_flavor, &self.agent_images) + .await?; debug!("Adding crds to testsys cluster"); for crd in crds { let crd = client @@ -170,6 +229,10 @@ pub(crate) enum TestType { /// variance this will run sonobuoy in "quick" mode. For ECS variants, this will run a simple /// ECS task. Quick, + /// Migration testing ensures that all bottlerocket migrations work as expected. Instances will + /// be created at the starting version, migrated to the target version and back to the starting + /// version with validation testing. + Migration, } derive_fromstr_from_deserialize!(TestType); @@ -177,7 +240,6 @@ derive_fromstr_from_deserialize!(TestType); #[derive(Clone, Debug, Deserialize)] pub(crate) struct Image { pub(crate) id: String, - // This is used to deserialize amis.json } #[derive(Debug, Parser)] @@ -206,6 +268,14 @@ pub(crate) struct TestsysImages { )] pub(crate) sonobuoy_test: String, + /// Migration test agent uri. If not provided the latest released test agent will be used. + #[clap( + long = "migration-test-agent-image", + env = "TESTSYS_MIGRATION_TEST_AGENT_IMAGE", + default_value = "public.ecr.aws/bottlerocket-test-system/migration-test-agent:v0.0.1" + )] + pub(crate) migration_test: String, + /// Images pull secret. This is the name of a Kubernetes secret that will be used to /// pull the container image from a private registry. For example, if you created a pull secret /// with `kubectl create secret docker-registry regcred` then you would pass