diff --git a/pipelines/e2e/Dockerfile.setup b/pipelines/e2e/Dockerfile.setup index 040363c17f..fce9d352df 100644 --- a/pipelines/e2e/Dockerfile.setup +++ b/pipelines/e2e/Dockerfile.setup @@ -25,7 +25,7 @@ RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_V wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \ mv yq_linux_amd64 /usr/local/bin/yq && \ chmod +x /usr/local/bin/yq && \ - apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip gcc python3-dev libc-dev py3-virtualenv docker && \ + apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip gcc python3-dev libc-dev py3-virtualenv docker openvpn && \ ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ chmod 700 get_helm.sh && \ diff --git a/pipelines/e2e/Jenkinsfile b/pipelines/e2e/Jenkinsfile index 849bae6925..0aff81de82 100644 --- a/pipelines/e2e/Jenkinsfile +++ b/pipelines/e2e/Jenkinsfile @@ -28,6 +28,10 @@ node { usernamePassword(credentialsId: CREDS_ID, passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'), string(credentialsId: 'DO_CREDS', variable: 'DO_TOKEN'), string(credentialsId: REGISTRATION_CODE_ID, variable: 'REGISTRATION_CODE'), + file(credentialsId: 'vpn.ovpn', variable: 'VPN_CONFIG'), + file(credentialsId: 'login.conf', variable: 'LOGIN_CONFIG'), + usernamePassword(credentialsId: 'LAB_API_KEY', passwordVariable: 'LAB_SECRET_KEY', usernameVariable: 'LAB_ACCESS_KEY'), + string(credentialsId: 'LAB_URL', variable: 'LAB_URL'), ]) { if (params.SEND_SLACK_NOTIFICATION) { @@ -42,7 +46,8 @@ node { stage('airgap build') { sh "airgap/scripts/build.sh" - sh """ docker run -itd --name airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} \ + sh """ docker run -itd --cap-add=NET_ADMIN \ + --name airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} \ --env TF_VAR_longhorn_version=${LONGHORN_INSTALL_VERSION} \ --env TF_VAR_do_token=${DO_TOKEN} \ --env TF_VAR_aws_access_key=${AWS_ACCESS_KEY} \ @@ -118,11 +123,17 @@ node { --env TF_VAR_cis_hardening=${CIS_HARDENING} \ --env TF_VAR_resources_owner=longhorn-long-running \ --env TF_VAR_extra_block_device=${RUN_V2_TEST} \ + --env TF_VAR_lab_url=${LAB_URL} \ + --env TF_VAR_lab_access_key=${LAB_ACCESS_KEY} \ + --env TF_VAR_lab_secret_key=${LAB_SECRET_KEY} \ --env IMAGE_NAME=${imageName} \ -v /var/run/docker.sock:/var/run/docker.sock \ --mount source="vol-${imageName}",target=/tmp \ ${imageName} """ + + sh "docker cp ${VPN_CONFIG} ${JOB_BASE_NAME}-${BUILD_NUMBER}:/src/longhorn-tests/vpn.ovpn" + sh "docker cp ${LOGIN_CONFIG} ${JOB_BASE_NAME}-${BUILD_NUMBER}:/src/longhorn-tests/login.conf" } timeout(60) { diff --git a/pipelines/e2e/scripts/main.tf b/pipelines/e2e/scripts/main.tf new file mode 100644 index 0000000000..a0d02d80e9 --- /dev/null +++ b/pipelines/e2e/scripts/main.tf @@ -0,0 +1,135 @@ +terraform { + required_providers { + rancher2 = { + source = "rancher/rancher2" + version = "3.0.0-rc2" + } + } +} + +provider "rancher2" { + api_url = "" + insecure = true + access_key = "" + secret_key = "" +} + +data "rancher2_cluster_v2" "hal-cluster" { + name = "hal" +} + +resource "rancher2_cloud_credential" "credential" { + name = "test-credential" + harvester_credential_config { + cluster_id = data.rancher2_cluster_v2.hal-cluster.cluster_v1_id + cluster_type = "imported" + kubeconfig_content = data.rancher2_cluster_v2.hal-cluster.kube_config + } +} + +resource "rancher2_machine_config_v2" "test-machine-config" { + + generate_name = "test-machine-config" + + harvester_config { + + vm_namespace = "longhorn-qa" + + cpu_count = "4" + memory_size = "8" + + disk_info = <- + ssh-rsa +package_update: true +packages: + - qemu-guest-agent + - iptables +runcmd: + - - systemctl + - enable + - '--now' + - qemu-guest-agent.service +EOF + } +} + +resource "rancher2_cluster_v2" "test-cluster-0" { + + name = "test-cluster-0" + + kubernetes_version = "v1.27.13+rke2r1" + + rke_config { + machine_pools { + name = "control-plane-pool" + cloud_credential_secret_name = rancher2_cloud_credential.credential.id + control_plane_role = true + etcd_role = true + worker_role = false + quantity = 1 + machine_config { + kind = rancher2_machine_config_v2.test-machine-config.kind + name = rancher2_machine_config_v2.test-machine-config.name + } + } + machine_pools { + name = "worker-pool" + cloud_credential_secret_name = rancher2_cloud_credential.credential.id + control_plane_role = false + etcd_role = false + worker_role = true + quantity = 3 + machine_config { + kind = rancher2_machine_config_v2.test-machine-config.kind + name = rancher2_machine_config_v2.test-machine-config.name + } + } + machine_selector_config { + config = { + cloud-provider-name = "" + } + } + machine_global_config = < test_framework/load_balancer_url -fi - -if [[ "${TF_VAR_k8s_distro_name}" == "k3s" ]]; then - terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw instance_mapping | jq 'map({(.name | split(".")[0]): .id}) | add' | jq -s add > /tmp/instance_mapping +if [[ ${LONGHORN_TEST_CLOUDPROVIDER} == "aws" ]]; then + if [[ "${TF_VAR_create_load_balancer}" == true ]]; then + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw load_balancer_url > test_framework/load_balancer_url + fi + if [[ "${TF_VAR_k8s_distro_name}" == "k3s" ]]; then + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw instance_mapping | jq 'map({(.name | split(".")[0]): .id}) | add' | jq -s add > /tmp/instance_mapping + fi + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw controlplane_public_ip > /tmp/controlplane_public_ip +elif [[ ${LONGHORN_TEST_CLOUDPROVIDER} == "harvester" ]]; then + terraform output -raw kube_config > test_framework/kube_config.yaml fi -terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw controlplane_public_ip > /tmp/controlplane_public_ip - exit $? diff --git a/pipelines/utilities/vpn.sh b/pipelines/utilities/vpn.sh new file mode 100755 index 0000000000..e8c2088e89 --- /dev/null +++ b/pipelines/utilities/vpn.sh @@ -0,0 +1,10 @@ +connect_to_vpn(){ + mkdir -p /dev/net + mknod /dev/net/tun c 10 200 + chmod 600 /dev/net/tun + openvpn --config vpn.ovpn --daemon + sleep 10 + ps aux + cat /etc/resolv.conf + cat /var/log/openvpn.log +} diff --git a/test_framework/terraform/harvester/ubuntu/main.tf b/test_framework/terraform/harvester/ubuntu/main.tf new file mode 100644 index 0000000000..6985c6e871 --- /dev/null +++ b/test_framework/terraform/harvester/ubuntu/main.tf @@ -0,0 +1,142 @@ +terraform { + required_providers { + rancher2 = { + source = "rancher/rancher2" + version = "3.0.0-rc2" + } + } +} + +provider "rancher2" { + api_url = var.lab_url + insecure = true + access_key = var.lab_access_key + secret_key = var.lab_secret_key +} + +resource "random_string" "random_suffix" { + length = 8 + special = false + lower = true + upper = false +} + +data "rancher2_cluster_v2" "hal-cluster" { + name = "hal" +} + +resource "rancher2_cloud_credential" "e2e-credential" { + name = "e2e-credential-${random_string.random_suffix.id}" + harvester_credential_config { + cluster_id = data.rancher2_cluster_v2.hal-cluster.cluster_v1_id + cluster_type = "imported" + kubeconfig_content = data.rancher2_cluster_v2.hal-cluster.kube_config + } +} + +resource "rancher2_machine_config_v2" "e2e-machine-config" { + + generate_name = "e2e-machine-config-${random_string.random_suffix.id}" + + harvester_config { + + vm_namespace = "longhorn-qa" + + cpu_count = "4" + memory_size = "8" + + disk_info = <- + ${file(var.ssh_public_key_file_path)} +package_update: true +packages: + - qemu-guest-agent + - iptables +runcmd: + - - systemctl + - enable + - '--now' + - qemu-guest-agent.service +EOF + } +} + +resource "rancher2_cluster_v2" "e2e-cluster" { + + name = "e2e-cluster-${random_string.random_suffix.id}" + + kubernetes_version = var.k8s_distro_version + + rke_config { + machine_pools { + name = "control-plane-pool" + cloud_credential_secret_name = rancher2_cloud_credential.e2e-credential.id + control_plane_role = true + etcd_role = true + worker_role = false + quantity = 1 + machine_config { + kind = rancher2_machine_config_v2.e2e-machine-config.kind + name = rancher2_machine_config_v2.e2e-machine-config.name + } + } + machine_pools { + name = "worker-pool" + cloud_credential_secret_name = rancher2_cloud_credential.e2e-credential.id + control_plane_role = false + etcd_role = false + worker_role = true + quantity = 3 + machine_config { + kind = rancher2_machine_config_v2.e2e-machine-config.kind + name = rancher2_machine_config_v2.e2e-machine-config.name + } + } + machine_selector_config { + config = { + cloud-provider-name = "" + } + } + machine_global_config = <