Skip to content

Commit

Permalink
ci: config vpn
Browse files Browse the repository at this point in the history
Signed-off-by: Yang Chiu <[email protected]>
  • Loading branch information
yangchiu committed May 28, 2024
1 parent 099c719 commit f4a610b
Show file tree
Hide file tree
Showing 11 changed files with 401 additions and 12 deletions.
2 changes: 1 addition & 1 deletion pipelines/e2e/Dockerfile.setup
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_V
wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \
mv yq_linux_amd64 /usr/local/bin/yq && \
chmod +x /usr/local/bin/yq && \
apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip gcc python3-dev libc-dev py3-virtualenv docker && \
apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip gcc python3-dev libc-dev py3-virtualenv docker openvpn && \
ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
chmod 700 get_helm.sh && \
Expand Down
13 changes: 12 additions & 1 deletion pipelines/e2e/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ node {
usernamePassword(credentialsId: CREDS_ID, passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'),
string(credentialsId: 'DO_CREDS', variable: 'DO_TOKEN'),
string(credentialsId: REGISTRATION_CODE_ID, variable: 'REGISTRATION_CODE'),
file(credentialsId: 'vpn.ovpn', variable: 'VPN_CONFIG'),
file(credentialsId: 'login.conf', variable: 'LOGIN_CONFIG'),
usernamePassword(credentialsId: 'LAB_API_KEY', passwordVariable: 'LAB_SECRET_KEY', usernameVariable: 'LAB_ACCESS_KEY'),
string(credentialsId: 'LAB_URL', variable: 'LAB_URL'),
]) {

if (params.SEND_SLACK_NOTIFICATION) {
Expand Down Expand Up @@ -76,7 +80,8 @@ node {
echo "Using registration coce: $REGISTRATION_CODE_ID"

sh "pipelines/e2e/scripts/build.sh"
sh """ docker run -itd --name ${JOB_BASE_NAME}-${BUILD_NUMBER} \
sh """ docker run -itd --cap-add=NET_ADMIN \
--name ${JOB_BASE_NAME}-${BUILD_NUMBER} \
--env AIR_GAP_INSTALLATION=${AIR_GAP_INSTALLATION} \
--env REGISTRY_URL=${REGISTRY_URL} \
--env REGISTRY_USERNAME=${REGISTRY_USERNAME} \
Expand Down Expand Up @@ -118,11 +123,17 @@ node {
--env TF_VAR_cis_hardening=${CIS_HARDENING} \
--env TF_VAR_resources_owner=longhorn-long-running \
--env TF_VAR_extra_block_device=${RUN_V2_TEST} \
--env TF_VAR_lab_url=${LAB_URL} \
--env TF_VAR_lab_access_key=${LAB_ACCESS_KEY} \
--env TF_VAR_lab_secret_key=${LAB_SECRET_KEY} \
--env IMAGE_NAME=${imageName} \
-v /var/run/docker.sock:/var/run/docker.sock \
--mount source="vol-${imageName}",target=/tmp \
${imageName}
"""

sh "docker cp ${VPN_CONFIG} ${JOB_BASE_NAME}-${BUILD_NUMBER}:/src/longhorn-tests/vpn.ovpn"
sh "docker cp ${LOGIN_CONFIG} ${JOB_BASE_NAME}-${BUILD_NUMBER}:/src/longhorn-tests/login.conf"
}

timeout(60) {
Expand Down
135 changes: 135 additions & 0 deletions pipelines/e2e/scripts/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
terraform {
required_providers {
rancher2 = {
source = "rancher/rancher2"
version = "3.0.0-rc2"
}
}
}

provider "rancher2" {
api_url = ""
insecure = true
access_key = ""
secret_key = ""
}

data "rancher2_cluster_v2" "hal-cluster" {
name = "hal"
}

resource "rancher2_cloud_credential" "credential" {
name = "test-credential"
harvester_credential_config {
cluster_id = data.rancher2_cluster_v2.hal-cluster.cluster_v1_id
cluster_type = "imported"
kubeconfig_content = data.rancher2_cluster_v2.hal-cluster.kube_config
}
}

resource "rancher2_machine_config_v2" "test-machine-config" {

generate_name = "test-machine-config"

harvester_config {

vm_namespace = "longhorn-qa"

cpu_count = "4"
memory_size = "8"

disk_info = <<EOF
{
"disks": [{
"imageName": "longhorn-qa/image-nbv7f",
"size": 100,
"bootOrder": 1
}]
}
EOF

network_info = <<EOF
{
"interfaces": [{
"networkName": "longhorn-qa/vlan104"
}]
}
EOF

ssh_user = "ubuntu"

user_data = <<EOF
#cloud-config
ssh_authorized_keys:
- >-
ssh-rsa
package_update: true
packages:
- qemu-guest-agent
- iptables
runcmd:
- - systemctl
- enable
- '--now'
- qemu-guest-agent.service
EOF
}
}

resource "rancher2_cluster_v2" "test-cluster-0" {

name = "test-cluster-0"

kubernetes_version = "v1.27.13+rke2r1"

rke_config {
machine_pools {
name = "control-plane-pool"
cloud_credential_secret_name = rancher2_cloud_credential.credential.id
control_plane_role = true
etcd_role = true
worker_role = false
quantity = 1
machine_config {
kind = rancher2_machine_config_v2.test-machine-config.kind
name = rancher2_machine_config_v2.test-machine-config.name
}
}
machine_pools {
name = "worker-pool"
cloud_credential_secret_name = rancher2_cloud_credential.credential.id
control_plane_role = false
etcd_role = false
worker_role = true
quantity = 3
machine_config {
kind = rancher2_machine_config_v2.test-machine-config.kind
name = rancher2_machine_config_v2.test-machine-config.name
}
}
machine_selector_config {
config = {
cloud-provider-name = ""
}
}
machine_global_config = <<EOF
cni: "calico"
disable-kube-proxy: false
etcd-expose-metrics: false
EOF
upgrade_strategy {
control_plane_concurrency = "10%"
worker_concurrency = "10%"
}
etcd {
snapshot_schedule_cron = "0 */5 * * *"
snapshot_retention = 5
}
chart_values = ""
}
}

output "cluster" {
value = rancher2_cluster_v2.test-cluster-0.kube_config
sensitive = "true"
}
24 changes: 24 additions & 0 deletions pipelines/e2e/scripts/vpn
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
$ docker run --cap-add=NET_ADMIN -it alpine /bin/sh
/ # history
0 apk add openvpn
1 mkdir -p /dev/net
2 mknod /dev/net/tun c 10 200
3 chmod 600 /dev/net/tun
4 vi vpn.ovpn
#
script-security 2
up /etc/openvpn/up.sh
down /etc/openvpn/down.sh
#
5 vi login.conf
6 openvpn --config vpn.ovpn
7 openvpn --config vpn.ovpn --daemon
8 ps aux
9 cat /etc/resolv.conf
10 wget url
14 traceroute url
20 vi kubeconfig
23 wget -q https://storage.googleapis.com/kubernetes-release/release/v1.30.0/bin/linux/amd64/kubectl
24 mv kubectl /usr/local/bin/kubectl
25 chmod +x /usr/local/bin/kubectl
29 export KUBECONFIG=/kubeconfig
4 changes: 3 additions & 1 deletion pipelines/utilities/kubeconfig.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
set_kubeconfig(){
# rke2, rke and k3s all support amd64
# but only k3s supports arm64
if [[ "${TF_VAR_arch}" == "amd64" ]] ; then
if [[ "${LONGHORN_TEST_CLOUDPROVIDER}" == "harvester" ]]; then
export KUBECONFIG="${PWD}/test_framework/kube_config.yaml"
elif [[ "${TF_VAR_arch}" == "amd64" ]]; then
if [[ "${TF_VAR_k8s_distro_name}" == "rke" ]]; then
export KUBECONFIG="${PWD}/test_framework/kube_config_rke.yml"
elif [[ "${TF_VAR_k8s_distro_name}" == "rke2" ]]; then
Expand Down
6 changes: 5 additions & 1 deletion pipelines/utilities/longhorn_ui.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,9 @@ setup_longhorn_ui_nodeport(){
}

export_longhorn_ui_url(){
export LONGHORN_CLIENT_URL="http://$(cat /tmp/controlplane_public_ip):30000"
if [[ ${LONGHORN_TEST_CLOUDPROVIDER} == "harvester" ]]; then
export LONGHORN_CLIENT_URL="${TF_VAR_lab_url}/k8s/clusters/$(cat /tmp/cluster_id)/api/v1/namespaces/longhorn-system/services/http:longhorn-frontend:80/proxy"
else
export LONGHORN_CLIENT_URL="http://$(cat /tmp/controlplane_public_ip):30000"
fi
}
1 change: 1 addition & 0 deletions pipelines/utilities/run_longhorn_e2e_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ run_longhorn_e2e_test_out_of_cluster(){
cp "${KUBECONFIG}" /tmp/kubeconfig
CONTAINER_NAME="e2e-container-${IMAGE_NAME}"
docker run --pull=always \
--network=container:"${IMAGE_NAME}" \
--name "${CONTAINER_NAME}" \
-e LONGHORN_BACKUPSTORE="${LONGHORN_BACKUPSTORES}" \
-e LONGHORN_BACKUPSTORE_POLL_INTERVAL="${LONGHORN_BACKUPSTORE_POLL_INTERVAL}" \
Expand Down
24 changes: 16 additions & 8 deletions pipelines/utilities/terraform_setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

set -x

if [[ ${LONGHORN_TEST_CLOUDPROVIDER} == "harvester" ]]; then
source pipelines/utilities/vpn.sh
connect_to_vpn
fi

if [[ ${TF_VAR_arch} == "amd64" ]]; then
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} init
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} apply -auto-approve -no-color
Expand All @@ -16,14 +21,17 @@ else
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} apply -auto-approve -no-color
fi

if [[ "${TF_VAR_create_load_balancer}" == true ]]; then
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw load_balancer_url > test_framework/load_balancer_url
fi

if [[ "${TF_VAR_k8s_distro_name}" == "k3s" ]]; then
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw instance_mapping | jq 'map({(.name | split(".")[0]): .id}) | add' | jq -s add > /tmp/instance_mapping
if [[ ${LONGHORN_TEST_CLOUDPROVIDER} == "aws" ]]; then
if [[ "${TF_VAR_create_load_balancer}" == true ]]; then
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw load_balancer_url > test_framework/load_balancer_url
fi
if [[ "${TF_VAR_k8s_distro_name}" == "k3s" ]]; then
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw instance_mapping | jq 'map({(.name | split(".")[0]): .id}) | add' | jq -s add > /tmp/instance_mapping
fi
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw controlplane_public_ip > /tmp/controlplane_public_ip
elif [[ ${LONGHORN_TEST_CLOUDPROVIDER} == "harvester" ]]; then
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw kube_config > test_framework/kube_config.yaml
terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw cluster_id > /tmp/cluster_id
fi

terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw controlplane_public_ip > /tmp/controlplane_public_ip

exit $?
8 changes: 8 additions & 0 deletions pipelines/utilities/vpn.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
connect_to_vpn(){
mkdir -p /dev/net
mknod /dev/net/tun c 10 200
chmod 600 /dev/net/tun
openvpn --config vpn.ovpn --daemon
sleep 10
cat /var/log/openvpn.log
}
Loading

0 comments on commit f4a610b

Please sign in to comment.