kola/kubeadm/test: add testdata for script

this testdata will assert that the installation scripts are well rendered following the CNI, it also allows to help reproduce issues with the tests using directly the testdata Signed-off-by: Mathieu Tortuyaux <[email protected]>
flatcar · Jul 29, 2021 · 11d3514 · 11d3514
1 parent 4aabe45
commit 11d3514
Show file tree

Hide file tree

Showing 4 changed files with 357 additions and 0 deletions.
diff --git a/kola/tests/kubeadm/kubeadm_test.go b/kola/tests/kubeadm/kubeadm_test.go
@@ -16,6 +16,8 @@
 package kubeadm
 
 import (
+	"fmt"
+	"io/ioutil"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -47,4 +49,32 @@ func TestRenderTemplate(t *testing.T) {
 		assert.Equal(t, "Hello, world !", res.String())
 
 	})
+	t.Run("SuccessMasterScript", func(t *testing.T) {
+		for _, CNI := range CNIs {
+			res, err := render(
+				masterScript,
+				map[string]interface{}{
+					"CNI":            CNI,
+					"Endpoints":      []string{"http://1.2.3.4:2379"},
+					"Params":         "amd64",
+					"CNIVersion":     "v0.8.7",
+					"CRIctlVersion":  "v1.17.0",
+					"ReleaseVersion": "v0.4.0",
+					"Release":        "v1.21.0",
+					"DownloadDir":    "/opt/bin",
+					"PodSubnet":      "192.168.0.0/17",
+					"KubeadmSum":     "0673408403a3474c868ae86109f11f9114bca7ddce204be0d169316fb3ce0edefa4b2a472ba9b8308e423e6b927d4098ac36296405570f444f39551fb1c4bbb4",
+					"KubeletSum":     "530689c0cc32ef1830f7ae26ac10995f815043d48a905141e23a34a5e61522c4ee2ff46953648c47c5592d7c2ffa40ce90469a697f36f68475b8da5abd73f9f5",
+					"CRIctlSum":      "e258f4607a89b8d44c700036e636dd42cc3e2ed27a3bb13beef736f80f64f10b7974c01259a66131d3f7b44ed0c61b1ca0ea91597c416a9c095c432de5112d44",
+					"CNISum":         "8f2cbee3b5f94d59f919054dccfe99a8e3db5473b553d91da8af4763e811138533e05df4dbeab16b3f774852b4184a7994968f5e036a3f531ad1ac4620d10ede",
+					"KubectlSum":     "9557d298146ef62ffbcf05b3591bf1ce74f345628370447a4f614b5f64e367b5bfa8e397cc4755da9ea38f1ba04c95c65c313e735550ffc3b03c197e936c3e11",
+				},
+				false,
+			)
+			require.Nil(t, err)
+			script, err := ioutil.ReadFile(fmt.Sprintf("testdata/master-%s-script.sh", CNI))
+			require.Nil(t, err)
+			assert.Equal(t, string(script), res.String())
+		}
+	})
 }
diff --git a/kola/tests/kubeadm/testdata/master-calico-script.sh b/kola/tests/kubeadm/testdata/master-calico-script.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+set -euo pipefail
+
+export RELEASE_VERSION=v0.4.0
+export DOWNLOAD_DIR=/opt/bin
+export PATH="${PATH}:${DOWNLOAD_DIR}"
+
+# create the required directory
+mkdir --parent \
+    /etc/systemd/system/kubelet.service.d \
+    ${HOME}/.kube \
+    /home/core/.kube
+
+# we download and install the various requirements:
+# * kubelet service and kubeadm dropin
+
+curl --retry-delay 1 \
+    --retry 60 \
+    --retry-connrefused \
+    --retry-max-time 60 \
+    --connect-timeout 20 \
+    --fail \
+    -sSL \
+    "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" |
+    sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service
+
+curl --retry-delay 1 \
+    --retry 60 \
+    --retry-connrefused \
+    --retry-max-time 60 \
+    --connect-timeout 20 \
+    --fail \
+    -sSL \
+    "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" |
+    sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+# we create the kubeadm config
+# plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted
+# etcd is also defined as external. The provided one has some issues with docker and selinux
+# (permission denied with /var/lib/etcd) so it can't boot properly
+cat << EOF > kubeadm-config.yaml
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: InitConfiguration
+nodeRegistration:
+  kubeletExtraArgs:
+    volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+networking:
+  podSubnet: 192.168.0.0/17
+controllerManager:
+  extraArgs:
+    flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+etcd:
+  external:
+    endpoints:
+    
+      - http://1.2.3.4:2379
+    
+EOF
+
+
+cat << EOF > calico.yaml
+# Source: https://docs.projectcalico.org/manifests/custom-resources.yaml
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+      - blockSize: 26
+        cidr: 192.168.0.0/17
+        encapsulation: VXLANCrossSubnet
+        natOutgoing: Enabled
+        nodeSelector: all()
+  flexVolumePath: /opt/libexec/kubernetes/kubelet-plugins/volume/exec/
+EOF
+
+
+{
+    systemctl enable --quiet --now kubelet
+    kubeadm config images pull
+    kubeadm init --config kubeadm-config.yaml
+    cp /etc/kubernetes/admin.conf $HOME/.kube/config
+    cp /etc/kubernetes/admin.conf /home/core/.kube/config
+    chown -R core:core /home/core/.kube; chmod a+r /home/core/.kube/config;
+
+
+    kubectl create -f https://docs.projectcalico.org/manifests/tigera-operator.yaml
+    kubectl apply -f calico.yaml
+
+
+
+} 1>&2
+
+
+URL=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}')
+prefix="https://"
+short_url=${URL#"${prefix}"}
+token=$(kubeadm token create)
+certHashes=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')
+
+cat << EOF
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: ${short_url}
+    token: ${token}
+    caCertHashes:
+    - sha256:${certHashes}
+controlPlane:
+nodeRegistration:
+  kubeletExtraArgs:
+    volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+EOF
diff --git a/kola/tests/kubeadm/testdata/master-cilium-script.sh b/kola/tests/kubeadm/testdata/master-cilium-script.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+set -euo pipefail
+
+export RELEASE_VERSION=v0.4.0
+export DOWNLOAD_DIR=/opt/bin
+export PATH="${PATH}:${DOWNLOAD_DIR}"
+
+# create the required directory
+mkdir --parent \
+    /etc/systemd/system/kubelet.service.d \
+    ${HOME}/.kube \
+    /home/core/.kube
+
+# we download and install the various requirements:
+# * kubelet service and kubeadm dropin
+
+curl --retry-delay 1 \
+    --retry 60 \
+    --retry-connrefused \
+    --retry-max-time 60 \
+    --connect-timeout 20 \
+    --fail \
+    -sSL \
+    "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" |
+    sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service
+
+curl --retry-delay 1 \
+    --retry 60 \
+    --retry-connrefused \
+    --retry-max-time 60 \
+    --connect-timeout 20 \
+    --fail \
+    -sSL \
+    "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" |
+    sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+# we create the kubeadm config
+# plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted
+# etcd is also defined as external. The provided one has some issues with docker and selinux
+# (permission denied with /var/lib/etcd) so it can't boot properly
+cat << EOF > kubeadm-config.yaml
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: InitConfiguration
+nodeRegistration:
+  kubeletExtraArgs:
+    volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+networking:
+  podSubnet: 192.168.0.0/17
+controllerManager:
+  extraArgs:
+    flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+etcd:
+  external:
+    endpoints:
+    
+      - http://1.2.3.4:2379
+    
+EOF
+
+
+
+{
+    systemctl enable --quiet --now kubelet
+    kubeadm config images pull
+    kubeadm init --config kubeadm-config.yaml
+    cp /etc/kubernetes/admin.conf $HOME/.kube/config
+    cp /etc/kubernetes/admin.conf /home/core/.kube/config
+    chown -R core:core /home/core/.kube; chmod a+r /home/core/.kube/config;
+
+
+
+
+    sudo tar -xf /opt/bin/cilium.tar.gz -C /opt/bin
+    /opt/bin/cilium install \
+        --config enable-endpoint-routes=true \
+        --config cluster-pool-ipv4-cidr=192.168.0.0/17
+    # --wait will wait for status to report success
+    /opt/bin/cilium status --wait
+
+} 1>&2
+
+
+URL=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}')
+prefix="https://"
+short_url=${URL#"${prefix}"}
+token=$(kubeadm token create)
+certHashes=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')
+
+cat << EOF
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: ${short_url}
+    token: ${token}
+    caCertHashes:
+    - sha256:${certHashes}
+controlPlane:
+nodeRegistration:
+  kubeletExtraArgs:
+    volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+EOF
diff --git a/kola/tests/kubeadm/testdata/master-flannel-script.sh b/kola/tests/kubeadm/testdata/master-flannel-script.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+set -euo pipefail
+
+export RELEASE_VERSION=v0.4.0
+export DOWNLOAD_DIR=/opt/bin
+export PATH="${PATH}:${DOWNLOAD_DIR}"
+
+# create the required directory
+mkdir --parent \
+    /etc/systemd/system/kubelet.service.d \
+    ${HOME}/.kube \
+    /home/core/.kube
+
+# we download and install the various requirements:
+# * kubelet service and kubeadm dropin
+
+curl --retry-delay 1 \
+    --retry 60 \
+    --retry-connrefused \
+    --retry-max-time 60 \
+    --connect-timeout 20 \
+    --fail \
+    -sSL \
+    "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" |
+    sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service
+
+curl --retry-delay 1 \
+    --retry 60 \
+    --retry-connrefused \
+    --retry-max-time 60 \
+    --connect-timeout 20 \
+    --fail \
+    -sSL \
+    "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" |
+    sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+# we create the kubeadm config
+# plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted
+# etcd is also defined as external. The provided one has some issues with docker and selinux
+# (permission denied with /var/lib/etcd) so it can't boot properly
+cat << EOF > kubeadm-config.yaml
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: InitConfiguration
+nodeRegistration:
+  kubeletExtraArgs:
+    volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+networking:
+  podSubnet: 192.168.0.0/17
+controllerManager:
+  extraArgs:
+    flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+etcd:
+  external:
+    endpoints:
+    
+      - http://1.2.3.4:2379
+    
+EOF
+
+
+
+{
+    systemctl enable --quiet --now kubelet
+    kubeadm config images pull
+    kubeadm init --config kubeadm-config.yaml
+    cp /etc/kubernetes/admin.conf $HOME/.kube/config
+    cp /etc/kubernetes/admin.conf /home/core/.kube/config
+    chown -R core:core /home/core/.kube; chmod a+r /home/core/.kube/config;
+
+
+
+    curl -sSfL https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml > kube-flannel.yml
+    sed -i "s#10.244.0.0/16#192.168.0.0/17#" kube-flannel.yml
+    kubectl apply -f kube-flannel.yml
+
+
+} 1>&2
+
+
+URL=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}')
+prefix="https://"
+short_url=${URL#"${prefix}"}
+token=$(kubeadm token create)
+certHashes=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')
+
+cat << EOF
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: ${short_url}
+    token: ${token}
+    caCertHashes:
+    - sha256:${certHashes}
+controlPlane:
+nodeRegistration:
+  kubeletExtraArgs:
+    volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+EOF