diff --git a/kola/tests/kubeadm/kubeadm_test.go b/kola/tests/kubeadm/kubeadm_test.go index 2ce17e00c..63c4c5b95 100644 --- a/kola/tests/kubeadm/kubeadm_test.go +++ b/kola/tests/kubeadm/kubeadm_test.go @@ -16,6 +16,8 @@ package kubeadm import ( + "fmt" + "io/ioutil" "testing" "github.com/stretchr/testify/assert" @@ -47,4 +49,32 @@ func TestRenderTemplate(t *testing.T) { assert.Equal(t, "Hello, world !", res.String()) }) + t.Run("SuccessMasterScript", func(t *testing.T) { + for _, CNI := range CNIs { + res, err := render( + masterScript, + map[string]interface{}{ + "CNI": CNI, + "Endpoints": []string{"http://1.2.3.4:2379"}, + "Params": "amd64", + "CNIVersion": "v0.8.7", + "CRIctlVersion": "v1.17.0", + "ReleaseVersion": "v0.4.0", + "Release": "v1.21.0", + "DownloadDir": "/opt/bin", + "PodSubnet": "192.168.0.0/17", + "KubeadmSum": "0673408403a3474c868ae86109f11f9114bca7ddce204be0d169316fb3ce0edefa4b2a472ba9b8308e423e6b927d4098ac36296405570f444f39551fb1c4bbb4", + "KubeletSum": "530689c0cc32ef1830f7ae26ac10995f815043d48a905141e23a34a5e61522c4ee2ff46953648c47c5592d7c2ffa40ce90469a697f36f68475b8da5abd73f9f5", + "CRIctlSum": "e258f4607a89b8d44c700036e636dd42cc3e2ed27a3bb13beef736f80f64f10b7974c01259a66131d3f7b44ed0c61b1ca0ea91597c416a9c095c432de5112d44", + "CNISum": "8f2cbee3b5f94d59f919054dccfe99a8e3db5473b553d91da8af4763e811138533e05df4dbeab16b3f774852b4184a7994968f5e036a3f531ad1ac4620d10ede", + "KubectlSum": "9557d298146ef62ffbcf05b3591bf1ce74f345628370447a4f614b5f64e367b5bfa8e397cc4755da9ea38f1ba04c95c65c313e735550ffc3b03c197e936c3e11", + }, + false, + ) + require.Nil(t, err) + script, err := ioutil.ReadFile(fmt.Sprintf("testdata/master-%s-script.sh", CNI)) + require.Nil(t, err) + assert.Equal(t, string(script), res.String()) + } + }) } diff --git a/kola/tests/kubeadm/testdata/master-calico-script.sh b/kola/tests/kubeadm/testdata/master-calico-script.sh new file mode 100644 index 000000000..311c78d4e --- /dev/null +++ b/kola/tests/kubeadm/testdata/master-calico-script.sh @@ -0,0 +1,120 @@ +#!/bin/bash +set -euo pipefail + +export RELEASE_VERSION=v0.4.0 +export DOWNLOAD_DIR=/opt/bin +export PATH="${PATH}:${DOWNLOAD_DIR}" + +# create the required directory +mkdir --parent \ + /etc/systemd/system/kubelet.service.d \ + ${HOME}/.kube \ + /home/core/.kube + +# we download and install the various requirements: +# * kubelet service and kubeadm dropin + +curl --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + --fail \ + -sSL \ + "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | + sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service + +curl --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + --fail \ + -sSL \ + "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | + sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +# we create the kubeadm config +# plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted +# etcd is also defined as external. The provided one has some issues with docker and selinux +# (permission denied with /var/lib/etcd) so it can't boot properly +cat << EOF > kubeadm-config.yaml +apiVersion: kubeadm.k8s.io/v1beta2 +kind: InitConfiguration +nodeRegistration: + kubeletExtraArgs: + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: ClusterConfiguration +networking: + podSubnet: 192.168.0.0/17 +controllerManager: + extraArgs: + flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +etcd: + external: + endpoints: + + - http://1.2.3.4:2379 + +EOF + + +cat << EOF > calico.yaml +# Source: https://docs.projectcalico.org/manifests/custom-resources.yaml +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + # Note: The ipPools section cannot be modified post-install. + ipPools: + - blockSize: 26 + cidr: 192.168.0.0/17 + encapsulation: VXLANCrossSubnet + natOutgoing: Enabled + nodeSelector: all() + flexVolumePath: /opt/libexec/kubernetes/kubelet-plugins/volume/exec/ +EOF + + +{ + systemctl enable --quiet --now kubelet + kubeadm config images pull + kubeadm init --config kubeadm-config.yaml + cp /etc/kubernetes/admin.conf $HOME/.kube/config + cp /etc/kubernetes/admin.conf /home/core/.kube/config + chown -R core:core /home/core/.kube; chmod a+r /home/core/.kube/config; + + + kubectl create -f https://docs.projectcalico.org/manifests/tigera-operator.yaml + kubectl apply -f calico.yaml + + + +} 1>&2 + + +URL=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}') +prefix="https://" +short_url=${URL#"${prefix}"} +token=$(kubeadm token create) +certHashes=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //') + +cat << EOF +apiVersion: kubeadm.k8s.io/v1beta2 +kind: JoinConfiguration +discovery: + bootstrapToken: + apiServerEndpoint: ${short_url} + token: ${token} + caCertHashes: + - sha256:${certHashes} +controlPlane: +nodeRegistration: + kubeletExtraArgs: + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +EOF diff --git a/kola/tests/kubeadm/testdata/master-cilium-script.sh b/kola/tests/kubeadm/testdata/master-cilium-script.sh new file mode 100644 index 000000000..9ee6aac76 --- /dev/null +++ b/kola/tests/kubeadm/testdata/master-cilium-script.sh @@ -0,0 +1,105 @@ +#!/bin/bash +set -euo pipefail + +export RELEASE_VERSION=v0.4.0 +export DOWNLOAD_DIR=/opt/bin +export PATH="${PATH}:${DOWNLOAD_DIR}" + +# create the required directory +mkdir --parent \ + /etc/systemd/system/kubelet.service.d \ + ${HOME}/.kube \ + /home/core/.kube + +# we download and install the various requirements: +# * kubelet service and kubeadm dropin + +curl --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + --fail \ + -sSL \ + "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | + sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service + +curl --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + --fail \ + -sSL \ + "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | + sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +# we create the kubeadm config +# plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted +# etcd is also defined as external. The provided one has some issues with docker and selinux +# (permission denied with /var/lib/etcd) so it can't boot properly +cat << EOF > kubeadm-config.yaml +apiVersion: kubeadm.k8s.io/v1beta2 +kind: InitConfiguration +nodeRegistration: + kubeletExtraArgs: + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: ClusterConfiguration +networking: + podSubnet: 192.168.0.0/17 +controllerManager: + extraArgs: + flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +etcd: + external: + endpoints: + + - http://1.2.3.4:2379 + +EOF + + + +{ + systemctl enable --quiet --now kubelet + kubeadm config images pull + kubeadm init --config kubeadm-config.yaml + cp /etc/kubernetes/admin.conf $HOME/.kube/config + cp /etc/kubernetes/admin.conf /home/core/.kube/config + chown -R core:core /home/core/.kube; chmod a+r /home/core/.kube/config; + + + + + sudo tar -xf /opt/bin/cilium.tar.gz -C /opt/bin + /opt/bin/cilium install \ + --config enable-endpoint-routes=true \ + --config cluster-pool-ipv4-cidr=192.168.0.0/17 + # --wait will wait for status to report success + /opt/bin/cilium status --wait + +} 1>&2 + + +URL=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}') +prefix="https://" +short_url=${URL#"${prefix}"} +token=$(kubeadm token create) +certHashes=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //') + +cat << EOF +apiVersion: kubeadm.k8s.io/v1beta2 +kind: JoinConfiguration +discovery: + bootstrapToken: + apiServerEndpoint: ${short_url} + token: ${token} + caCertHashes: + - sha256:${certHashes} +controlPlane: +nodeRegistration: + kubeletExtraArgs: + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +EOF diff --git a/kola/tests/kubeadm/testdata/master-flannel-script.sh b/kola/tests/kubeadm/testdata/master-flannel-script.sh new file mode 100644 index 000000000..a25ca49a0 --- /dev/null +++ b/kola/tests/kubeadm/testdata/master-flannel-script.sh @@ -0,0 +1,102 @@ +#!/bin/bash +set -euo pipefail + +export RELEASE_VERSION=v0.4.0 +export DOWNLOAD_DIR=/opt/bin +export PATH="${PATH}:${DOWNLOAD_DIR}" + +# create the required directory +mkdir --parent \ + /etc/systemd/system/kubelet.service.d \ + ${HOME}/.kube \ + /home/core/.kube + +# we download and install the various requirements: +# * kubelet service and kubeadm dropin + +curl --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + --fail \ + -sSL \ + "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | + sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service + +curl --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + --fail \ + -sSL \ + "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | + sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +# we create the kubeadm config +# plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted +# etcd is also defined as external. The provided one has some issues with docker and selinux +# (permission denied with /var/lib/etcd) so it can't boot properly +cat << EOF > kubeadm-config.yaml +apiVersion: kubeadm.k8s.io/v1beta2 +kind: InitConfiguration +nodeRegistration: + kubeletExtraArgs: + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: ClusterConfiguration +networking: + podSubnet: 192.168.0.0/17 +controllerManager: + extraArgs: + flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +etcd: + external: + endpoints: + + - http://1.2.3.4:2379 + +EOF + + + +{ + systemctl enable --quiet --now kubelet + kubeadm config images pull + kubeadm init --config kubeadm-config.yaml + cp /etc/kubernetes/admin.conf $HOME/.kube/config + cp /etc/kubernetes/admin.conf /home/core/.kube/config + chown -R core:core /home/core/.kube; chmod a+r /home/core/.kube/config; + + + + curl -sSfL https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml > kube-flannel.yml + sed -i "s#10.244.0.0/16#192.168.0.0/17#" kube-flannel.yml + kubectl apply -f kube-flannel.yml + + +} 1>&2 + + +URL=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}') +prefix="https://" +short_url=${URL#"${prefix}"} +token=$(kubeadm token create) +certHashes=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //') + +cat << EOF +apiVersion: kubeadm.k8s.io/v1beta2 +kind: JoinConfiguration +discovery: + bootstrapToken: + apiServerEndpoint: ${short_url} + token: ${token} + caCertHashes: + - sha256:${certHashes} +controlPlane: +nodeRegistration: + kubeletExtraArgs: + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +EOF