Merge pull request #169 from kinvolk/tormath1/remove-rkt-kubelet-wrapper

remove rkt and kubelet-wrapper from kubernetes tests

kola/tests/kubernetes/basic.go

@@ -1,3 +1,4 @@
+// Copyright 2021 Kinvolk GmbH
 // Copyright 2015 CoreOS, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -51,7 +52,7 @@ func init() {
 				Name:        "google.kubernetes.basic." + r + "." + t,
 				Run:         f,
 				ClusterSize: 0,
-				Platforms:   []string{"gce", "do", "aws"}, // TODO: fix packet, esx
+				Platforms:   []string{"gce", "do", "aws", "qemu"}, // TODO: fix packet, esx
 				Distros:     []string{"cl"},
 			})
 		}
@@ -68,12 +69,6 @@ func CoreOSBasic(c cluster.TestCluster, version, runtime string) {
 	if err := nginxCheck(c, k.master, k.workers); err != nil {
 		c.Fatal(err)
 	}
-
-	// http://kubernetes.io/v1.0/docs/user-guide/secrets/ Also, ensures
-	// https://github.com/coreos/bugs/issues/447 does not re-occur.
-	if err := secretCheck(c, k.master, k.workers); err != nil {
-		c.Fatal(err)
-	}
 }
 
 func nodeCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error {
@@ -105,9 +100,18 @@ func nodeCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.
 
 func nginxCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error {
 	pod := strings.NewReader(nginxPodYAML)
+	secret := strings.NewReader(secretYAML)
 	if err := platform.InstallFile(pod, master, "./nginx-pod.yaml"); err != nil {
 		return err
 	}
+	if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil {
+		return err
+	}
+
+	if _, err := c.SSH(master, "./kubectl create -f secret.yaml"); err != nil {
+		return err
+	}
+
 	if _, err := c.SSH(master, "./kubectl create -f nginx-pod.yaml"); err != nil {
 		return err
 	}
@@ -135,57 +139,7 @@ func nginxCheck(c cluster.TestCluster, master platform.Machine, nodes []platform
 	return nil
 }
 
-func secretCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error {
-	// create yaml files
-	secret := strings.NewReader(secretYAML)
-	pod := strings.NewReader(secretPodYAML)
-	if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil {
-		return err
-	}
-	if err := platform.InstallFile(pod, master, "./secret-pod.yaml"); err != nil {
-		return err
-	}
-
-	if _, err := c.SSH(master, "./kubectl create -f secret.yaml"); err != nil {
-		return err
-	}
-	_, err := c.SSH(master, "./kubectl describe secret test-secret")
-	if err != nil {
-		return err
-	}
-
-	b, err := c.SSH(master, "./kubectl create -f secret-pod.yaml")
-	if err != nil {
-		return err
-	}
-	expectedOutput := "value-1"
-	if strings.Contains(strings.TrimSpace(string(b)), expectedOutput) {
-		return fmt.Errorf("error detecting secret pod")
-	}
-
-	return nil
-}
-
 const (
-	secretPodYAML = `apiVersion: v1
-kind: Pod
-metadata:
-  name: secret-test-pod
-spec:
-  containers:
-    - name: test-container
-      image: kubernetes/mounttest:0.1
-      command: [ "/mt", "--file_content=/etc/secret-volume/data-1" ]
-      volumeMounts:
-          # name must match the volume name below
-          - name: secret-volume
-            mountPath: /etc/secret-volume
-  volumes:
-    - name: secret-volume
-      secret:
-        secretName: test-secret
-  restartPolicy: Never`
-
 	secretYAML = `apiVersion: v1
 kind: Secret
 metadata:
@@ -205,5 +159,13 @@ spec:
   - name: nginx
     image: nginx
     ports:
-    - containerPort: 80`
+    - containerPort: 80
+    volumeMounts:
+      # name must match the volume name below
+      - name: secret-volume
+        mountPath: /etc/secret-volume
+  volumes:
+    - name: secret-volume
+      secret:
+        secretName: test-secret`
 )

kola/tests/kubernetes/controllerInstall.go

@@ -2,7 +2,11 @@ package kubernetes
 
 // https://github.com/coreos/coreos-kubernetes/tree/master/multi-node/generic.
 const controllerInstallScript = `#!/bin/bash
-set -e
+export CNI_VERSION="v0.9.1"
+
+# Download dir used to store the kubernetes
+# related components
+export DOWNLOAD_DIR=/opt/bin
 
 # List of etcd servers (http://ip:port), comma separated
 export ETCD_ENDPOINTS={{.ETCD_ENDPOINTS}}
@@ -36,7 +40,7 @@ export DNS_SERVICE_IP=192.168.128.10
 # Whether to use Calico for Kubernetes network policy.
 export USE_CALICO=false
 
-# Determines the container runtime for kubernetes to use. Accepts 'docker' or 'rkt'.
+# Determines the container runtime for kubernetes to use. Accepts 'docker'.
 export CONTAINER_RUNTIME={{.CONTAINER_RUNTIME}}
 
 # The above settings can optionally be overridden using an environment file:
@@ -92,37 +96,21 @@ function init_templates {
     if [ ! -f $TEMPLATE ]; then
         echo "TEMPLATE: $TEMPLATE"
         mkdir -p $(dirname $TEMPLATE)
-        RKT_FLAGS=""
-        if [[ ${CONTAINER_RUNTIME} = "rkt" ]]; then
-            RKT_FLAGS="--rkt-path=/usr/bin/rkt --rkt-stage1-image=coreos.com/rkt/stage1-coreos "
-        fi
-        KUBE_EXEC=""
-        if [[ $K8S_VER > "v1.18" ]]; then
-            KUBE_EXEC="kubelet"
-        fi
         cat << EOF > $TEMPLATE
 [Service]
-Environment=KUBELET_IMAGE_TAG=${K8S_VER}
-Environment=KUBELET_IMAGE_URL=docker://${HYPERKUBE_IMAGE_REPO}
-Environment="RKT_RUN_ARGS=--volume dns,kind=host,source=/etc/resolv.conf \
-  --mount volume=dns,target=/etc/resolv.conf \
-  --volume=rkt,kind=host,source=/opt/bin/host-rkt \
-  --mount volume=rkt,target=/usr/bin/rkt \
-  --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
-  --mount volume=var-lib-rkt,target=/var/lib/rkt \
-  --volume=stage,kind=host,source=/tmp \
-  --mount volume=stage,target=/tmp \
-  --insecure-options=image"
+Requires=docker.service
+After=docker.service
+ExecStartPre=/usr/bin/docker pull ${HYPERKUBE_IMAGE_REPO}:${K8S_VER}
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
-ExecStart=/usr/lib/flatcar/kubelet-wrapper ${KUBE_EXEC} \
+ExecStart=/opt/bin/kubelet \
   --register-schedulable=false \
   --kubeconfig=/etc/kubernetes/master-kubeconfig.yaml \
   --cni-conf-dir=/etc/kubernetes/cni/net.d \
   --network-plugin=cni \
   --container-runtime=${CONTAINER_RUNTIME} \
-  ${RKT_FLAGS} \
   --pod-manifest-path=/etc/kubernetes/manifests \
-  --hostname-override=${ADVERTISE_IP}
+  --hostname-override=${ADVERTISE_IP} \
+  --volume-plugin-dir=/opt/libexec/kubernetes/kubelet-plugins/volume/exec/
 Restart=always
 RestartSec=10
 CPUAccounting=true
@@ -156,65 +144,6 @@ current-context: kubelet-context
 EOF
     fi
 
-    local TEMPLATE=/opt/bin/host-rkt
-    if [ ! -f $TEMPLATE ]; then
-        echo "TEMPLATE: $TEMPLATE"
-        mkdir -p $(dirname $TEMPLATE)
-        cat << EOF > $TEMPLATE
-#!/bin/sh
-# This is bind mounted into the kubelet rootfs and all rkt shell-outs go
-# through this rkt wrapper. It essentially enters the host mount namespace
-# (which it is already in) only for the purpose of breaking out of the chroot
-# before calling rkt. It makes things like rkt gc work and avoids bind mounting
-# in certain rkt filesystem dependancies into the kubelet rootfs. This can
-# eventually be obviated when the write-api stuff gets upstream and rkt gc is
-# through the api-server. Related issue:
-# https://github.com/coreos/rkt/issues/2878
-exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@"
-EOF
-    fi
-
-
-    local TEMPLATE=/etc/systemd/system/load-rkt-stage1.service
-    if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then
-        echo "TEMPLATE: $TEMPLATE"
-        mkdir -p $(dirname $TEMPLATE)
-        cat << EOF > $TEMPLATE
-[Unit]
-Description=Load rkt stage1 images
-Documentation=http://github.com/coreos/rkt
-Requires=network-online.target
-After=network-online.target
-Before=rkt-api.service
-
-[Service]
-RemainAfterExit=yes
-Type=oneshot
-ExecStart=/usr/bin/rkt fetch /usr/lib/rkt/stage1-images/stage1-coreos.aci /usr/lib/rkt/stage1-images/stage1-fly.aci  --insecure-options=image
-
-[Install]
-RequiredBy=rkt-api.service
-EOF
-    fi
-
-    local TEMPLATE=/etc/systemd/system/rkt-api.service
-    if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then
-        echo "TEMPLATE: $TEMPLATE"
-        mkdir -p $(dirname $TEMPLATE)
-        cat << EOF > $TEMPLATE
-[Unit]
-Before=kubelet.service
-
-[Service]
-ExecStart=/usr/bin/rkt api-service
-Restart=always
-RestartSec=10
-
-[Install]
-RequiredBy=kubelet.service
-EOF
-    fi
-
     local TEMPLATE=/etc/systemd/system/calico-node.service
     if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then
         echo "TEMPLATE: $TEMPLATE"
@@ -262,8 +191,6 @@ kind: Pod
 metadata:
   name: kube-proxy
   namespace: kube-system
-  annotations:
-    rkt.alpha.kubernetes.io/stage1-name-override: coreos.com/rkt/stage1-fly
 spec:
   hostNetwork: true
   containers:
@@ -374,6 +301,7 @@ spec:
     - --leader-elect=true
     - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
     - --root-ca-file=/etc/kubernetes/ssl/ca.pem
+    - --flex-volume-plugin-dir=/opt/libexec/kubernetes/kubelet-plugins/volume/exec/
     resources:
       requests:
         cpu: 200m
@@ -587,31 +515,6 @@ EOF
                 "protocol": "TCP"
               }
             ]
-          },
-          {
-            "args": [
-              "-cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null",
-              "-port=8080",
-              "-quiet"
-            ],
-            "image": "gcr.io/google_containers/exechealthz-amd64:1.0",
-            "name": "healthz",
-            "ports": [
-              {
-                "containerPort": 8080,
-                "protocol": "TCP"
-              }
-            ],
-            "resources": {
-              "limits": {
-                "cpu": "10m",
-                "memory": "20Mi"
-              },
-              "requests": {
-                "cpu": "10m",
-                "memory": "20Mi"
-              }
-            }
           }
         ],
         "dnsPolicy": "Default"
@@ -994,24 +897,24 @@ function enable_calico_policy {
     curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/calico-system.json)" "http://127.0.0.1:8080/api/v1/namespaces/" > /dev/null
 }
 
+mkdir --parent /opt/cni/bin
+curl -sSL --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/kubelet
+curl -sSL "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz
+
+chmod +x kubelet
+mv kubelet $DOWNLOAD_DIR/
+
 init_config
 init_templates
 
-chmod +x /opt/bin/host-rkt
-
 init_flannel
 
 systemctl stop update-engine; systemctl mask update-engine
 
 systemctl daemon-reload
 
-if [ $CONTAINER_RUNTIME = "rkt" ]; then
-        systemctl enable load-rkt-stage1
-        systemctl enable rkt-api
-fi
-
-systemctl enable flanneld; systemctl start flanneld
-systemctl enable kubelet; systemctl start kubelet
+systemctl enable --now flanneld
+systemctl enable --now kubelet
 
 if [ $USE_CALICO = "true" ]; then
         systemctl enable calico-node; systemctl start calico-node

kola/tests/kubernetes/setup.go

@@ -268,8 +268,6 @@ func stripSemverSuffix(v string) (string, error) {
 
 // Run and configure the coreos-kubernetes generic install scripts.
 func runInstallScript(c cluster.TestCluster, m platform.Machine, script string, options map[string]string) {
-	c.MustSSH(m, "sudo stat /usr/lib/flatcar/kubelet-wrapper")
-
 	var buffer = new(bytes.Buffer)
 
 	tmpl, err := template.New("installScript").Parse(script)