From 78cc9eb7693496770cd4ecd69daf4f2717610bdf Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 10 Nov 2021 17:06:19 +0100 Subject: [PATCH 1/3] kola/test: remove legacy kubernetes test Signed-off-by: Mathieu Tortuyaux --- kola/tests/kubernetes/basic.go | 174 ---- kola/tests/kubernetes/controllerInstall.go | 925 --------------------- kola/tests/kubernetes/setup.go | 363 -------- kola/tests/kubernetes/workerInstall.go | 300 ------- 4 files changed, 1762 deletions(-) delete mode 100644 kola/tests/kubernetes/basic.go delete mode 100644 kola/tests/kubernetes/controllerInstall.go delete mode 100644 kola/tests/kubernetes/setup.go delete mode 100644 kola/tests/kubernetes/workerInstall.go diff --git a/kola/tests/kubernetes/basic.go b/kola/tests/kubernetes/basic.go deleted file mode 100644 index 9e77b976a..000000000 --- a/kola/tests/kubernetes/basic.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2021 Kinvolk GmbH -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubernetes - -import ( - "bytes" - "fmt" - "strings" - "time" - - "github.com/coreos/go-semver/semver" - "github.com/flatcar-linux/mantle/kola/cluster" - "github.com/flatcar-linux/mantle/kola/register" - "github.com/flatcar-linux/mantle/platform" - "github.com/flatcar-linux/mantle/util" -) - -// register a separate test for each version tag -var basicTags = []string{ - "v1.14.10", - "v1.16.8", - "v1.18.0", -} - -// regester each tag once per runtime -var runtimes = []string{ - "docker", -} - -func init() { - for i := range basicTags { - for j := range runtimes { - // use closure to store version and runtime in a Test - t, r := basicTags[i], runtimes[j] - f := func(c cluster.TestCluster) { - CoreOSBasic(c, t, r) - } - - register.Register(®ister.Test{ - Name: "google.kubernetes.basic." + r + "." + t, - Run: f, - ClusterSize: 0, - Platforms: []string{"gce", "do", "aws", "qemu", "azure"}, // TODO: fix packet, esx - Distros: []string{"cl"}, - // incompatible with docker >=20.10 - EndVersion: semver.Version{Major: 2956}, - }) - } - } -} - -// Run basic smoke tests on cluster. Assumes master is machine index 1, -// workers make up the rest. -func CoreOSBasic(c cluster.TestCluster, version, runtime string) { - // only one worker node to run on VMware which has max 3 machines for one test currently (the other two are one for etcd and one controller) - k := setupCluster(c, 1, version, runtime) - - // start nginx pod and curl endpoint - if err := nginxCheck(c, k.master, k.workers); err != nil { - c.Fatal(err) - } -} - -func nodeCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error { - b, err := c.SSH(master, "./kubectl get nodes") - if err != nil { - return err - } - - // parse kubectl output for IPs - addrMap := map[string]struct{}{} - for _, line := range strings.Split(string(b), "\n")[1:] { - addrMap[strings.SplitN(line, " ", 2)[0]] = struct{}{} - } - - // add master to node list because it is now normal to register - // master nodes but have it set as unschedulable in kubernetes v1.2+ - nodes = append(nodes, master) - - if len(addrMap) != len(nodes) { - return fmt.Errorf("cannot detect all nodes in kubectl output \n%v\n%v", addrMap, nodes) - } - for _, node := range nodes { - if _, ok := addrMap[node.PrivateIP()]; !ok { - return fmt.Errorf("node IP missing from kubectl get nodes") - } - } - return nil -} - -func nginxCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error { - pod := strings.NewReader(nginxPodYAML) - secret := strings.NewReader(secretYAML) - if err := platform.InstallFile(pod, master, "./nginx-pod.yaml"); err != nil { - return err - } - if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil { - return err - } - - if _, err := c.SSH(master, "./kubectl create -f secret.yaml"); err != nil { - return err - } - - if _, err := c.SSH(master, "./kubectl create -f nginx-pod.yaml"); err != nil { - return err - } - // wait for pod status to be 'Running' - podIsRunning := func() error { - b, err := c.SSH(master, "./kubectl get pod nginx --template={{.status.phase}}") - if err != nil { - return err - } - if !bytes.Equal(b, []byte("Running")) { - return fmt.Errorf("nginx pod not running: %s", b) - } - return nil - } - if err := util.Retry(10, 10*time.Second, podIsRunning); err != nil { - return err - } - - // delete pod - _, err := c.SSH(master, "./kubectl delete pods nginx") - if err != nil { - return err - } - - return nil -} - -const ( - secretYAML = `apiVersion: v1 -kind: Secret -metadata: - name: test-secret -data: - data-1: dmFsdWUtMQ0K - data-2: dmFsdWUtMg0KDQo=` - - nginxPodYAML = `apiVersion: v1 -kind: Pod -metadata: - name: nginx - labels: - app: nginx -spec: - containers: - - name: nginx - image: ghcr.io/kinvolk/nginx - ports: - - containerPort: 80 - volumeMounts: - # name must match the volume name below - - name: secret-volume - mountPath: /etc/secret-volume - volumes: - - name: secret-volume - secret: - secretName: test-secret` -) diff --git a/kola/tests/kubernetes/controllerInstall.go b/kola/tests/kubernetes/controllerInstall.go deleted file mode 100644 index 1db49e18e..000000000 --- a/kola/tests/kubernetes/controllerInstall.go +++ /dev/null @@ -1,925 +0,0 @@ -package kubernetes - -// https://github.com/coreos/coreos-kubernetes/tree/master/multi-node/generic. -const controllerInstallScript = `#!/bin/bash -export CNI_VERSION="v0.9.1" - -# Download dir used to store the kubernetes -# related components -export DOWNLOAD_DIR=/opt/bin - -# List of etcd servers (http://ip:port), comma separated -export ETCD_ENDPOINTS={{.ETCD_ENDPOINTS}} - -# Specify the version (vX.Y.Z) of Kubernetes assets to deploy -export K8S_VER={{.K8S_VER}} - -# Hyperkube image repository to use. -export HYPERKUBE_IMAGE_REPO={{.HYPERKUBE_IMAGE_REPO}} - -# The CIDR network to use for pod IPs. -# Each pod launched in the cluster will be assigned an IP out of this range. -# Each node will be configured such that these IPs will be routable using the flannel overlay network. -export POD_NETWORK=192.168.0.0/17 - -# The CIDR network to use for service cluster IPs. -# Each service will be assigned a cluster IP out of this range. -# This must not overlap with any IP ranges assigned to the POD_NETWORK, or other existing network infrastructure. -# Routing to these IPs is handled by a proxy service local to each node, and are not required to be routable between nodes. -export SERVICE_IP_RANGE=192.168.128.0/24 - -# The IP address of the Kubernetes API Service -# If the SERVICE_IP_RANGE is changed above, this must be set to the first IP in that range. -export K8S_SERVICE_IP=192.168.128.1 - -# The IP address of the cluster DNS service. -# This IP must be in the range of the SERVICE_IP_RANGE and cannot be the first IP in the range. -# This same IP must be configured on all worker nodes to enable DNS service discovery. -export DNS_SERVICE_IP=192.168.128.10 - -# Whether to use Calico for Kubernetes network policy. -export USE_CALICO=false - -# Determines the container runtime for kubernetes to use. Accepts 'docker'. -export CONTAINER_RUNTIME={{.CONTAINER_RUNTIME}} - -# The above settings can optionally be overridden using an environment file: -ENV_FILE=/run/coreos-kubernetes/options.env - -# ------------- - -function init_config { - local REQUIRED=('ADVERTISE_IP' 'POD_NETWORK' 'ETCD_ENDPOINTS' 'SERVICE_IP_RANGE' 'K8S_SERVICE_IP' 'DNS_SERVICE_IP' 'K8S_VER' 'HYPERKUBE_IMAGE_REPO' 'USE_CALICO') - - if [ -f $ENV_FILE ]; then - export $(cat $ENV_FILE | xargs) - fi - - if [ -z $ADVERTISE_IP ]; then - systemctl start coreos-metadata - export ADVERTISE_IP=$(cat /run/metadata/flatcar | grep -v IPV6 | grep IP | grep -E '(PRIVATE|LOCAL|DYNAMIC)' | cut -d = -f 2) - fi - - for REQ in "${REQUIRED[@]}"; do - if [ -z "$(eval echo \$$REQ)" ]; then - echo "Missing required config value: ${REQ}" - exit 1 - fi - done -} - -function init_flannel { - echo "Waiting for etcd..." - while true - do - IFS=',' read -ra ES <<< "$ETCD_ENDPOINTS" - for ETCD in "${ES[@]}"; do - echo "Trying: $ETCD" - if [ -n "$(curl --silent "$ETCD/v2/machines")" ]; then - local ACTIVE_ETCD=$ETCD - break - fi - sleep 1 - done - if [ -n "$ACTIVE_ETCD" ]; then - break - fi - done - RES=$(curl --silent -X PUT -d "value={\"Network\":\"$POD_NETWORK\",\"Backend\":{\"Type\":\"vxlan\"}}" "$ACTIVE_ETCD/v2/keys/coreos.com/network/config?prevExist=false") - if [ -z "$(echo $RES | grep '"action":"create"')" ] && [ -z "$(echo $RES | grep 'Key already exists')" ]; then - echo "Unexpected error configuring flannel pod network: $RES" - fi -} - -function init_templates { - local TEMPLATE=/etc/systemd/system/kubelet.service - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -[Service] -Requires=docker.service -After=docker.service -ExecStartPre=/usr/bin/docker pull ${HYPERKUBE_IMAGE_REPO}:${K8S_VER} -ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests -ExecStart=/opt/bin/kubelet \ - --register-schedulable=false \ - --kubeconfig=/etc/kubernetes/master-kubeconfig.yaml \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --container-runtime=${CONTAINER_RUNTIME} \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --hostname-override=${ADVERTISE_IP} \ - --volume-plugin-dir=/opt/libexec/kubernetes/kubelet-plugins/volume/exec/ -Restart=always -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -[Install] -WantedBy=multi-user.target -EOF - fi - - local TEMPLATE=/etc/kubernetes/master-kubeconfig.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -kind: Config -clusters: -- name: local - cluster: - server: http://127.0.0.1:8080 - clusterDNS: ${DNS_SERVICE_IP} - clusterDomain: cluster.local -users: -- name: kubelet -contexts: -- context: - cluster: local - user: kubelet - name: kubelet-context -current-context: kubelet-context -EOF - fi - - local TEMPLATE=/etc/systemd/system/calico-node.service - if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -[Unit] -Description=Calico per-host agent -Requires=network-online.target -After=network-online.target - -[Service] -Slice=machine.slice -Environment=CALICO_DISABLE_FILE_LOGGING=true -Environment=HOSTNAME=${ADVERTISE_IP} -Environment=IP=${ADVERTISE_IP} -Environment=FELIX_FELIXHOSTNAME=${ADVERTISE_IP} -Environment=CALICO_NETWORKING=false -Environment=NO_DEFAULT_POOLS=true -Environment=ETCD_ENDPOINTS=${ETCD_ENDPOINTS} -ExecStart=/usr/bin/rkt run --inherit-env --stage1-from-dir=stage1-fly.aci \ ---volume=modules,kind=host,source=/lib/modules,readOnly=false \ ---mount=volume=modules,target=/lib/modules \ ---trust-keys-from-https --insecure-options=image docker://quay.io/calico/node:v0.19.0 -KillMode=mixed -Restart=always -TimeoutStartSec=0 - -[Install] -WantedBy=multi-user.target -EOF - fi - - KUBE_PREFIX="" - if [[ $K8S_VER > "v1.16" ]]; then - KUBE_PREFIX="kube-" - fi - - local TEMPLATE=/etc/kubernetes/manifests/kube-proxy.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Pod -metadata: - name: kube-proxy - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-proxy - image: ${HYPERKUBE_IMAGE_REPO}:$K8S_VER - command: - - /hyperkube - - ${KUBE_PREFIX}proxy - - --master=http://127.0.0.1:8080 - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/ssl/certs - name: ssl-certs-host - readOnly: true - - mountPath: /var/run/dbus - name: dbus - readOnly: false - volumes: - - hostPath: - path: /usr/share/ca-certificates - name: ssl-certs-host - - hostPath: - path: /var/run/dbus - name: dbus -EOF - fi - - local TEMPLATE=/etc/kubernetes/manifests/kube-apiserver.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Pod -metadata: - name: kube-apiserver - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-apiserver - image: ${HYPERKUBE_IMAGE_REPO}:$K8S_VER - command: - - /hyperkube - - ${KUBE_PREFIX}apiserver - - --bind-address=0.0.0.0 - - --etcd-servers=${ETCD_ENDPOINTS} - - --allow-privileged=true - - --service-cluster-ip-range=${SERVICE_IP_RANGE} - - --secure-port=443 - - --advertise-address=${ADVERTISE_IP} - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota - - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem - - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem - - --client-ca-file=/etc/kubernetes/ssl/ca.pem - - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem - - --runtime-config=extensions/v1beta1/networkpolicies=true - livenessProbe: - httpGet: - host: 127.0.0.1 - port: 8080 - path: /healthz - initialDelaySeconds: 15 - timeoutSeconds: 15 - ports: - - containerPort: 443 - hostPort: 443 - name: https - - containerPort: 8080 - hostPort: 8080 - name: local - volumeMounts: - - mountPath: /etc/kubernetes/ssl - name: ssl-certs-kubernetes - readOnly: true - - mountPath: /etc/ssl/certs - name: ssl-certs-host - readOnly: true - volumes: - - hostPath: - path: /etc/kubernetes/ssl - name: ssl-certs-kubernetes - - hostPath: - path: /usr/share/ca-certificates - name: ssl-certs-host -EOF - fi - - local TEMPLATE=/etc/kubernetes/manifests/kube-controller-manager.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Pod -metadata: - name: kube-controller-manager - namespace: kube-system -spec: - containers: - - name: kube-controller-manager - image: ${HYPERKUBE_IMAGE_REPO}:$K8S_VER - command: - - /hyperkube - - ${KUBE_PREFIX}controller-manager - - --master=http://127.0.0.1:8080 - - --leader-elect=true - - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem - - --root-ca-file=/etc/kubernetes/ssl/ca.pem - - --flex-volume-plugin-dir=/opt/libexec/kubernetes/kubelet-plugins/volume/exec/ - resources: - requests: - cpu: 200m - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10252 - initialDelaySeconds: 15 - timeoutSeconds: 15 - volumeMounts: - - mountPath: /etc/kubernetes/ssl - name: ssl-certs-kubernetes - readOnly: true - - mountPath: /etc/ssl/certs - name: ssl-certs-host - readOnly: true - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/ssl - name: ssl-certs-kubernetes - - hostPath: - path: /usr/share/ca-certificates - name: ssl-certs-host -EOF - fi - - local TEMPLATE=/etc/kubernetes/manifests/kube-scheduler.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Pod -metadata: - name: kube-scheduler - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-scheduler - image: ${HYPERKUBE_IMAGE_REPO}:$K8S_VER - command: - - /hyperkube - - ${KUBE_PREFIX}scheduler - - --master=http://127.0.0.1:8080 - - --leader-elect=true - resources: - requests: - cpu: 100m - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10251 - initialDelaySeconds: 15 - timeoutSeconds: 15 -EOF - fi - - local TEMPLATE=/etc/kubernetes/manifests/calico-policy-controller.yaml - if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Pod -metadata: - name: calico-policy-controller - namespace: calico-system -spec: - hostNetwork: true - containers: - # The Calico policy controller. - - name: kube-policy-controller - image: calico/kube-policy-controller:v0.2.0 - env: - - name: ETCD_ENDPOINTS - value: "${ETCD_ENDPOINTS}" - - name: K8S_API - value: "http://127.0.0.1:8080" - - name: LEADER_ELECTION - value: "true" - # Leader election container used by the policy controller. - - name: leader-elector - image: quay.io/calico/leader-elector:v0.1.0 - imagePullPolicy: IfNotPresent - args: - - "--election=calico-policy-election" - - "--election-namespace=calico-system" - - "--http=127.0.0.1:4040" -EOF - fi - - local TEMPLATE=/srv/kubernetes/manifests/calico-system.json - if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": "calico-system" - } -} -EOF - fi - - local TEMPLATE=/srv/kubernetes/manifests/kube-dns-rc.json - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "apiVersion": "v1", - "kind": "ReplicationController", - "metadata": { - "labels": { - "k8s-app": "kube-dns", - "kubernetes.io/cluster-service": "true", - "version": "v15" - }, - "name": "kube-dns-v15", - "namespace": "kube-system" - }, - "spec": { - "replicas": 1, - "selector": { - "k8s-app": "kube-dns", - "version": "v15" - }, - "template": { - "metadata": { - "labels": { - "k8s-app": "kube-dns", - "kubernetes.io/cluster-service": "true", - "version": "v15" - } - }, - "spec": { - "containers": [ - { - "args": [ - "--domain=cluster.local.", - "--dns-port=10053" - ], - "image": "gcr.io/google_containers/kubedns-amd64:1.3", - "livenessProbe": { - "failureThreshold": 5, - "httpGet": { - "path": "/healthz", - "port": 8080, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "kubedns", - "ports": [ - { - "containerPort": 10053, - "name": "dns-local", - "protocol": "UDP" - }, - { - "containerPort": 10053, - "name": "dns-tcp-local", - "protocol": "TCP" - } - ], - "readinessProbe": { - "httpGet": { - "path": "/readiness", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 30, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "cpu": "100m", - "memory": "200Mi" - }, - "requests": { - "cpu": "100m", - "memory": "50Mi" - } - } - }, - { - "args": [ - "--cache-size=1000", - "--no-resolv", - "--server=127.0.0.1#10053" - ], - "image": "gcr.io/google_containers/kube-dnsmasq-amd64:1.3", - "name": "dnsmasq", - "ports": [ - { - "containerPort": 53, - "name": "dns", - "protocol": "UDP" - }, - { - "containerPort": 53, - "name": "dns-tcp", - "protocol": "TCP" - } - ] - } - ], - "dnsPolicy": "Default" - } - } - } -} -EOF - fi - - local TEMPLATE=/srv/kubernetes/manifests/kube-dns-svc.json - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "labels": { - "k8s-app": "kube-dns", - "kubernetes.io/cluster-service": "true", - "kubernetes.io/name": "KubeDNS" - }, - "name": "kube-dns", - "namespace": "kube-system" - }, - "spec": { - "clusterIP": "$DNS_SERVICE_IP", - "ports": [ - { - "name": "dns", - "port": 53, - "protocol": "UDP" - }, - { - "name": "dns-tcp", - "port": 53, - "protocol": "TCP" - } - ], - "selector": { - "k8s-app": "kube-dns" - } - } -} -EOF - fi - - local TEMPLATE=/srv/kubernetes/manifests/heapster-de.json - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "labels": { - "k8s-app": "heapster", - "kubernetes.io/cluster-service": "true", - "version": "v1.1.0" - }, - "name": "heapster-v1.1.0", - "namespace": "kube-system" - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": { - "k8s-app": "heapster", - "version": "v1.1.0" - } - }, - "template": { - "metadata": { - "labels": { - "k8s-app": "heapster", - "version": "v1.1.0" - } - }, - "spec": { - "containers": [ - { - "command": [ - "/heapster", - "--source=kubernetes.summary_api:''" - ], - "image": "gcr.io/google_containers/heapster:v1.1.0", - "name": "heapster", - "resources": { - "limits": { - "cpu": "100m", - "memory": "200Mi" - }, - "requests": { - "cpu": "100m", - "memory": "200Mi" - } - } - }, - { - "command": [ - "/pod_nanny", - "--cpu=100m", - "--extra-cpu=0.5m", - "--memory=200Mi", - "--extra-memory=4Mi", - "--threshold=5", - "--deployment=heapster-v1.1.0", - "--container=heapster", - "--poll-period=300000", - "--estimator=exponential" - ], - "env": [ - { - "name": "MY_POD_NAME", - "valueFrom": { - "fieldRef": { - "fieldPath": "metadata.name" - } - } - }, - { - "name": "MY_POD_NAMESPACE", - "valueFrom": { - "fieldRef": { - "fieldPath": "metadata.namespace" - } - } - } - ], - "image": "gcr.io/google_containers/addon-resizer:1.3", - "name": "heapster-nanny", - "resources": { - "limits": { - "cpu": "50m", - "memory": "100Mi" - }, - "requests": { - "cpu": "50m", - "memory": "100Mi" - } - } - } - ] - } - } - } -} -EOF - fi - - local TEMPLATE=/srv/kubernetes/manifests/heapster-svc.json - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "labels": { - "kubernetes.io/cluster-service": "true", - "kubernetes.io/name": "Heapster" - }, - "name": "heapster", - "namespace": "kube-system" - }, - "spec": { - "ports": [ - { - "port": 80, - "targetPort": 8082 - } - ], - "selector": { - "k8s-app": "heapster" - } - } -} -EOF - fi - - local TEMPLATE=/srv/kubernetes/manifests/kube-dashboard-rc.json - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "apiVersion": "v1", - "kind": "ReplicationController", - "metadata": { - "labels": { - "k8s-app": "kubernetes-dashboard", - "kubernetes.io/cluster-service": "true", - "version": "v1.1.0" - }, - "name": "kubernetes-dashboard-v1.1.0", - "namespace": "kube-system" - }, - "spec": { - "replicas": 1, - "selector": { - "k8s-app": "kubernetes-dashboard" - }, - "template": { - "metadata": { - "labels": { - "k8s-app": "kubernetes-dashboard", - "kubernetes.io/cluster-service": "true", - "version": "v1.1.0" - } - }, - "spec": { - "containers": [ - { - "image": "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.1.0", - "livenessProbe": { - "httpGet": { - "path": "/", - "port": 9090 - }, - "initialDelaySeconds": 30, - "timeoutSeconds": 30 - }, - "name": "kubernetes-dashboard", - "ports": [ - { - "containerPort": 9090 - } - ], - "resources": { - "limits": { - "cpu": "100m", - "memory": "50Mi" - }, - "requests": { - "cpu": "100m", - "memory": "50Mi" - } - } - } - ] - } - } - } -} -EOF - fi - - local TEMPLATE=/srv/kubernetes/manifests/kube-dashboard-svc.json - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "labels": { - "k8s-app": "kubernetes-dashboard", - "kubernetes.io/cluster-service": "true" - }, - "name": "kubernetes-dashboard", - "namespace": "kube-system" - }, - "spec": { - "ports": [ - { - "port": 80, - "targetPort": 9090 - } - ], - "selector": { - "k8s-app": "kubernetes-dashboard" - } - } -} -EOF - fi - - local TEMPLATE=/run/flannel/options.env - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -FLANNELD_IFACE=$ADVERTISE_IP -FLANNELD_ETCD_ENDPOINTS=$ETCD_ENDPOINTS -EOF - fi - - local TEMPLATE=/etc/systemd/system/docker.service.d/40-flannel.conf - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -[Service] -EnvironmentFile=/etc/kubernetes/cni/docker_opts_cni.env -EOF - fi - - local TEMPLATE=/etc/kubernetes/cni/docker_opts_cni.env - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -DOCKER_OPT_BIP="" -DOCKER_OPT_IPMASQ="" -EOF - fi - - local TEMPLATE=/etc/kubernetes/cni/net.d/10-calico.conf - if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "name": "calico", - "cniVersion": "0.2.0", - "type": "flannel", - "delegate": { - "type": "calico", - "etcd_endpoints": "$ETCD_ENDPOINTS", - "log_level": "none", - "log_level_stderr": "info", - "hostname": "${ADVERTISE_IP}", - "policy": { - "type": "k8s", - "k8s_api_root": "http://127.0.0.1:8080/api/v1/" - } - } -} -EOF - fi - - local TEMPLATE=/etc/kubernetes/cni/net.d/10-flannel.conf - if [ "${USE_CALICO}" = "false" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "name": "podnet", - "cniVersion": "0.2.0", - "type": "flannel", - "delegate": { - "isDefaultGateway": true - } -} -EOF - fi -} - -function start_addons { - echo "Waiting for Kubernetes API..." - until curl --silent "http://127.0.0.1:8080/version" - do - sleep 5 - done - echo - echo "K8S: DNS addon" - curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dns-rc.json)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/replicationcontrollers" > /dev/null - curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dns-svc.json)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/services" > /dev/null - echo "K8S: Heapster addon" - curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/heapster-de.json)" "http://127.0.0.1:8080/apis/extensions/v1beta1/namespaces/kube-system/deployments" > /dev/null - curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/heapster-svc.json)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/services" > /dev/null - echo "K8S: Dashboard addon" - curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dashboard-rc.json)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/replicationcontrollers" > /dev/null - curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dashboard-svc.json)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/services" > /dev/null -} - -function enable_calico_policy { - echo "Waiting for Kubernetes API..." - until curl --silent "http://127.0.0.1:8080/version" - do - sleep 5 - done - echo - echo "K8S: Calico Policy" - curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/calico-system.json)" "http://127.0.0.1:8080/api/v1/namespaces/" > /dev/null -} - -mkdir --parent /opt/cni/bin -curl -sSL --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/kubelet -curl -sSL "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - -chmod +x kubelet -mv kubelet $DOWNLOAD_DIR/ - -init_config -init_templates - -init_flannel - -systemctl stop update-engine; systemctl mask update-engine - -systemctl daemon-reload - -systemctl enable --now flanneld -systemctl enable --now kubelet - -if [ $USE_CALICO = "true" ]; then - systemctl enable calico-node; systemctl start calico-node - enable_calico_policy -fi - -start_addons -echo "DONE"` diff --git a/kola/tests/kubernetes/setup.go b/kola/tests/kubernetes/setup.go deleted file mode 100644 index 583f7b39b..000000000 --- a/kola/tests/kubernetes/setup.go +++ /dev/null @@ -1,363 +0,0 @@ -// Copyright 2016 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubernetes - -import ( - "bytes" - "fmt" - "regexp" - "strings" - "text/template" - "time" - - "github.com/coreos/pkg/capnslog" - - "github.com/flatcar-linux/mantle/kola/cluster" - "github.com/flatcar-linux/mantle/kola/tests/etcd" - "github.com/flatcar-linux/mantle/platform" - "github.com/flatcar-linux/mantle/platform/conf" - "github.com/flatcar-linux/mantle/util" -) - -var plog = capnslog.NewPackageLogger("github.com/flatcar-linux/mantle", "kola/tests/kubernetes") - -// kCluster just keeps track of which machines are which in a -// platform.TestCluster with kubernetes running. -type kCluster struct { - etcd platform.Machine - master platform.Machine - workers []platform.Machine -} - -// resolve ambiguity with TestCluster.Name() -type clusterWrapper struct { - *cluster.TestCluster -} - -func (cw clusterWrapper) Name() string { - return cw.Cluster.Name() -} - -// Setup a multi-node cluster based on generic scrips from coreos-kubernetes repo. -// https://github.com/coreos/coreos-kubernetes/tree/master/multi-node/generic -func setupCluster(c cluster.TestCluster, nodes int, version, runtime string) *kCluster { - plog.Infof("Creating single-node etcd") // etcd is on a separate machine to ensures flannel works without a local etcd - etcdNode, err := c.NewMachine(etcdConfig) - if err != nil { - c.Fatalf("error creating etcd: %v", err) - } - - if err := etcd.GetClusterHealth(c, etcdNode, 1); err != nil { - c.Fatalf("error checking etcd health: %v", err) - } - - plog.Infof("Creating master node") - master, err := c.NewMachine(noLocalEtcdConfig) - if err != nil { - c.Fatalf("error creating master: %v", err) - } - - options := map[string]string{ - "HYPERKUBE_IMAGE_REPO": "gcr.io/google-containers/hyperkube", - "MASTER_HOST": master.PrivateIP(), - "ETCD_ENDPOINTS": fmt.Sprintf("http://%v:2379", etcdNode.PrivateIP()), - "CONTROLLER_ENDPOINT": fmt.Sprintf("https://%v:443", master.PrivateIP()), - "K8S_SERVICE_IP": "192.168.128.1", - "K8S_VER": version, - "CONTAINER_RUNTIME": runtime, - } - - plog.Infof("Generating TLS assets on the master") - if err := generateMasterTLSAssets(c, master, options); err != nil { - c.Fatalf("error creating master tls: %v", err) - } - - plog.Infof("Creating worker nodes") - workers, err := platform.NewMachines(clusterWrapper{&c}, nil, nodes) - if err != nil { - c.Fatalf("error creating workers: %v", err) - } - - // generate tls assets on workers by transfering ca from master - plog.Infof("Generating TLS assets on the workers") - if err := generateWorkerTLSAssets(c, master, workers); err != nil { - c.Fatalf("error creating worker tls: %v", err) - } - - plog.Infof("Configuring nodes by running the install scripts") - runInstallScript(c, master, controllerInstallScript, options) - - for _, worker := range workers { - runInstallScript(c, worker, workerInstallScript, options) - } - - plog.Infof("Configuring kubectl on the master") - if err := configureKubectl(c, master, master.PrivateIP(), version); err != nil { - c.Fatalf("error configuring master kubectl: %v", err) - } - - plog.Infof("Waiting for all nodes to appear on kubectl") - f := func() error { - return nodeCheck(c, master, workers) - } - if err := util.Retry(15, 30*time.Second, f); err != nil { - c.Fatalf("error waiting for nodes: %v", err) - } - - cluster := &kCluster{ - etcd: etcdNode, - master: master, - workers: workers, - } - return cluster -} - -func generateMasterTLSAssets(c cluster.TestCluster, master platform.Machine, options map[string]string) error { - var buffer = new(bytes.Buffer) - - tmpl, err := template.New("masterCNF").Parse(masterCNF) - if err != nil { - return err - } - if err := tmpl.Execute(buffer, options); err != nil { - return err - } - - if err := platform.InstallFile(buffer, master, "/home/core/openssl.cnf"); err != nil { - return err - } - - var cmds = []string{ - // gen master assets - "openssl genrsa -out ca-key.pem 2048", - `openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"`, - "openssl genrsa -out apiserver-key.pem 2048", - `openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf`, - "openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf", - - // gen cluster admin keypair - "openssl genrsa -out admin-key.pem 2048", - `openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"`, - "openssl x509 -req -in admin.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin.pem -days 365", - - // move into /etc/kubernetes/ssl - "sudo mkdir -p /etc/kubernetes/ssl", - "sudo cp /home/core/ca.pem /etc/kubernetes/ssl/ca.pem", - "sudo cp /home/core/apiserver.pem /etc/kubernetes/ssl/apiserver.pem", - "sudo cp /home/core/apiserver-key.pem /etc/kubernetes/ssl/apiserver-key.pem", - "sudo chmod 600 /etc/kubernetes/ssl/*-key.pem", - "sudo chown root:root /etc/kubernetes/ssl/*-key.pem", - } - - for _, cmd := range cmds { - b, err := c.SSH(master, cmd) - if err != nil { - return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b) - } - } - return nil -} - -func generateWorkerTLSAssets(c cluster.TestCluster, master platform.Machine, workers []platform.Machine) error { - for i, worker := range workers { - // copy tls assets from master to workers - err := platform.TransferFile(master, "/etc/kubernetes/ssl/ca.pem", worker, "/home/core/ca.pem") - if err != nil { - return err - } - err = platform.TransferFile(master, "/home/core/ca-key.pem", worker, "/home/core/ca-key.pem") - if err != nil { - return err - } - - // place worker-openssl.cnf on workers - cnf := strings.Replace(workerCNF, "{{.WORKER_IP}}", worker.PrivateIP(), -1) - in := strings.NewReader(cnf) - if err := platform.InstallFile(in, worker, "/home/core/worker-openssl.cnf"); err != nil { - return err - } - - // gen certs - workerFQDN := fmt.Sprintf("kube-worker-%v", i) - cmds := []string{ - fmt.Sprintf("openssl genrsa -out worker-key.pem 2048"), - fmt.Sprintf(`openssl req -new -key worker-key.pem -out %v-worker.csr -subj "/CN=%v" -config worker-openssl.cnf`, workerFQDN, workerFQDN), - fmt.Sprintf(`openssl x509 -req -in %v-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out worker.pem -days 365 -extensions v3_req -extfile worker-openssl.cnf`, workerFQDN), - - // move into /etc/kubernetes/ssl - "sudo mkdir -p /etc/kubernetes/ssl", - "sudo chmod 600 /home/core/*-key.pem", - "sudo chown root:root /home/core/*-key.pem", - "sudo cp /home/core/worker.pem /etc/kubernetes/ssl/worker.pem", - "sudo cp /home/core/worker-key.pem /etc/kubernetes/ssl/worker-key.pem", - "sudo cp /home/core/ca.pem /etc/kubernetes/ssl/ca.pem", - } - - for _, cmd := range cmds { - b, err := c.SSH(worker, cmd) - if err != nil { - return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b) - } - } - } - return nil -} - -// https://coreos.com/kubernetes/docs/latest/configure-kubectl.html -func configureKubectl(c cluster.TestCluster, m platform.Machine, server string, version string) error { - // ignore suffix like '-coreos.1' to grab upstream kubelet - version, err := stripSemverSuffix(version) - if err != nil { - return err - } - - var ( - ca = "/home/core/ca.pem" - adminKey = "/home/core/admin-key.pem" - adminCert = "/home/core/admin.pem" - kubeURL = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%v/bin/linux/amd64/kubectl", version) - ) - - if _, err := c.SSH(m, "wget -q "+kubeURL); err != nil { - return err - } - if _, err := c.SSH(m, "chmod +x ./kubectl"); err != nil { - return err - } - - // cmds to configure kubectl - cmds := []string{ - fmt.Sprintf("./kubectl config set-cluster default-cluster --server=https://%v --certificate-authority=%v", server, ca), - fmt.Sprintf("./kubectl config set-credentials default-admin --certificate-authority=%v --client-key=%v --client-certificate=%v", ca, adminKey, adminCert), - "./kubectl config set-context default-system --cluster=default-cluster --user=default-admin", - "./kubectl config use-context default-system", - } - for _, cmd := range cmds { - b, err := c.SSH(m, cmd) - if err != nil { - return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b) - } - } - return nil -} - -var semverPrefix = regexp.MustCompile(`^v[\d]+\.[\d]+\.[\d]+`) - -// Strip semver suffix -- e.g., v1.1.8_coreos.1 --> v1.1.8. If no match -// found, return error. -func stripSemverSuffix(v string) (string, error) { - v = semverPrefix.FindString(v) - if v == "" { - return "", fmt.Errorf("error stripping semver suffix") - } - - return v, nil -} - -// Run and configure the coreos-kubernetes generic install scripts. -func runInstallScript(c cluster.TestCluster, m platform.Machine, script string, options map[string]string) { - var buffer = new(bytes.Buffer) - - tmpl, err := template.New("installScript").Parse(script) - if err != nil { - c.Fatal(err) - } - if err := tmpl.Execute(buffer, options); err != nil { - c.Fatal(err) - } - - if err := platform.InstallFile(buffer, m, "/home/core/install.sh"); err != nil { - c.Fatal(err) - } - - // use client to collect stderr - client, err := m.SSHClient() - if err != nil { - c.Fatal(err) - } - defer client.Close() - session, err := client.NewSession() - if err != nil { - c.Fatal(err) - } - defer session.Close() - - stderr := bytes.NewBuffer(nil) - session.Stderr = stderr - - err = session.Start("sudo /home/core/install.sh") - if err != nil { - c.Fatal(err) - } - - // timeout script to prevent it looping forever - errc := make(chan error) - go func() { - errc <- session.Wait() - }() - select { - case err := <-errc: - if err != nil { - c.Fatal(err) - } - case <-time.After(time.Minute * 7): - c.Fatal("Timed out waiting for install script to finish.") - } -} - -var ( - etcdConfig = conf.ContainerLinuxConfig(` -etcd: - advertise_client_urls: http://{PRIVATE_IPV4}:2379 - listen_client_urls: http://0.0.0.0:2379 -systemd: - units: - - name: etcd-member.service - enabled: true -`) - noLocalEtcdConfig = conf.ContainerLinuxConfig(` -systemd: - units: - - name: etcd-member.service - mask: true -`) -) - -const ( - masterCNF = `[req] -req_extensions = v3_req -distinguished_name = req_distinguished_name -[req_distinguished_name] -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName = @alt_names -[alt_names] -DNS.1 = kubernetes -DNS.2 = kubernetes.default -IP.1 = {{.K8S_SERVICE_IP}} -IP.2 = {{.MASTER_HOST}}` - - workerCNF = `[req] -req_extensions = v3_req -distinguished_name = req_distinguished_name -[req_distinguished_name] -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName = @alt_names -[alt_names] -IP.1 = {{.WORKER_IP}}` -) diff --git a/kola/tests/kubernetes/workerInstall.go b/kola/tests/kubernetes/workerInstall.go deleted file mode 100644 index 0fec5fa11..000000000 --- a/kola/tests/kubernetes/workerInstall.go +++ /dev/null @@ -1,300 +0,0 @@ -package kubernetes - -// https://github.com/coreos/coreos-kubernetes/tree/master/multi-node/generic. -const workerInstallScript = `#!/bin/bash -export CNI_VERSION="v0.9.1" - -# Download dir used to store the kubernetes -# related components -export DOWNLOAD_DIR=/opt/bin - -# List of etcd servers (http://ip:port), comma separated -export ETCD_ENDPOINTS={{.ETCD_ENDPOINTS}} - -# The endpoint the worker node should use to contact controller nodes (https://ip:port) -# In HA configurations this should be an external DNS record or loadbalancer in front of the control nodes. -# However, it is also possible to point directly to a single control node. -export CONTROLLER_ENDPOINT={{.CONTROLLER_ENDPOINT}} - -# Specify the version (vX.Y.Z) of Kubernetes assets to deploy -export K8S_VER={{.K8S_VER}} - -# Hyperkube image repository to use. -export HYPERKUBE_IMAGE_REPO={{.HYPERKUBE_IMAGE_REPO}} - -# The IP address of the cluster DNS service. -# This must be the same DNS_SERVICE_IP used when configuring the controller nodes. -export DNS_SERVICE_IP=192.168.128.10 - -# Whether to use Calico for Kubernetes network policy. -export USE_CALICO=false - -# Determines the container runtime for kubernetes to use. Accepts 'docker'. -export CONTAINER_RUNTIME={{.CONTAINER_RUNTIME}} - -# The above settings can optionally be overridden using an environment file: -ENV_FILE=/run/coreos-kubernetes/options.env - -# ------------- - -function init_config { - local REQUIRED=( 'ADVERTISE_IP' 'ETCD_ENDPOINTS' 'CONTROLLER_ENDPOINT' 'DNS_SERVICE_IP' 'K8S_VER' 'HYPERKUBE_IMAGE_REPO' 'USE_CALICO' ) - - if [ -f $ENV_FILE ]; then - export $(cat $ENV_FILE | xargs) - fi - - if [ -z $ADVERTISE_IP ]; then - systemctl start coreos-metadata - export ADVERTISE_IP=$(cat /run/metadata/flatcar | grep -v IPV6 | grep IP | grep -E '(PRIVATE|LOCAL|DYNAMIC)' | cut -d = -f 2) - fi - - for REQ in "${REQUIRED[@]}"; do - if [ -z "$(eval echo \$$REQ)" ]; then - echo "Missing required config value: ${REQ}" - exit 1 - fi - done -} - -function init_templates { - local TEMPLATE=/etc/systemd/system/kubelet.service - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -[Service] -Requires=docker.service -After=docker.service -ExecStartPre=/usr/bin/docker pull ${HYPERKUBE_IMAGE_REPO}:${K8S_VER} -ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests -ExecStart=/opt/bin/kubelet \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --container-runtime=${CONTAINER_RUNTIME} \ - --register-node=true \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --hostname-override=${ADVERTISE_IP} \ - --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \ - --tls-cert-file=/etc/kubernetes/ssl/worker.pem \ - --tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem \ - --volume-plugin-dir=/opt/libexec/kubernetes/kubelet-plugins/volume/exec/ -Restart=always -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -[Install] -WantedBy=multi-user.target -EOF - fi - - local TEMPLATE=/etc/systemd/system/calico-node.service - if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -[Unit] -Description=Calico per-host agent -Requires=network-online.target -After=network-online.target - -[Service] -Slice=machine.slice -Environment=CALICO_DISABLE_FILE_LOGGING=true -Environment=HOSTNAME=${ADVERTISE_IP} -Environment=IP=${ADVERTISE_IP} -Environment=FELIX_FELIXHOSTNAME=${ADVERTISE_IP} -Environment=CALICO_NETWORKING=false -Environment=NO_DEFAULT_POOLS=true -Environment=ETCD_ENDPOINTS=${ETCD_ENDPOINTS} -ExecStart=/usr/bin/rkt run --inherit-env --stage1-from-dir=stage1-fly.aci \ ---volume=modules,kind=host,source=/lib/modules,readOnly=false \ ---mount=volume=modules,target=/lib/modules \ ---trust-keys-from-https --insecure-options=image docker://quay.io/calico/node:v0.19.0 -KillMode=mixed -Restart=always -TimeoutStartSec=0 - -[Install] -WantedBy=multi-user.target -EOF - fi - - local TEMPLATE=/etc/kubernetes/worker-kubeconfig.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Config -clusters: -- name: local - cluster: - certificate-authority: /etc/kubernetes/ssl/ca.pem - server: ${CONTROLLER_ENDPOINT} - clusterDNS: ${DNS_SERVICE_IP} - clusterDomain: cluster.local -users: -- name: kubelet - user: - client-certificate: /etc/kubernetes/ssl/worker.pem - client-key: /etc/kubernetes/ssl/worker-key.pem -contexts: -- context: - cluster: local - user: kubelet - name: kubelet-context -current-context: kubelet-context -EOF - fi - - KUBE_PREFIX="" - if [[ $K8S_VER > "v1.16" ]]; then - KUBE_PREFIX="kube-" - fi - - local TEMPLATE=/etc/kubernetes/manifests/kube-proxy.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Pod -metadata: - name: kube-proxy - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-proxy - image: ${HYPERKUBE_IMAGE_REPO}:$K8S_VER - command: - - /hyperkube - - ${KUBE_PREFIX}proxy - - --master=${CONTROLLER_ENDPOINT} - - --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/ssl/certs - name: "ssl-certs" - - mountPath: /etc/kubernetes/worker-kubeconfig.yaml - name: "kubeconfig" - readOnly: true - - mountPath: /etc/kubernetes/ssl - name: "etc-kube-ssl" - readOnly: true - - mountPath: /var/run/dbus - name: dbus - readOnly: false - volumes: - - name: "ssl-certs" - hostPath: - path: "/usr/share/ca-certificates" - - name: "kubeconfig" - hostPath: - path: "/etc/kubernetes/worker-kubeconfig.yaml" - - name: "etc-kube-ssl" - hostPath: - path: "/etc/kubernetes/ssl" - - hostPath: - path: /var/run/dbus - name: dbus -EOF - fi - - local TEMPLATE=/run/flannel/options.env - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -FLANNELD_IFACE=$ADVERTISE_IP -FLANNELD_ETCD_ENDPOINTS=$ETCD_ENDPOINTS -EOF - fi - - local TEMPLATE=/etc/systemd/system/docker.service.d/40-flannel.conf - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -[Service] -EnvironmentFile=/etc/kubernetes/cni/docker_opts_cni.env -EOF - fi - - local TEMPLATE=/etc/kubernetes/cni/docker_opts_cni.env - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -DOCKER_OPT_BIP="" -DOCKER_OPT_IPMASQ="" -EOF - fi - - local TEMPLATE=/etc/kubernetes/cni/net.d/10-calico.conf - if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "name": "calico", - "cniVersion": "0.2.0", - "type": "flannel", - "delegate": { - "type": "calico", - "etcd_endpoints": "$ETCD_ENDPOINTS", - "log_level": "none", - "log_level_stderr": "info", - "hostname": "${ADVERTISE_IP}", - "policy": { - "type": "k8s", - "k8s_api_root": "${CONTROLLER_ENDPOINT}:443/api/v1/", - "k8s_client_key": "/etc/kubernetes/ssl/worker-key.pem", - "k8s_client_certificate": "/etc/kubernetes/ssl/worker.pem" - } - } -} -EOF - fi - - local TEMPLATE=/etc/kubernetes/cni/net.d/10-flannel.conf - if [ "${USE_CALICO}" = "false" ] && [ ! -f "${TEMPLATE}" ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -{ - "name": "podnet", - "cniVersion": "0.2.0", - "type": "flannel", - "delegate": { - "isDefaultGateway": true - } -} -EOF - fi - -} - -mkdir --parent /opt/cni/bin -curl -sSL --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/kubelet -curl -sSL "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - -chmod +x kubelet -mv kubelet $DOWNLOAD_DIR/ - -init_config -init_templates - -systemctl stop update-engine; systemctl mask update-engine - -systemctl daemon-reload - -systemctl enable --now flanneld -systemctl enable --now kubelet - -if [ $USE_CALICO = "true" ]; then - systemctl enable calico-node; systemctl start calico-node -fi` From 3084a452b65caf4dbf4632f96c182639ee237f32 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 10 Nov 2021 17:06:31 +0100 Subject: [PATCH 2/3] kola/registry: remove legacy kubernetes tests this one are deprecated since we run docker >= 20.10 Signed-off-by: Mathieu Tortuyaux --- kola/registry/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/kola/registry/registry.go b/kola/registry/registry.go index 7143a2333..795b25b4d 100644 --- a/kola/registry/registry.go +++ b/kola/registry/registry.go @@ -9,7 +9,6 @@ import ( _ "github.com/flatcar-linux/mantle/kola/tests/flannel" _ "github.com/flatcar-linux/mantle/kola/tests/ignition" _ "github.com/flatcar-linux/mantle/kola/tests/kubeadm" - _ "github.com/flatcar-linux/mantle/kola/tests/kubernetes" _ "github.com/flatcar-linux/mantle/kola/tests/locksmith" _ "github.com/flatcar-linux/mantle/kola/tests/metadata" _ "github.com/flatcar-linux/mantle/kola/tests/misc" From edcd39b7c521a43d2ec0d0fab6d71593a52999d4 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 10 Nov 2021 17:10:51 +0100 Subject: [PATCH 3/3] changelog: add entry Signed-off-by: Mathieu Tortuyaux --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed0ea161..769939ac9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Renamed the project name from `github.com/coreos/mantle` to `github.com/flatcar-linux/mantle` ([#241](https://github.com/flatcar-linux/mantle/pull/241)) ### Removed +- Legacy Kola Kubernetes tests ([#250](https://github.com/flatcar-linux/mantle/pull/250)) ## [0.17.0] - 05/10/2021 ### Security