Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove rkt and kubelet-wrapper from kubernetes tests #169

Merged
merged 9 commits into from
Apr 22, 2021
Merged
78 changes: 20 additions & 58 deletions kola/tests/kubernetes/basic.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// Copyright 2021 Kinvolk GmbH
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -51,7 +52,7 @@ func init() {
Name: "google.kubernetes.basic." + r + "." + t,
Run: f,
ClusterSize: 0,
Platforms: []string{"gce", "do", "aws"}, // TODO: fix packet, esx
Platforms: []string{"gce", "do", "aws", "qemu"}, // TODO: fix packet, esx
Distros: []string{"cl"},
})
}
Expand All @@ -68,12 +69,6 @@ func CoreOSBasic(c cluster.TestCluster, version, runtime string) {
if err := nginxCheck(c, k.master, k.workers); err != nil {
c.Fatal(err)
}

// http://kubernetes.io/v1.0/docs/user-guide/secrets/ Also, ensures
// https://github.com/coreos/bugs/issues/447 does not re-occur.
if err := secretCheck(c, k.master, k.workers); err != nil {
c.Fatal(err)
}
}

func nodeCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error {
Expand Down Expand Up @@ -105,9 +100,18 @@ func nodeCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.

func nginxCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error {
pod := strings.NewReader(nginxPodYAML)
secret := strings.NewReader(secretYAML)
if err := platform.InstallFile(pod, master, "./nginx-pod.yaml"); err != nil {
return err
}
if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil {
return err
}

if _, err := c.SSH(master, "./kubectl create -f secret.yaml"); err != nil {
return err
}

if _, err := c.SSH(master, "./kubectl create -f nginx-pod.yaml"); err != nil {
return err
}
Expand Down Expand Up @@ -135,57 +139,7 @@ func nginxCheck(c cluster.TestCluster, master platform.Machine, nodes []platform
return nil
}

func secretCheck(c cluster.TestCluster, master platform.Machine, nodes []platform.Machine) error {
// create yaml files
secret := strings.NewReader(secretYAML)
pod := strings.NewReader(secretPodYAML)
if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil {
return err
}
if err := platform.InstallFile(pod, master, "./secret-pod.yaml"); err != nil {
return err
}

if _, err := c.SSH(master, "./kubectl create -f secret.yaml"); err != nil {
return err
}
_, err := c.SSH(master, "./kubectl describe secret test-secret")
if err != nil {
return err
}

b, err := c.SSH(master, "./kubectl create -f secret-pod.yaml")
if err != nil {
return err
}
expectedOutput := "value-1"
if strings.Contains(strings.TrimSpace(string(b)), expectedOutput) {
return fmt.Errorf("error detecting secret pod")
}

return nil
}

const (
secretPodYAML = `apiVersion: v1
kind: Pod
metadata:
name: secret-test-pod
spec:
containers:
- name: test-container
image: kubernetes/mounttest:0.1
command: [ "/mt", "--file_content=/etc/secret-volume/data-1" ]
volumeMounts:
# name must match the volume name below
- name: secret-volume
mountPath: /etc/secret-volume
volumes:
- name: secret-volume
secret:
secretName: test-secret
restartPolicy: Never`

secretYAML = `apiVersion: v1
kind: Secret
metadata:
Expand All @@ -205,5 +159,13 @@ spec:
- name: nginx
image: nginx
ports:
- containerPort: 80`
- containerPort: 80
volumeMounts:
# name must match the volume name below
- name: secret-volume
mountPath: /etc/secret-volume
volumes:
- name: secret-volume
secret:
secretName: test-secret`
)
141 changes: 22 additions & 119 deletions kola/tests/kubernetes/controllerInstall.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ package kubernetes

// https://github.com/coreos/coreos-kubernetes/tree/master/multi-node/generic.
const controllerInstallScript = `#!/bin/bash
set -e
export CNI_VERSION="v0.9.1"

# Download dir used to store the kubernetes
# related components
export DOWNLOAD_DIR=/opt/bin

# List of etcd servers (http://ip:port), comma separated
export ETCD_ENDPOINTS={{.ETCD_ENDPOINTS}}
Expand Down Expand Up @@ -36,7 +40,7 @@ export DNS_SERVICE_IP=192.168.128.10
# Whether to use Calico for Kubernetes network policy.
export USE_CALICO=false

# Determines the container runtime for kubernetes to use. Accepts 'docker' or 'rkt'.
# Determines the container runtime for kubernetes to use. Accepts 'docker'.
export CONTAINER_RUNTIME={{.CONTAINER_RUNTIME}}

# The above settings can optionally be overridden using an environment file:
Expand Down Expand Up @@ -92,37 +96,21 @@ function init_templates {
if [ ! -f $TEMPLATE ]; then
echo "TEMPLATE: $TEMPLATE"
mkdir -p $(dirname $TEMPLATE)
RKT_FLAGS=""
if [[ ${CONTAINER_RUNTIME} = "rkt" ]]; then
RKT_FLAGS="--rkt-path=/usr/bin/rkt --rkt-stage1-image=coreos.com/rkt/stage1-coreos "
fi
KUBE_EXEC=""
if [[ $K8S_VER > "v1.18" ]]; then
KUBE_EXEC="kubelet"
fi
cat << EOF > $TEMPLATE
[Service]
Environment=KUBELET_IMAGE_TAG=${K8S_VER}
Environment=KUBELET_IMAGE_URL=docker://${HYPERKUBE_IMAGE_REPO}
Environment="RKT_RUN_ARGS=--volume dns,kind=host,source=/etc/resolv.conf \
--mount volume=dns,target=/etc/resolv.conf \
--volume=rkt,kind=host,source=/opt/bin/host-rkt \
--mount volume=rkt,target=/usr/bin/rkt \
--volume var-lib-rkt,kind=host,source=/var/lib/rkt \
--mount volume=var-lib-rkt,target=/var/lib/rkt \
--volume=stage,kind=host,source=/tmp \
--mount volume=stage,target=/tmp \
--insecure-options=image"
Requires=docker.service
After=docker.service
ExecStartPre=/usr/bin/docker pull ${HYPERKUBE_IMAGE_REPO}:${K8S_VER}
ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
ExecStart=/usr/lib/flatcar/kubelet-wrapper ${KUBE_EXEC} \
ExecStart=/opt/bin/kubelet \
--register-schedulable=false \
--kubeconfig=/etc/kubernetes/master-kubeconfig.yaml \
--cni-conf-dir=/etc/kubernetes/cni/net.d \
--network-plugin=cni \
--container-runtime=${CONTAINER_RUNTIME} \
${RKT_FLAGS} \
--pod-manifest-path=/etc/kubernetes/manifests \
--hostname-override=${ADVERTISE_IP}
--hostname-override=${ADVERTISE_IP} \
--volume-plugin-dir=/opt/libexec/kubernetes/kubelet-plugins/volume/exec/
Restart=always
RestartSec=10
CPUAccounting=true
Expand Down Expand Up @@ -156,65 +144,6 @@ current-context: kubelet-context
EOF
fi

local TEMPLATE=/opt/bin/host-rkt
if [ ! -f $TEMPLATE ]; then
echo "TEMPLATE: $TEMPLATE"
mkdir -p $(dirname $TEMPLATE)
cat << EOF > $TEMPLATE
#!/bin/sh
# This is bind mounted into the kubelet rootfs and all rkt shell-outs go
# through this rkt wrapper. It essentially enters the host mount namespace
# (which it is already in) only for the purpose of breaking out of the chroot
# before calling rkt. It makes things like rkt gc work and avoids bind mounting
# in certain rkt filesystem dependancies into the kubelet rootfs. This can
# eventually be obviated when the write-api stuff gets upstream and rkt gc is
# through the api-server. Related issue:
# https://github.com/coreos/rkt/issues/2878
exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@"
EOF
fi


local TEMPLATE=/etc/systemd/system/load-rkt-stage1.service
if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then
echo "TEMPLATE: $TEMPLATE"
mkdir -p $(dirname $TEMPLATE)
cat << EOF > $TEMPLATE
[Unit]
Description=Load rkt stage1 images
Documentation=http://github.com/coreos/rkt
Requires=network-online.target
After=network-online.target
Before=rkt-api.service

[Service]
RemainAfterExit=yes
Type=oneshot
ExecStart=/usr/bin/rkt fetch /usr/lib/rkt/stage1-images/stage1-coreos.aci /usr/lib/rkt/stage1-images/stage1-fly.aci --insecure-options=image

[Install]
RequiredBy=rkt-api.service
EOF
fi

local TEMPLATE=/etc/systemd/system/rkt-api.service
if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then
echo "TEMPLATE: $TEMPLATE"
mkdir -p $(dirname $TEMPLATE)
cat << EOF > $TEMPLATE
[Unit]
Before=kubelet.service

[Service]
ExecStart=/usr/bin/rkt api-service
Restart=always
RestartSec=10

[Install]
RequiredBy=kubelet.service
EOF
fi

local TEMPLATE=/etc/systemd/system/calico-node.service
if [ "${USE_CALICO}" = "true" ] && [ ! -f "${TEMPLATE}" ]; then
echo "TEMPLATE: $TEMPLATE"
Expand Down Expand Up @@ -262,8 +191,6 @@ kind: Pod
metadata:
name: kube-proxy
namespace: kube-system
annotations:
rkt.alpha.kubernetes.io/stage1-name-override: coreos.com/rkt/stage1-fly
spec:
hostNetwork: true
containers:
Expand Down Expand Up @@ -374,6 +301,7 @@ spec:
- --leader-elect=true
- --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
- --root-ca-file=/etc/kubernetes/ssl/ca.pem
- --flex-volume-plugin-dir=/opt/libexec/kubernetes/kubelet-plugins/volume/exec/
resources:
requests:
cpu: 200m
Expand Down Expand Up @@ -587,31 +515,6 @@ EOF
"protocol": "TCP"
}
]
},
{
"args": [
"-cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null",
"-port=8080",
"-quiet"
],
"image": "gcr.io/google_containers/exechealthz-amd64:1.0",
"name": "healthz",
"ports": [
{
"containerPort": 8080,
"protocol": "TCP"
}
],
"resources": {
"limits": {
"cpu": "10m",
"memory": "20Mi"
},
"requests": {
"cpu": "10m",
"memory": "20Mi"
}
}
}
],
"dnsPolicy": "Default"
Expand Down Expand Up @@ -994,24 +897,24 @@ function enable_calico_policy {
curl --silent -H "Content-Type: application/json" -XPOST -d"$(cat /srv/kubernetes/manifests/calico-system.json)" "http://127.0.0.1:8080/api/v1/namespaces/" > /dev/null
}

mkdir --parent /opt/cni/bin
curl -sSL --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/kubelet
curl -sSL "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz

chmod +x kubelet
mv kubelet $DOWNLOAD_DIR/

init_config
init_templates

chmod +x /opt/bin/host-rkt

init_flannel

systemctl stop update-engine; systemctl mask update-engine

systemctl daemon-reload

if [ $CONTAINER_RUNTIME = "rkt" ]; then
systemctl enable load-rkt-stage1
systemctl enable rkt-api
fi

systemctl enable flanneld; systemctl start flanneld
systemctl enable kubelet; systemctl start kubelet
systemctl enable --now flanneld
systemctl enable --now kubelet

if [ $USE_CALICO = "true" ]; then
systemctl enable calico-node; systemctl start calico-node
Expand Down
2 changes: 0 additions & 2 deletions kola/tests/kubernetes/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,8 +268,6 @@ func stripSemverSuffix(v string) (string, error) {

// Run and configure the coreos-kubernetes generic install scripts.
func runInstallScript(c cluster.TestCluster, m platform.Machine, script string, options map[string]string) {
c.MustSSH(m, "sudo stat /usr/lib/flatcar/kubelet-wrapper")

var buffer = new(bytes.Buffer)

tmpl, err := template.New("installScript").Parse(script)
Expand Down
Loading