kubeadm support (#1631)

* kubeadm support

* move k8s master to a subtask
* disable k8s secrets when using kubeadm
* fix etcd cert serial var
* move simple auth users to master role
* make a kubeadm-specific env file for kubelet
* add non-ha CI job

* change ci boolean vars to json format

* fixup

* Update create-gce.yml

* Update create-gce.yml

* Update create-gce.yml

Author: mattymo

.gitlab-ci.yml

@@ -53,6 +53,7 @@ before_script:
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   UPGRADE_TEST: "false"
+  KUBEADM_ENABLED: "false"
   RESOLVCONF_MODE: docker_dns
   LOG_LEVEL: "-vv"
   ETCD_DEPLOYMENT: "docker"
@@ -117,16 +118,19 @@ before_script:
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
       -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml
@@ -144,17 +148,19 @@ before_script:
       -e ansible_ssh_user=${SSH_USER}
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
       -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
       -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       $PLAYBOOK;
@@ -178,14 +184,18 @@ before_script:
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e deploy_netchecker=true
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -221,14 +231,18 @@ before_script:
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e deploy_netchecker=true
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -280,6 +294,17 @@ before_script:
   UPGRADE_TEST: "graceful"
   STARTUP_SCRIPT: ""
 
+.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_MACHINE_TYPE: "n1-standard-2"
+  CLOUD_REGION: europe-west1-b
+  CLUSTER_MODE: default
+  KUBEADM_ENABLED: "true"
+  STARTUP_SCRIPT: ""
+
 .rhel7_weave_variables: &rhel7_weave_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: weave
@@ -470,6 +495,27 @@ ubuntu-canal-ha-rbac-triggers:
   when: on_success
   only: ['triggers']
 
+ubuntu-canal-kubeadm-rbac:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+ubuntu-canal-kubeadm-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
 rhel7-weave:
   stage: deploy-gce-part1
   <<: *job

cluster.yml

@@ -69,6 +69,17 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/master, tags: master }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 

inventory/group_vars/all.yml

@@ -82,6 +82,12 @@ bin_dir: /usr/local/bin
 #openstack_lbaas_monitor_timeout: "30s"
 #openstack_lbaas_monitor_max_retries: "3"
 
+## Uncomment to enable experimental kubeadm deployment mode
+#kubeadm_enabled: false
+#kubeadm_token_first: "{{ lookup('password', 'credentials/kubeadm_token_first length=6  chars=ascii_letters,digits') }}"
+#kubeadm_token_second: "{{ lookup('password', 'credentials/kubeadm_token_second length=16 chars=ascii_letters,digits') }}"
+#kubeadm_token: "{{ kubeadm_token_first }}.{{ kubeadm_token_second }}"
+#
 ## Set these proxy values in order to update docker daemon to use proxies
 #http_proxy: ""
 #https_proxy: ""

library/kube.py

@@ -135,24 +135,27 @@ def _execute_nofail(self, cmd):
             return None
         return out.splitlines()
 
-    def create(self, check=True):
+    def create(self, check=True, force=True):
         if check and self.exists():
             return []
 
         cmd = ['apply']
 
+        if force:
+            cmd.append('--force')
+
         if not self.filename:
             self.module.fail_json(msg='filename required to create')
 
         cmd.append('--filename=' + self.filename)
 
         return self._execute(cmd)
 
-    def replace(self):
+    def replace(self, force=True):
 
         cmd = ['apply']
 
-        if self.force:
+        if force:
             cmd.append('--force')
 
         if not self.filename:

roles/download/defaults/main.yml

@@ -19,6 +19,7 @@ download_always_pull: False
 
 # Versions
 kube_version: v1.7.3
+kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.4
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
@@ -31,11 +32,13 @@ flannel_version: "v0.8.0"
 flannel_cni_version: "v0.2.0"
 pod_infra_version: 3.0
 
-# Download URL's
+# Download URLs
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
+kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"
 
 # Checksums
 etcd_checksum: "274c46a7f8d26f7ae99d6880610f54933cbcf7f3beafa19236c52eb5df8c7a0b"
+kubeadm_checksum: "378e6052f8b178f8e6a38e8637681c72d389443b66b78b51b8ddc9a162c655c3"
 
 # Containers
 # Possible values: host, docker
@@ -132,6 +135,15 @@ downloads:
     container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
+  kubeadm:
+    version: "{{ kubeadm_version }}"
+    dest: "kubeadm"
+    sha256: "{{ kubeadm_checksum }}"
+    source_url: "{{ kubeadm_download_url }}"
+    url: "{{ kubeadm_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
   hyperkube:
     container: true
     repo: "{{ hyperkube_image_repo }}"

roles/etcd/tasks/main.yml

@@ -11,7 +11,7 @@
 
 - name: "Gen_certs | Get etcd certificate serials"
   shell: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial | cut -d= -f2"
-  register: "node-{{ inventory_hostname }}_serial"
+  register: "etcd_client_cert_serial"
   when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort
 
 - include: "install_{{ etcd_deployment_type }}.yml"

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -8,6 +8,18 @@
   delay: 6
   when: inventory_hostname == groups['kube-master'][0]
 
+- name: kubeadm | Delete kubeadm kubedns
+  kube:
+    name: "kubedns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "deploy"
+    state: absent
+  when:
+    - kubeadm_enabled|default(false)
+    - kubeadm_init.changed|default(false)
+    - inventory_hostname == groups['kube-master'][0]
+
 - name: Kubernetes Apps | Lay Down KubeDNS Template
   template:
     src: "{{item.file}}"

roles/kubernetes/kubeadm/tasks/main.yml

@@ -0,0 +1,41 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint }}
+      {%- endif %}
+  when: not is_kube_master
+  tags: facts
+
+- name: Create kubeadm client config
+  template:
+    src: kubeadm-client.conf.j2
+    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    backup: yes
+  when: not is_kube_master
+  register: kubeadm_client_conf
+
+- name: Join to cluster if needed
+  command: kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks
+  register: kubeadm_join
+  when: not is_kube_master and kubeadm_client_conf.changed
+
+- name: Update server field in kubelet kubeconfig
+  replace:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '(\s+){{ first_kube_master }}:{{ kube_apiserver_port }}(\s+.*)?$'
+    replace: '\1{{ kube_apiserver_endpoint }}\2'
+    backup: yes
+  when: not is_kube_master and kubeadm_discovery_address != kube_apiserver_endpoint
+
+# FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
+- name: Symlink kubelet kubeconfig for calico/canal
+  file:
+    src: "{{ kube_config_dir }}//kubelet.conf"
+    dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    state: link
+    force: yes
+  when: kube_network_plugin in ['calico','canal']

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -0,0 +1,6 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: NodeConfiguration
+caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+token: {{ kubeadm_token }}
+discoveryTokenAPIServers:
+- {{ kubeadm_discovery_address | replace("https://", "")}}

roles/kubernetes/master/defaults/main.yml

@@ -66,3 +66,7 @@ apiserver_custom_flags: []
 controller_mgr_custom_flags: []
 
 scheduler_custom_flags: []
+
+# kubeadm settings
+# Value of 0 means it never expires
+kubeadm_token_ttl: 0

roles/kubernetes/master/handlers/main.yml

@@ -44,3 +44,7 @@
   until: result.status == 200
   retries: 20
   delay: 6
+
+- name: Master | set secret_changed
+  set_fact:
+    secret_changed: true

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -0,0 +1,35 @@
+---
+- name: kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: >-
+      kubernetes
+      kubernetes.default
+      kubernetes.default.svc
+      kubernetes.default.svc.{{ dns_domain }}
+      {{ kube_apiserver_ip }}
+      localhost
+      127.0.0.1
+      {{ ' '.join(groups['kube-master']) }}
+      {%- if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+      {{ apiserver_loadbalancer_domain_name }}
+      {%- endif %}
+      {%- for host in groups['kube-master'] -%}
+      {%- if hostvars[host]['access_ip'] is defined %}{{ hostvars[host]['access_ip'] }}{% endif -%}
+      {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+      {%- endfor %}
+  tags: facts
+
+- name: kubeadm | Copy etcd cert dir under k8s cert dir
+  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
+  changed_when: false
+
+- name: kubeadm | Create kubeadm config
+  template:
+    src: kubeadm-config.yaml.j2
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kubeadm_config
+
+- name: kubeadm | Initialize cluster
+  command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  register: kubeadm_init
+  when: kubeadm_config.changed