Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flannel using CNI #1486

Merged
merged 1 commit into from
Aug 25, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions docs/flannel.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,6 @@ ip a show dev flannel.1
valid_lft forever preferred_lft forever
```

* Docker must be configured with a bridge ip in the flannel subnet.

```
ps aux | grep docker
root 20196 1.7 2.7 1260616 56840 ? Ssl 10:18 0:07 /usr/bin/docker daemon --bip=10.233.16.1/24 --mtu=1450
```

* Try to run a container and check its ip address

```
Expand Down
2 changes: 1 addition & 1 deletion roles/docker/templates/docker-options.conf.j2
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[Service]
Environment="DOCKER_OPTS={{ docker_options | default('') }} \
--iptables={% if kube_network_plugin == 'flannel' %}true{% else %}false{% endif %}"
--iptables=false"
11 changes: 10 additions & 1 deletion roles/download/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ calico_ctl_version: "v1.4.0"
calico_cni_version: "v1.10.0"
calico_policy_version: "v0.7.0"
weave_version: 2.0.1
flannel_version: v0.8.0
flannel_version: "v0.8.0"
flannel_cni_version: "v0.2.0"
pod_infra_version: 3.0

# Download URL's
Expand All @@ -43,6 +44,8 @@ etcd_image_repo: "quay.io/coreos/etcd"
etcd_image_tag: "{{ etcd_version }}"
flannel_image_repo: "quay.io/coreos/flannel"
flannel_image_tag: "{{ flannel_version }}"
flannel_cni_image_repo: "quay.io/coreos/flannel-cni"
flannel_cni_image_tag: "{{ flannel_cni_version }}"
calicoctl_image_repo: "quay.io/calico/ctl"
calicoctl_image_tag: "{{ calico_ctl_version }}"
calico_node_image_repo: "quay.io/calico/node"
Expand Down Expand Up @@ -138,6 +141,12 @@ downloads:
tag: "{{ flannel_image_tag }}"
sha256: "{{ flannel_digest_checksum|default(None) }}"
enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
flannel_cni:
container: true
repo: "{{ flannel_cni_image_repo }}"
tag: "{{ flannel_cni_image_tag }}"
sha256: "{{ flannel_cni_digest_checksum|default(None) }}"
enabled: "{{ kube_network_plugin == 'flannel' }}"
calicoctl:
container: true
repo: "{{ calicoctl_image_repo }}"
Expand Down
22 changes: 22 additions & 0 deletions roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
- name: "Flannel | Create ServiceAccount ClusterRole and ClusterRoleBinding"
command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/cni-flannel-rbac.yml"
run_once: true
when: rbac_enabled and flannel_rbac_manifest.changed

- name: Flannel | Start Resources
kube:
name: "kube-flannel"
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/cni-flannel.yml"
resource: "ds"
namespace: "{{system_namespace}}"
state: "{{ item | ternary('latest','present') }}"
with_items: "{{ flannel_manifest.changed }}"
when: inventory_hostname == groups['kube-master'][0]

- name: Flannel | Wait for flannel subnet.env file presence
wait_for:
path: /run/flannel/subnet.env
delay: 5
timeout: 600
3 changes: 3 additions & 0 deletions roles/kubernetes-apps/network_plugin/meta/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ dependencies:
- role: kubernetes-apps/network_plugin/canal
when: kube_network_plugin == 'canal'
tags: canal
- role: kubernetes-apps/network_plugin/flannel
when: kube_network_plugin == 'flannel'
tags: flannel
- role: kubernetes-apps/network_plugin/weave
when: kube_network_plugin == 'weave'
tags: weave
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,12 @@ spec:
- --cloud-provider={{cloud_provider}}
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
- --allocate-node-cidrs=true
- --configure-cloud-routes=true
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel"] %}
- --allocate-node-cidrs=true
- --cluster-cidr={{ kube_pods_subnet }}
- --service-cluster-ip-range={{ kube_service_addresses }}
{% endif %}
{% if kube_feature_gates %}
- --feature-gates={{ kube_feature_gates|join(',') }}
Expand Down
2 changes: 1 addition & 1 deletion roles/kubernetes/node/templates/kubelet.j2
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% endif %}

KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %}
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave"] %}
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
{% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}
DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock"
Expand Down
7 changes: 4 additions & 3 deletions roles/network_plugin/flannel/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
---
# Flannel public IP
# The address that flannel should advertise as how to access the system
flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"
# Disabled until https://github.com/coreos/flannel/issues/712 is fixed
# flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"

## interface that should be used for flannel operations
## This is actually an inventory node-level item
Expand All @@ -17,5 +18,5 @@ flannel_cpu_limit: 300m
flannel_memory_requests: 64M
flannel_cpu_requests: 150m

flannel_cert_dir: /etc/flannel/certs
etcd_cert_dir: /etc/ssl/etcd/ssl
# Legacy directory, will be removed if found.
flannel_cert_dir: /etc/flannel/certs
4 changes: 4 additions & 0 deletions roles/network_plugin/flannel/handlers/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
failed_when: false
notify: Flannel | restart docker

- name: Flannel | delete flannel interface
command: ip link delete flannel.1
failed_when: false

# special cases for atomic because it defaults to live-restore: true
# So we disable live-restore to pickup the new flannel IP. After
# we enable it, we have to restart docker again to pickup the new
Expand Down
3 changes: 3 additions & 0 deletions roles/network_plugin/flannel/meta/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,6 @@ dependencies:
- role: download
file: "{{ downloads.flannel }}"
tags: download
- role: download
file: "{{ downloads.flannel_cni }}"
tags: download
112 changes: 38 additions & 74 deletions roles/network_plugin/flannel/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -1,83 +1,47 @@
---
- name: Flannel | Set Flannel etcd configuration
command: |-
{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
set /{{ cluster_name }}/network/config \
'{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
- include: pre-upgrade.yml

- name: Flannel | Create flannel certs directory
file:
dest: "{{ flannel_cert_dir }}"
state: directory
mode: 0750
owner: root
group: root

- name: Flannel | Link etcd certificates for flanneld
file:
src: "{{ etcd_cert_dir }}/{{ item.s }}"
dest: "{{ flannel_cert_dir }}/{{ item.d }}"
state: hard
force: yes
with_items:
- {s: "ca.pem", d: "ca_cert.crt"}
- {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
- {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}

- name: Flannel | Create flannel pod manifest
template:
src: flannel-pod.yml
dest: "{{kube_manifest_dir}}/flannel-pod.manifest"
notify: Flannel | delete default docker bridge

- name: Flannel | Wait for flannel subnet.env file presence
wait_for:
path: /run/flannel/subnet.env
delay: 5
timeout: 600

- name: Flannel | Get flannel_subnet from subnet.env
shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_SUBNET" {print $2}'
register: flannel_subnet_output
- name: Flannel | Verify if br_netfilter module exists
shell: "modinfo br_netfilter"
register: modinfo_br_netfilter
failed_when: modinfo_br_netfilter.rc not in [0, 1]
changed_when: false
check_mode: no

- set_fact:
flannel_subnet: "{{ flannel_subnet_output.stdout }}"
- name: Flannel | Enable br_netfilter module
modprobe:
name: br_netfilter
state: present
when: modinfo_br_netfilter.rc == 0

- name: Flannel | Get flannel_mtu from subnet.env
shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_MTU" {print $2}'
register: flannel_mtu_output
# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module
- name: Flannel | Check if bridge-nf-call-iptables key exists
command: "sysctl net.bridge.bridge-nf-call-iptables"
failed_when: false
changed_when: false
check_mode: no

- set_fact:
flannel_mtu: "{{ flannel_mtu_output.stdout }}"

- set_fact:
docker_options_file: >-
{%- if ansible_os_family == "Debian" -%}/etc/default/docker{%- elif ansible_os_family == "RedHat" -%}/etc/sysconfig/docker{%- endif -%}
tags: facts

- set_fact:
docker_options_name: >-
{%- if ansible_os_family == "Debian" -%}DOCKER_OPTS{%- elif ansible_os_family == "RedHat" -%}other_args{%- endif -%}
tags: facts

- set_fact:
docker_network_options: '"--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"'
register: sysctl_bridge_nf_call_iptables

- name: Flannel | Enable bridge-nf-call tables
sysctl:
name: "{{ item }}"
state: present
value: 1
reload: yes
when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0
with_items:
- net.bridge.bridge-nf-call-iptables
- net.bridge.bridge-nf-call-arptables
- net.bridge.bridge-nf-call-ip6tables

- name: Flannel | Ensure path for docker network systemd drop-in
file:
path: "/etc/systemd/system/docker.service.d"
state: directory
owner: root
- name: Flannel | Create cni-flannel-rbac manifest
template:
src: cni-flannel-rbac.yml.j2
dest: "{{ kube_config_dir }}/cni-flannel-rbac.yml"
register: flannel_rbac_manifest
when: inventory_hostname == groups['kube-master'][0] and rbac_enabled

- name: Flannel | Create docker network systemd drop-in
- name: Flannel | Create cni-flannel manifest
template:
src: flannel-options.conf.j2
dest: "/etc/systemd/system/docker.service.d/flannel-options.conf"
notify:
- Flannel | restart docker
src: cni-flannel.yml.j2
dest: "{{ kube_config_dir }}/cni-flannel.yml"
register: flannel_manifest
when: inventory_hostname == groups['kube-master'][0]
19 changes: 19 additions & 0 deletions roles/network_plugin/flannel/tasks/pre-upgrade.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
- name: Flannel pre-upgrade | Purge legacy flannel systemd unit file
file:
path: "/etc/systemd/system/docker.service.d/flannel-options.conf"
state: absent
notify:
- Flannel | delete default docker bridge

- name: Flannel pre-upgrade | Purge legacy Flannel static pod manifest
file:
path: "{{ kube_manifest_dir }}/flannel-pod.manifest"
state: absent
notify:
- Flannel | delete flannel interface

- name: Flannel pre-upgrade | Remove Flannel's certificate directory not required by CNI
file:
dest: "{{ flannel_cert_dir }}"
state: absent
44 changes: 44 additions & 0 deletions roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: "{{system_namespace}}"
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: flannel
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: "{{system_namespace}}"
Loading