Move Server to Deployment instead of DaemonSet

README.md

@@ -63,6 +63,18 @@ When your process starts an AWS SDK library will normally use a chain of credent
 ## Deploying to Kubernetes
 Please see the `deploy` directory for example manifests for deploying to Kubernetes.
 
+There are two options for deployment of `kiam-server`, a `Deployment` and a `DaemonSet`. Some users have found the termination handling of the `DaemonSet` pods causes issues when cycling master nodes (see https://github.com/uswitch/kiam/issues/245). Only deploy **one** of the two `server.*.yaml` files, not _both_.
+
+```
+kubectl apply -f deploy/server.deployment.yaml
+kubectl apply -f deploy/service.yaml
+```
+or
+```
+kubectl apply -f deploy/server.daemonset.yaml
+kubectl apply -f deploy/service.yaml
+```
+
 TLS assets must be created to mutually authenticate the agents and server processes; notes are in [docs/TLS.md](docs/TLS.md).
 
 Please also make note of how to configure IAM in your AWS account; notes in [docs/IAM.md](docs/IAM.md).

deploy/server.yaml → deploy/server.daemonset.yaml

@@ -84,19 +84,3 @@ spec:
             initialDelaySeconds: 3
             periodSeconds: 10
             timeoutSeconds: 10
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: kiam-server
-  namespace: kube-system
-spec:
-  clusterIP: None
-  selector:
-    app: kiam
-    role: server
-  ports:
-  - name: grpclb
-    port: 443
-    targetPort: 443
-    protocol: TCP

deploy/server.deployment.yaml

@@ -0,0 +1,99 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: kube-system
+  name: kiam-server
+  labels:
+    app: kiam
+    role: server
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: kiam
+      role: server
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9620"
+      labels:
+        app: kiam
+        role: server
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: "app"
+                operator: In
+                values:
+                - kiam
+            topologyKey: "kubernetes.io/hostname"
+      serviceAccountName: kiam-server
+      nodeSelector:
+        kubernetes.io/role: master
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            # for AWS linux or RHEL distros
+            # path: /etc/pki/ca-trust/extracted/pem/
+            # debian or ubuntu distros
+            # path: /etc/ssl/certs
+            path: /usr/share/ca-certificates
+        - name: tls
+          secret:
+            secretName: kiam-server-tls
+      containers:
+        - name: kiam
+          image: quay.io/uswitch/kiam:master # USE A TAGGED RELEASE IN PRODUCTION
+          imagePullPolicy: Always
+          command:
+            - /kiam
+          args:
+            - server
+            - --json-log
+            - --level=warn
+            - --bind=0.0.0.0:443
+            - --cert=/etc/kiam/tls/server.pem
+            - --key=/etc/kiam/tls/server-key.pem
+            - --ca=/etc/kiam/tls/ca.pem
+            - --role-base-arn-autodetect
+            - --sync=1m
+            - --prometheus-listen-addr=0.0.0.0:9620
+            - --prometheus-sync-interval=5s
+          volumeMounts:
+            - mountPath: /etc/ssl/certs
+              name: ssl-certs
+            - mountPath: /etc/kiam/tls
+              name: tls
+          livenessProbe:
+            exec:
+              command:
+              - /kiam
+              - health
+              - --cert=/etc/kiam/tls/server.pem
+              - --key=/etc/kiam/tls/server-key.pem
+              - --ca=/etc/kiam/tls/ca.pem
+              - --server-address=127.0.0.1:443
+              - --gateway-timeout-creation=1s
+              - --timeout=5s
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 10
+          readinessProbe:
+            exec:
+              command:
+              - /kiam
+              - health
+              - --cert=/etc/kiam/tls/server.pem
+              - --key=/etc/kiam/tls/server-key.pem
+              - --ca=/etc/kiam/tls/ca.pem
+              - --server-address=127.0.0.1:443
+              - --gateway-timeout-creation=1s
+              - --timeout=5s
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 10

deploy/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kiam-server
+  namespace: kube-system
+spec:
+  clusterIP: None
+  selector:
+    app: kiam
+    role: server
+  ports:
+  - name: grpclb
+    port: 443
+    targetPort: 443
+    protocol: TCP