Overhaul Kubernetes integration

Dockerfile

@@ -29,11 +29,11 @@ RUN go install github.com/quentin-m/etcd-cloud-operator/cmd/tester
 # Copy ECO and etcdctl into an Alpine Linux container image.
 FROM alpine
 
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates docker-cli
+RUN update-ca-certificates
 COPY --from=builder /go/bin/operator /operator
 COPY --from=builder /go/bin/tester /tester
 COPY --from=builder /etcd/etcdctl /usr/local/bin/etcdctl
 
-
 ENTRYPOINT ["/operator"]
 CMD ["-config", "/etc/eco/eco.yaml"]

README.md

@@ -1,6 +1,6 @@
 # etcd-cloud-operator
 
-Maintained by a former CoreOS engineer and inspired from the [etcd-operator]
+Maintained by a former CoreOS engineer and inspired from the [etcd-operator](https://github.com/coreos/etcd-operator)
 designed for Kubernetes, the etcd-cloud-operator automatically bootstraps,
 monitors, snapshots and recovers etcd clusters on cloud providers.
 
@@ -40,37 +40,27 @@ failures.
     in the config file. See [init-acl.md](./docs/init-acl.md) for more information.
 
 -   _JWT auth token support_: JWT auth token can be enabled by specifying the
-    `jwt-auth-token-config` in the config file, similar to the etcd [-auth-token](https://etcd.io/docs/v3.3/op-guide/configuration/#--auth-token)
-    flag.
-    The JWT auth token is **HIGHLY** recommended for [production deployment](https://etcd.io/docs/v3.2/learning/auth_design/#two-types-of-tokens-simple-and-jwt),
+    `jwt-auth-token-config` in the config file, similar to the etcd 
+    [-auth-token](https://etcd.io/docs/v3.3/op-guide/configuration/#--auth-token) flag.
+    The JWT auth token is **HIGHLY** recommended for 
+    [production deployment](https://etcd.io/docs/v3.2/learning/auth_design/#two-types-of-tokens-simple-and-jwt),
     especially when the **init-acl** config is also enabled, the JWT auth token can help
     avoid the potential [invalid auth token issue](https://github.com/etcd-io/etcd/issues/9629).
 
-The operator and etcd cluster can be easily configured using a [YAML file]. The
+The operator and etcd cluster can be easily configured using a [YAML file](config.example.yaml). The
 configuration notably includes clients/peers TLS encryption/authentication, with
 the ability to automatically generate self-signed certificates if encryption
 is desired but authentication is not.
 
-A changelog is maintained at [CHANGELOG.md](CHANGELOG.md).
-
 ## How to try it?
 
 Running a managed etcd cluster using the operator is simply a matter of running
 the operator binary in a supported auto-scaling group (as of today, AWS and Kubernetes only).
 
--   _AWS_: You will need to provide IAM credentials with the following capabilities
-    in the container's environment, scoped to the appropriate instances:
-    "ec2:DescribeInstances"
-    "autoscaling:DescribeAutoScalingGroups"
-    "autoscaling:DescribeAutoScalingInstances"
-
--   Kubernetes: You can run the etcd-cloud-operator in a statefulset, but you will need to provide a
-    few environment variables. See the [Readme](docs/kubernetes/README.md) for the `sts` provider.
-    The easiest way to get to going is to use the included `helm` [chart](chart/etcd-cloud-operator).
+-   _Docker_: Head over to [docs/docker-testing](docs/docker-testing) for a single-line local 3-nodes deployment.
 
-A Terraform [module] is available to easily bring up production-grade etcd clusters
-managed by the the operator out, and integrate them into your infrastructure.
+-   _AWS_: A Terraform [module](terraform/platforms/aws) is available to easily bring up production-grade etcd clusters
+    managed by the operator within AWS.
 
-[etcd-operator]: https://github.com/coreos/etcd-operator
-[yaml file]: config.example.yaml
-[module]: terraform/platforms/aws
+-   _Kubernetes_: A basic [Helm chart](chart/etcd-cloud-operator) is available to quickly get started with a 3-nodes 
+    StatefulSet.

chart/etcd-cloud-operator/templates/configmap.yaml

@@ -15,9 +15,6 @@ data:
       check-interval: {{ .Values.config.checkInterval }}
       # The time after which, an unhealthy member will be removed from the cluster.
       unhealthy-member-ttl: {{ .Values.config.unhealthyMemberTTL }}
-      # Defines whether the operator will attempt to seed a new cluster from a
-      # snapshot after the managed cluster has lost quorum.
-      auto-disaster-recovery: {{ .Values.config.autoDisasterRecovery }}
       # Configuration of the etcd instance.
       etcd:
         # The address that clients should use to connect to the etcd cluster (i.e.

chart/etcd-cloud-operator/templates/networkpolicy.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.networkPolicy.enabled }}
 kind: NetworkPolicy
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1
 metadata:
   name: {{ include "etcd-cloud-operator.fullname" . }}
   labels:
@@ -13,21 +13,20 @@ spec:
     matchLabels:
       app.kubernetes.io/name: {{ include "etcd-cloud-operator.name" . }}
       app.kubernetes.io/instance: {{ .Release.Name }}
-      app: {{ include "etcd-cloud-operator.name" . }}
-      release: {{ .Release.Name | quote }}
   ingress:
-    # Allow inbound connections
+    {{- if not .Values.networkPolicy.allowExternal }}
     - ports:
       - port: 2379
-    {{- if not .Values.networkPolicy.allowExternal }}
-      from:
-      - podSelector:
-          matchLabels:
-            {{ include "etcd-cloud-operator.fullname" . }}-client: "true"
     {{- end }}
     # Allow prometheus scrapes, peer discovery and healthchecks
     - ports:
       - port: 2378
+      - port: 2379
       - port: 2380
       - port: 2381
+      from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: {{ include "etcd-cloud-operator.name" . }}
+              app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}

chart/etcd-cloud-operator/templates/pdb.yaml

@@ -3,7 +3,7 @@ kind: PodDisruptionBudget
 metadata:
   name: {{ include "etcd-cloud-operator.fullname" . }}
 spec:
-  maxUnavailable: {{ .Values.maxUnavailable }}
+  minAvailable: 60%
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "etcd-cloud-operator.name" . }}

chart/etcd-cloud-operator/templates/service.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "etcd-cloud-operator.fullname" . }}
+  name: {{ include "etcd-cloud-operator.fullname" . }}-discovery
   labels:
     app.kubernetes.io/name: {{ include "etcd-cloud-operator.name" . }}
     helm.sh/chart: {{ include "etcd-cloud-operator.chart" . }}
@@ -11,11 +11,24 @@ metadata:
 spec:
   clusterIP: None
   type: ClusterIP
+  publishNotReadyAddresses: true
   ports:
+    - port: 2378
+      targetPort: http
+      protocol: TCP
+      name: http
+    - port: 2379
+      targetPort: client
+      protocol: TCP
+      name: client
     - port: 2380
       targetPort: peer
       protocol: TCP
       name: peer
+    - port: 2381
+      protocol: TCP
+      targetPort: metrics
+      name: metrics
   selector:
     app.kubernetes.io/name: {{ include "etcd-cloud-operator.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
@@ -33,21 +46,12 @@ metadata:
       prometheus.io/scrape: "true"
       prometheus.io/port: "2381"
 spec:
-  clusterIP: None
   type: {{ .Values.service.type }}
   ports:
     - port: 2379
       targetPort: client
       protocol: TCP
       name: client
-    - port: 2378
-      targetPort: http
-      protocol: TCP
-      name: http
-    - port: 2381
-      protocol: TCP
-      targetPort: metrics
-      name: metrics
   selector:
     app.kubernetes.io/name: {{ include "etcd-cloud-operator.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}

chart/etcd-cloud-operator/templates/statefulset.yaml

@@ -9,8 +9,9 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
-  serviceName: {{ include "etcd-cloud-operator.fullname"  . }}
+  serviceName: {{ include "etcd-cloud-operator.fullname"  . }}-discovery
   replicas: {{ default 3 .Values.replicaCount }}
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "etcd-cloud-operator.name" . }}
@@ -26,14 +27,8 @@ spec:
           image: "{{ .Values.image.repository }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           volumeMounts:
-          {{- if .Values.persistence.enabled }}
           - name: data
-            mountPath: {{ .Values.config.etcd.dataDir }}
-          {{- end }}
-          {{- if eq .Values.config.snapshot.provider "file" }}
-          - name: snapshot
-            mountPath: /var/lib/snapshots
-          {{- end}}
+            mountPath: /var/lib
           - name: config
             mountPath: /etc/eco
             readOnly: true
@@ -55,17 +50,17 @@ spec:
             value: {{ .Values.config.etcd.clientTransportSecurity.keyFile }}
           {{- end }}
           - name: STATEFULSET_SERVICE_NAME
-            value: {{ include "etcd-cloud-operator.fullname" . }}
+            value: {{ include "etcd-cloud-operator.fullname" . }}-discovery
           - name: STATEFULSET_NAME
             value: {{ include "etcd-cloud-operator.fullname" . }}
-          - name: POD_IP
-            valueFrom:
-              fieldRef:
-                fieldPath: status.podIP
+          - name: STATEFULSET_REPLICAS
+            value: "{{ default 3 .Values.replicaCount }}"
           - name: STATEFULSET_NAMESPACE
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
+          - name: STATEFULSET_DNS_CLUSTER_SUFFIX
+            value: cluster.local
           ports:
             - name: client
               containerPort: 2379
@@ -95,7 +90,7 @@ spec:
               command:
               - /bin/sh
               - -c
-              - /usr/local/bin/etcdctl --endpoints=${POD_IP}:2379 endpoint health
+              - /usr/local/bin/etcdctl --endpoints=${HOSTNAME}:2379 endpoint health
             failureThreshold: {{ .Values.setupProbe.failureThreshold }}
             periodSeconds: {{ .Values.setupProbe.periodSeconds }}
           resources:
@@ -104,36 +99,31 @@ spec:
       - name: config
         configMap:
           name: {{ include "etcd-cloud-operator.fullname" . }}
-{{- if eq .Values.config.snapshot.provider "file" }}
-      - name: snapshot
-        emptyDir: {}
-{{- end }}
-{{- if and .Values.persistence.enabled .Values.persistence.existingClaim }}
+{{- if .Values.persistence.enabled }}
       - name: data
         persistentVolumeClaim:
-          claimName: {{ .Values.persistence.existingClaim }}
-{{- else if not .Values.persistence.enabled }}
-      - name: data
-        emptyDir: {}
-{{- else if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+          claimName: data
   volumeClaimTemplates:
   - metadata:
       name: data
     {{- with .Values.persistence.annotations }}
-      annotations:
-      {{- range $key, $value := . }}
-        {{ $key }}: {{ $value }}
-      {{- end }}
+    annotations:
+    {{- range $key, $value := . }}
+    {{ $key }}: {{ $value }}
+    {{- end }}
     {{- end }}
     spec:
       storageClassName: "{{ .Values.persistence.storageClass }}"
       accessModes:
       {{- range .Values.persistence.accessModes }}
-        - {{ . | quote }}
+      - {{ . | quote }}
       {{- end }}
       resources:
         requests:
           storage: {{ .Values.persistence.size | quote }}
+{{- else }}
+      - name: data
+        emptyDir: {}
 {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector: