From 6ab2885f44409572fb25648e8a46b107d5c7b3d2 Mon Sep 17 00:00:00 2001 From: Brandon Wagner Date: Mon, 4 May 2020 17:28:51 -0500 Subject: [PATCH] add webhook proxy test --- .../aws-node-termination-handler/README.md | 3 - .../templates/clusterrole.yaml | 27 +---- .../templates/daemonset.yaml | 3 - .../templates/regular-pod-test.yaml | 3 - .../templates/service.yaml | 2 +- config/helm/squid/.helmignore | 22 ++++ config/helm/squid/Chart.yaml | 9 ++ config/helm/squid/templates/_helpers.tpl | 57 ++++++++++ config/helm/squid/templates/clusterrole.yaml | 11 ++ .../squid/templates/clusterrolebinding.yaml | 12 ++ config/helm/squid/templates/daemonset.yaml | 34 ++++++ config/helm/squid/templates/psp.yaml | 60 ++++++++++ .../helm/squid/templates/serviceaccount.yaml | 11 ++ config/helm/squid/values.yaml | 18 +++ test/assets/squid.conf | 11 ++ test/e2e/webhook-http-proxy-test | 103 ++++++++++++++++++ 16 files changed, 350 insertions(+), 36 deletions(-) create mode 100644 config/helm/squid/.helmignore create mode 100644 config/helm/squid/Chart.yaml create mode 100644 config/helm/squid/templates/_helpers.tpl create mode 100644 config/helm/squid/templates/clusterrole.yaml create mode 100644 config/helm/squid/templates/clusterrolebinding.yaml create mode 100644 config/helm/squid/templates/daemonset.yaml create mode 100644 config/helm/squid/templates/psp.yaml create mode 100644 config/helm/squid/templates/serviceaccount.yaml create mode 100644 config/helm/squid/values.yaml create mode 100644 test/assets/squid.conf create mode 100755 test/e2e/webhook-http-proxy-test diff --git a/config/helm/aws-node-termination-handler/README.md b/config/helm/aws-node-termination-handler/README.md index b29b9112..671717f3 100644 --- a/config/helm/aws-node-termination-handler/README.md +++ b/config/helm/aws-node-termination-handler/README.md @@ -84,6 +84,3 @@ Parameter | Description | Default `procUptimeFile` | (Used for Testing) Specify the uptime file | `/proc/uptime` `securityContext.runAsUserID` | User ID to run the container | `1000` `securityContext.runAsGroupID` | Group ID to run the container | `1000` - - - diff --git a/config/helm/ec2-metadata-test-proxy/templates/clusterrole.yaml b/config/helm/ec2-metadata-test-proxy/templates/clusterrole.yaml index 9654e220..cbeb2a21 100644 --- a/config/helm/ec2-metadata-test-proxy/templates/clusterrole.yaml +++ b/config/helm/ec2-metadata-test-proxy/templates/clusterrole.yaml @@ -9,29 +9,4 @@ rules: - nodes verbs: - get - - patch - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - list -- apiGroups: - - "" - resources: - - pods/eviction - verbs: - - create -- apiGroups: - - extensions - resources: - - daemonsets - verbs: - - get -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get + \ No newline at end of file diff --git a/config/helm/ec2-metadata-test-proxy/templates/daemonset.yaml b/config/helm/ec2-metadata-test-proxy/templates/daemonset.yaml index f4344778..81fbb432 100644 --- a/config/helm/ec2-metadata-test-proxy/templates/daemonset.yaml +++ b/config/helm/ec2-metadata-test-proxy/templates/daemonset.yaml @@ -37,6 +37,3 @@ spec: - name: ENABLE_IMDS_V2 value: {{ .Values.ec2MetadataTestProxy.enableIMDSV2 | quote }} {{- end -}} - - - diff --git a/config/helm/ec2-metadata-test-proxy/templates/regular-pod-test.yaml b/config/helm/ec2-metadata-test-proxy/templates/regular-pod-test.yaml index f52ad715..69ccd562 100644 --- a/config/helm/ec2-metadata-test-proxy/templates/regular-pod-test.yaml +++ b/config/helm/ec2-metadata-test-proxy/templates/regular-pod-test.yaml @@ -27,6 +27,3 @@ spec: - name: PORT value: {{ .Values.regularPodTest.port | quote }} {{- end -}} - - - diff --git a/config/helm/ec2-metadata-test-proxy/templates/service.yaml b/config/helm/ec2-metadata-test-proxy/templates/service.yaml index dd7416c5..b73d3708 100644 --- a/config/helm/ec2-metadata-test-proxy/templates/service.yaml +++ b/config/helm/ec2-metadata-test-proxy/templates/service.yaml @@ -7,4 +7,4 @@ spec: app: {{ .Values.ec2MetadataTestProxy.label }} ports: - port: {{ .Values.ec2MetadataTestProxy.port }} - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/config/helm/squid/.helmignore b/config/helm/squid/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/config/helm/squid/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/config/helm/squid/Chart.yaml b/config/helm/squid/Chart.yaml new file mode 100644 index 00000000..def21d0d --- /dev/null +++ b/config/helm/squid/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +name: squid +description: A Helm chart for Squid +version: 0.0.1 +appVersion: 1.0.0 +home: https://github.com/aws/aws-node-termination-handler +icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png +sources: + - https://github.com/aws/aws-node-termination-handler/test/squid diff --git a/config/helm/squid/templates/_helpers.tpl b/config/helm/squid/templates/_helpers.tpl new file mode 100644 index 00000000..5ca6a6e1 --- /dev/null +++ b/config/helm/squid/templates/_helpers.tpl @@ -0,0 +1,57 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "squid.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "squid.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "squid.labels" -}} +app.kubernetes.io/name: {{ include "squid.name" . }} +helm.sh/chart: {{ include "squid.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +k8s-app: squid +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "squid.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "squid.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "squid.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/config/helm/squid/templates/clusterrole.yaml b/config/helm/squid/templates/clusterrole.yaml new file mode 100644 index 00000000..024aad46 --- /dev/null +++ b/config/helm/squid/templates/clusterrole.yaml @@ -0,0 +1,11 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "squid.fullname" . }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get diff --git a/config/helm/squid/templates/clusterrolebinding.yaml b/config/helm/squid/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..7ea846ef --- /dev/null +++ b/config/helm/squid/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "squid.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "squid.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "squid.fullname" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/config/helm/squid/templates/daemonset.yaml b/config/helm/squid/templates/daemonset.yaml new file mode 100644 index 00000000..10bece42 --- /dev/null +++ b/config/helm/squid/templates/daemonset.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ .Values.squid.label }} + labels: + app: {{ .Values.squid.label }} +spec: + selector: + matchLabels: + app: {{ .Values.squid.label }} + template: + metadata: + labels: + app: {{ .Values.squid.label }} + spec: + serviceAccountName: {{ template "squid.serviceAccountName" . }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: {{ .Values.squid.label }} + image: {{ .Values.squid.image.repository }}:{{ .Values.squid.image.tag }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: {{ .Values.squid.port }} + hostPort: {{ .Values.squid.port }} + volumeMounts: + - name: squid-config + mountPath: /etc/squid + readOnly: true + volumes: + - name: squid-config + configMap: + name: {{ .Values.squid.configMap }} + \ No newline at end of file diff --git a/config/helm/squid/templates/psp.yaml b/config/helm/squid/templates/psp.yaml new file mode 100644 index 00000000..abf07ecd --- /dev/null +++ b/config/helm/squid/templates/psp.yaml @@ -0,0 +1,60 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "squid.fullname" . }} + labels: +{{ include "squid.labels" . | indent 4 }} + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + privileged: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 1024 + max: 65535 + hostPID: false + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "squid.fullname" . }}-psp + labels: +{{ include "squid.labels" . | indent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "squid.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "squid.fullname" . }}-psp + labels: +{{ include "squid.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "squid.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "squid.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/config/helm/squid/templates/serviceaccount.yaml b/config/helm/squid/templates/serviceaccount.yaml new file mode 100644 index 00000000..109fe80b --- /dev/null +++ b/config/helm/squid/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "squid.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- with .Values.serviceAccount.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + labels: +{{ include "squid.labels" . | indent 4 }} diff --git a/config/helm/squid/values.yaml b/config/helm/squid/values.yaml new file mode 100644 index 00000000..19ea64ff --- /dev/null +++ b/config/helm/squid/values.yaml @@ -0,0 +1,18 @@ +nameOverride: "" +fullnameOverride: "" +priorityClassName: system-node-critical +podAnnotations: {} +rbac: + pspEnabled: true +serviceAccount: + name: squid-sa + create: true + annotations: {} +squid: + port: 3128 + label: squid + configMap: squidConfigMap + image: + repository: squid + tag: customtest + \ No newline at end of file diff --git a/test/assets/squid.conf b/test/assets/squid.conf new file mode 100644 index 00000000..09f5a67c --- /dev/null +++ b/test/assets/squid.conf @@ -0,0 +1,11 @@ +acl SSL_ports port 443 +acl CONNECT method CONNECT +acl all src 0.0.0.0/0 +http_access allow all +http_port 3128 +coredump_dir /var/spool/squid +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern (Release|Packages(.gz)*)$ 0 20% 2880 +refresh_pattern . 0 20% 4320 \ No newline at end of file diff --git a/test/e2e/webhook-http-proxy-test b/test/e2e/webhook-http-proxy-test new file mode 100755 index 00000000..be73dbda --- /dev/null +++ b/test/e2e/webhook-http-proxy-test @@ -0,0 +1,103 @@ +#!/bin/bash +set -euo pipefail + +# Available env vars: +# $TMP_DIR +# $CLUSTER_NAME +# $KUBECONFIG +# $NODE_TERMINATION_HANDLER_DOCKER_REPO +# $NODE_TERMINATION_HANDLER_DOCKER_TAG +# $EC2_METADATA_DOCKER_REPO +# $EC2_METADATA_DOCKER_TAG + +echo "Starting Webhook HTTP Proxy Test for Node Termination Handler" + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +SQUID_DOCKERHUB_IMG="sameersbn/squid:3.5.27-2" +SQUID_DOCKER_IMG="squid:customtest" + + +### LOCAL ONLY TESTS FOR 200 RESPONSE FROM LOCAL CLUSTER, MASTER WILL TEST WITH TRAVIS SECRET URL +if [[ -z $(env | grep "WEBHOOK_URL=") ]]; then + WEBHOOK_URL="http://127.0.0.1:$IMDS_PORT" +fi + + +docker pull $SQUID_DOCKERHUB_IMG +docker tag $SQUID_DOCKERHUB_IMG $SQUID_DOCKER_IMG +kind load docker-image --name $CLUSTER_NAME --nodes=$CLUSTER_NAME-worker,$CLUSTER_NAME-control-plane $SQUID_DOCKER_IMG + +kubectl delete configmap squid-config || : +kubectl create configmap squid-config --from-file=$SCRIPTPATH/../assets/squid.conf + +helm upgrade --install $CLUSTER_NAME-squid $SCRIPTPATH/../../config/helm/squid/ \ + --force \ + --namespace default \ + --set squid.configMap="squid-config" \ + --set squid.image.repository="squid" \ + --set squid.image.tag="customtest" + +sleep 20 + +helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node-termination-handler/ \ + --force \ + --namespace kube-system \ + --set instanceMetadataURL="http://localhost:$IMDS_PORT" \ + --set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \ + --set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \ + --set enableSpotInterruptionDraining="true" \ + --set enableScheduledEventDraining="true" \ + --set webhookURL="$WEBHOOK_URL" \ + --set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - InstanceType: \{\{ \.InstanceType \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\}\"\}" \ + --set webhookProxy="tcp://localhost:3128" + +helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-metadata-test-proxy/ \ + --force \ + --namespace default \ + --set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \ + --set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \ + --set ec2MetadataTestProxy.enableSpotITN="true" \ + --set ec2MetadataTestProxy.port="$IMDS_PORT" + +TAINT_CHECK_CYCLES=15 +TAINT_CHECK_SLEEP=15 + +DEPLOYED=0 + +for i in `seq 1 10`; do + if [[ $(kubectl get deployments regular-pod-test -o jsonpath='{.status.unavailableReplicas}') -eq 0 ]]; then + echo "✅ Verified regular-pod-test pod was scheduled and started!" + DEPLOYED=1 + break + fi + sleep 5 +done + +if [[ $DEPLOYED -eq 0 ]]; then + exit 2 +fi + +for i in `seq 1 $TAINT_CHECK_CYCLES`; do + if kubectl get nodes $CLUSTER_NAME-worker | grep SchedulingDisabled; then + echo "✅ Verified the worker node was cordoned!" + if [[ $(kubectl get deployments regular-pod-test -o=jsonpath='{.status.unavailableReplicas}') -eq 1 ]]; then + echo "✅ Verified the regular-pod-test pod was evicted!" + NTH_POD_NAME=$(get_nth_worker_pod) + if kubectl logs $NTH_POD_NAME -n kube-system | grep 'Webhook Success'; then + echo "✅ Webhook Successfully Sent $CLUSTER_NAME! ✅" + pods=$(kubectl get pods -o json) + ## queries for the squid pod on the worker node + squid_worker_pods=$(echo $pods | jq '.items[] | select( .metadata.name | contains("squid") ) | .metadata.name as $name | select( .spec.nodeName | contains("worker") ) | .spec.nodeName as $nodename | $name' -r) + ## return only 1 pod + if kubectl exec -it $(echo $squid_worker_pods | cut -d' ' -f1) -- cat /var/log/squid/access.log | grep -i 'TCP_MISS/200' | grep '200'; then + echo "✅ Verified the webhook POST used the http proxy" + exit 0 + fi + fi + fi + + fi + sleep $TAINT_CHECK_SLEEP +done + +exit 1