Skip to content

Commit f6ad3c8

Browse files
aleoliadamjensenbot
aleoli
authored and
adamjensenbot
committed
e2e plugin for eks
1 parent c93a90c commit f6ad3c8

File tree

8 files changed

+290
-4
lines changed

8 files changed

+290
-4
lines changed

‎deployments/liqo/templates/liqo-wireguard-gateway-server-template-eks.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
{{- $templateConfig := (merge (dict "name" "wireguard-server-eks" "module" "networking") .) -}}
1+
{{- $templateConfig := (merge (dict "name" "wireguard-server" "module" "networking") .) -}}
22
{{- $gatewayConfig := (merge (dict "name" "gateway" "module" "networking" "version" .Values.networking.gatewayTemplates.container.gateway.image.version) .) -}}
33
{{- $wireguardConfig := (merge (dict "name" "gateway-wireguard" "module" "networking" "version" .Values.networking.gatewayTemplates.container.wireguard.image.version) .) -}}
44
{{- $geneveConfig := (merge (dict "name" "gateway-geneve" "module" "networking" "version" .Values.networking.gatewayTemplates.container.geneve.image.version) .) -}}
55

6-
{{- if .Values.networking.enabled }}
6+
{{- if and .Values.networking.enabled .Values.authentication.awsConfig.accessKeyId }}
77

88
apiVersion: networking.liqo.io/v1beta1
99
kind: WgGatewayServerTemplate

‎deployments/liqo/templates/liqo-wireguard-gateway-server-template.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
{{- $wireguardConfig := (merge (dict "name" "gateway-wireguard" "module" "networking" "version" .Values.networking.gatewayTemplates.container.wireguard.image.version) .) -}}
44
{{- $geneveConfig := (merge (dict "name" "gateway-geneve" "module" "networking" "version" .Values.networking.gatewayTemplates.container.geneve.image.version) .) -}}
55

6-
{{- if .Values.networking.enabled }}
6+
{{- if and .Values.networking.enabled (not .Values.authentication.awsConfig.accessKeyId) }}
77

88
apiVersion: networking.liqo.io/v1beta1
99
kind: WgGatewayServerTemplate

‎pkg/identityManager/iamIdentityProvider.go

+37-1
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ func (identityProvider *iamIdentityProvider) ApproveSigningRequest(ctx context.C
162162
}
163163

164164
// the IAM username has to have <= 64 characters
165-
s := fmt.Sprintf("%s-%s", username, organization)
165+
s := fmt.Sprintf("%s-%s-%s", username, organization, identityProvider.localClusterID)
166166
h := sha256.New()
167167
_, _ = h.Write([]byte(s))
168168
bs := h.Sum(nil)
@@ -317,6 +317,42 @@ func (identityProvider *iamIdentityProvider) ensureIamAccessKey(ctx context.Cont
317317
}
318318

319319
createAccessKeyResult, err := iamSvc.CreateAccessKeyWithContext(ctx, createAccessKey)
320+
if err == nil {
321+
return createAccessKeyResult.AccessKey, nil
322+
}
323+
324+
// if the error is limit exceeded, we have to delete an existing access key
325+
if aerr, ok := err.(awserr.Error); ok { //nolint:errorlint // aws does not export a specific error type
326+
if aerr.Code() == iam.ErrCodeLimitExceededException {
327+
klog.Warningf("IAM user %v has reached the limit of access keys, Liqo will delete an existing access key", username)
328+
var accessKeyList *iam.ListAccessKeysOutput
329+
accessKeyList, err = iamSvc.ListAccessKeysWithContext(ctx, &iam.ListAccessKeysInput{
330+
UserName: aws.String(username),
331+
})
332+
if err != nil {
333+
klog.Error(err)
334+
return nil, err
335+
}
336+
337+
if len(accessKeyList.AccessKeyMetadata) == 0 {
338+
klog.Error("no access key found")
339+
return nil, fmt.Errorf("no access key found")
340+
}
341+
for _, accessKey := range accessKeyList.AccessKeyMetadata {
342+
_, err = iamSvc.DeleteAccessKeyWithContext(ctx, &iam.DeleteAccessKeyInput{
343+
AccessKeyId: accessKey.AccessKeyId,
344+
UserName: aws.String(username),
345+
})
346+
if err != nil {
347+
klog.Error(err)
348+
return nil, err
349+
}
350+
}
351+
352+
createAccessKeyResult, err = iamSvc.CreateAccessKeyWithContext(ctx, createAccessKey)
353+
}
354+
}
355+
320356
if err != nil {
321357
klog.Error(err)
322358
return nil, err

‎pkg/liqoctl/install/eks/policy.go

+2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ var policy = PolicyDocument{
3737
Action: []string{
3838
"iam:CreateUser",
3939
"iam:CreateAccessKey",
40+
"iam:ListAccessKeys",
41+
"iam:DeleteAccessKey",
4042
},
4143
Resource: "*",
4244
},

‎test/e2e/pipeline/infra/eks/clean.sh

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
3+
# This scripts expects the following variables to be set:
4+
# CLUSTER_NUMBER -> the number of liqo clusters
5+
# K8S_VERSION -> the Kubernetes version
6+
# CNI -> the CNI plugin used
7+
# TMPDIR -> the directory where the test-related files are stored
8+
# BINDIR -> the directory where the test-related binaries are stored
9+
# TEMPLATE_DIR -> the directory where to read the cluster templates
10+
# NAMESPACE -> the namespace where liqo is running
11+
# KUBECONFIGDIR -> the directory where the kubeconfigs are stored
12+
# LIQO_VERSION -> the liqo version to test
13+
# INFRA -> the Kubernetes provider for the infrastructure
14+
# LIQOCTL -> the path where liqoctl is stored
15+
# KUBECTL -> the path where kubectl is stored
16+
# EKSCTL -> the path where eksctl is stored
17+
# AWS_CLI -> the path where aws-cli is stored
18+
# POD_CIDR_OVERLAPPING -> the pod CIDR of the clusters is overlapping
19+
# CLUSTER_TEMPLATE_FILE -> the file where the cluster template is stored
20+
21+
set -e # Fail in case of error
22+
set -o nounset # Fail if undefined variables are used
23+
set -o pipefail # Fail if one of the piped commands fails
24+
25+
error() {
26+
local sourcefile=$1
27+
local lineno=$2
28+
echo "An error occurred at $sourcefile:$lineno."
29+
}
30+
trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
31+
32+
CLUSTER_NAME=cluster
33+
RUNNER_NAME=${RUNNER_NAME:-"test"}
34+
CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}"
35+
36+
PIDS=()
37+
38+
# Cleaning all remaining clusters
39+
for i in $(seq 1 "${CLUSTER_NUMBER}")
40+
do
41+
# if the cluster exists, delete it
42+
if "${EKSCTL}" get cluster --name "${CLUSTER_NAME}${i}" --region "eu-central-1" &> /dev/null; then
43+
echo "Deleting cluster ${CLUSTER_NAME}${i}"
44+
else
45+
echo "Cluster ${CLUSTER_NAME}${i} does not exist"
46+
continue
47+
fi
48+
"${EKSCTL}" delete cluster --name "${CLUSTER_NAME}${i}" --region "eu-central-1" --wait --force &
49+
PIDS+=($!)
50+
done
51+
52+
for PID in "${PIDS[@]}"; do
53+
wait "${PID}"
54+
done

‎test/e2e/pipeline/infra/eks/pre-requirements.sh

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/bin/bash
2+
#shellcheck disable=SC1091
3+
4+
# This scripts expects the following variables to be set:
5+
# CLUSTER_NUMBER -> the number of liqo clusters
6+
# K8S_VERSION -> the Kubernetes version
7+
# CNI -> the CNI plugin used
8+
# TMPDIR -> the directory where the test-related files are stored
9+
# BINDIR -> the directory where the test-related binaries are stored
10+
# TEMPLATE_DIR -> the directory where to read the cluster templates
11+
# NAMESPACE -> the namespace where liqo is running
12+
# KUBECONFIGDIR -> the directory where the kubeconfigs are stored
13+
# LIQO_VERSION -> the liqo version to test
14+
# INFRA -> the Kubernetes provider for the infrastructure
15+
# LIQOCTL -> the path where liqoctl is stored
16+
# KUBECTL -> the path where kubectl is stored
17+
# HELM -> the path where helm is stored
18+
# EKSCTL -> the path where eksctl is stored
19+
# AWS_CLI -> the path where aws-cli is stored
20+
# POD_CIDR_OVERLAPPING -> the pod CIDR of the clusters is overlapping
21+
# CLUSTER_TEMPLATE_FILE -> the file where the cluster template is stored
22+
23+
set -e # Fail in case of error
24+
set -o nounset # Fail if undefined variables are used
25+
set -o pipefail # Fail if one of the piped commands fails
26+
27+
error() {
28+
local sourcefile=$1
29+
local lineno=$2
30+
echo "An error occurred at $sourcefile:$lineno."
31+
}
32+
trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
33+
34+
FILEPATH=$(realpath "$0")
35+
WORKDIR=$(dirname "$FILEPATH")
36+
37+
# shellcheck source=../../utils.sh
38+
source "$WORKDIR/../../utils.sh"
39+
40+
setup_arch_and_os
41+
42+
install_kubectl "${OS}" "${ARCH}" "${K8S_VERSION}"
43+
44+
install_helm "${OS}" "${ARCH}"
45+
46+
if ! command -v "${EKSCTL}" &> /dev/null
47+
then
48+
ARCH=amd64
49+
PLATFORM=$(uname -s)_$ARCH
50+
echo "WARNING: eksctl could not be found. Downloading and installing it locally..."
51+
if ! curl --fail -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz"; then
52+
echo "Error: Unable to download eksctl for '${OS}-${ARCH}'"
53+
return 1
54+
fi
55+
tar -xzf "eksctl_$PLATFORM.tar.gz" && rm "eksctl_$PLATFORM.tar.gz"
56+
mv eksctl "${EKSCTL}"
57+
fi
58+
chmod +x "${EKSCTL}"
59+
echo "eksctl version:"
60+
"${EKSCTL}" version
61+
62+
if ! command -v "${AWS_CLI}" &> /dev/null
63+
then
64+
case $ARCH in
65+
arm64) AWS_ARCH="aarch64";;
66+
arm) AWS_ARCH="aarch64";;
67+
armv5) AWS_ARCH="aaarch64";;
68+
armv6) AWS_ARCH="aarch64";;
69+
amd64) AWS_ARCH="x86_64";;
70+
386) AWS_ARCH="x86_64";;
71+
*) echo "Error architecture '${ARCH}' unknown"; exit 1 ;;
72+
esac
73+
echo "WARNING: aws-cli could not be found. Downloading and installing it locally..."
74+
if ! curl --fail -sLO "https://awscli.amazonaws.com/awscli-exe-linux-$AWS_ARCH.zip"; then
75+
echo "Error: Unable to download aws-cli for '${OS}-${ARCH}'"
76+
return 1
77+
fi
78+
unzip awscli-exe-linux-${AWS_ARCH}.zip
79+
./aws/install -i "${BINDIR}/aws-tmp" -b "${BINDIR}"
80+
rm -rf aws awscli-exe-linux-${AWS_ARCH}.zip
81+
fi
82+
chmod +x "${AWS_CLI}"
83+
echo "aws-cli version:"
84+
"${AWS_CLI}" --version

‎test/e2e/pipeline/infra/eks/setup.sh

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/bin/bash
2+
#shellcheck disable=SC1091
3+
4+
# This scripts expects the following variables to be set:
5+
# CLUSTER_NUMBER -> the number of liqo clusters
6+
# K8S_VERSION -> the Kubernetes version
7+
# CNI -> the CNI plugin used
8+
# TMPDIR -> the directory where the test-related files are stored
9+
# BINDIR -> the directory where the test-related binaries are stored
10+
# TEMPLATE_DIR -> the directory where to read the cluster templates
11+
# NAMESPACE -> the namespace where liqo is running
12+
# KUBECONFIGDIR -> the directory where the kubeconfigs are stored
13+
# LIQO_VERSION -> the liqo version to test
14+
# INFRA -> the Kubernetes provider for the infrastructure
15+
# LIQOCTL -> the path where liqoctl is stored
16+
# KUBECTL -> the path where kubectl is stored
17+
# HELM -> the path where helm is stored
18+
# EKSCTL -> the path where eksctl is stored
19+
# AWS_CLI -> the path where aws-cli is stored
20+
# POD_CIDR_OVERLAPPING -> the pod CIDR of the clusters is overlapping
21+
# CLUSTER_TEMPLATE_FILE -> the file where the cluster template is stored
22+
# CNI -> the CNI plugin used
23+
24+
set -e # Fail in case of error
25+
set -o nounset # Fail if undefined variables are used
26+
set -o pipefail # Fail if one of the piped commands fails
27+
28+
error() {
29+
local sourcefile=$1
30+
local lineno=$2
31+
echo "An error occurred at $sourcefile:$lineno."
32+
}
33+
trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
34+
35+
FILEPATH=$(realpath "$0")
36+
WORKDIR=$(dirname "$FILEPATH")
37+
38+
# shellcheck source=../../utils.sh
39+
source "$WORKDIR/../../utils.sh"
40+
41+
CLUSTER_NAME=cluster
42+
43+
export POD_CIDR=10.200.0.0/16
44+
export POD_CIDR_OVERLAPPING=${POD_CIDR_OVERLAPPING:-"false"}
45+
46+
RUNNER_NAME=${RUNNER_NAME:-"test"}
47+
CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}"
48+
49+
PIDS=()
50+
51+
for i in $(seq 1 "${CLUSTER_NUMBER}");
52+
do
53+
if [[ ${POD_CIDR_OVERLAPPING} != "true" ]]; then
54+
export POD_CIDR="10.$((i * 10)).0.0/16"
55+
fi
56+
echo "Creating cluster ${CLUSTER_NAME}${i}"
57+
"${EKSCTL}" create cluster \
58+
--name "${CLUSTER_NAME}${i}" \
59+
--region "eu-central-1" \
60+
--instance-types c4.large,c5.large \
61+
--nodes 2 \
62+
--managed \
63+
--alb-ingress-access \
64+
--node-ami-family "AmazonLinux2" \
65+
--vpc-cidr "$POD_CIDR" \
66+
--kubeconfig "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}" &
67+
PIDS+=($!)
68+
done
69+
70+
for PID in "${PIDS[@]}"; do
71+
wait "${PID}"
72+
done
73+
74+
for i in $(seq 1 "${CLUSTER_NUMBER}");
75+
do
76+
CURRENT_CONTEXT=$("${KUBECTL}" config current-context --kubeconfig "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}")
77+
"${KUBECTL}" config set contexts."${CURRENT_CONTEXT}".namespace default --kubeconfig "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
78+
79+
# install local-path storage class
80+
install_local_path_storage "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
81+
82+
# Install metrics-server
83+
install_metrics_server "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
84+
85+
# Install kyverno for network tests
86+
install_kyverno "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
87+
88+
# Install AWS Load Balancer Controller
89+
"${HELM}" repo add eks https://aws.github.io/eks-charts
90+
"${HELM}" repo update
91+
"${HELM}" install aws-load-balancer-controller eks/aws-load-balancer-controller \
92+
-n kube-system \
93+
--set clusterName="${CLUSTER_NAME}${i}" \
94+
--kubeconfig "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
95+
done

‎test/e2e/pipeline/installer/liqoctl/setup.sh

+15
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,21 @@ do
7575
if [[ "${INFRA}" == "k3s" ]]; then
7676
COMMON_ARGS=("${COMMON_ARGS[@]}" --pod-cidr "${POD_CIDR}" --service-cidr "${SERVICE_CIDR}")
7777
fi
78+
if [[ "${INFRA}" == "eks" ]]; then
79+
CLUSTER_NAME=cluster
80+
RUNNER_NAME=${RUNNER_NAME:-"test"}
81+
CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}"
82+
COMMON_ARGS=("${COMMON_ARGS[@]}" --eks-cluster-region="eu-central-1" --eks-cluster-name="${CLUSTER_NAME}${i}")
83+
# do not fail if variables are not set
84+
set +u
85+
if [[ "${LIQO_AWS_USERNAME}" != "" ]]; then
86+
COMMON_ARGS=("${COMMON_ARGS[@]}" --user-name "${LIQO_AWS_USERNAME}")
87+
fi
88+
if [[ "${LIQO_AWS_POLICY_NAME}" != "" ]]; then
89+
COMMON_ARGS=("${COMMON_ARGS[@]}" --policy-name "${LIQO_AWS_POLICY_NAME}")
90+
fi
91+
set -u
92+
fi
7893
if [[ "${INFRA}" == "cluster-api" ]]; then
7994
LIQO_PROVIDER="kubeadm"
8095
COMMON_ARGS=("${COMMON_ARGS[@]}")

0 commit comments

Comments
 (0)