diff --git a/cluster-autoscaler/cloudprovider/hetzner/README.md b/cluster-autoscaler/cloudprovider/hetzner/README.md index 1a6758f06b9b..b3ba63c5ea8c 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/README.md +++ b/cluster-autoscaler/cloudprovider/hetzner/README.md @@ -5,9 +5,10 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes. # Configuration `HCLOUD_TOKEN` Required Hetzner Cloud token. -`HCLOUD_CLOUD_INIT` Base64 encoded Cloud Init yaml with commands to join the cluster +`HCLOUD_CLOUD_INIT` Base64 encoded Cloud Init yaml with commands to join the cluster, Sample [examples/cloud-init.txt for (Kubernetes 1.20.1)](examples/cloud-init.txt) `HCLOUD_IMAGE` Defaults to `ubuntu-20.04`, @see https://docs.hetzner.cloud/#images - +`HCLOUD_NETWORK` Default empty , The name of the network that is used in the cluster , @see https://docs.hetzner.cloud/#networks +`HCLOUD_SSH_KEY` Default empty , This SSH Key will have access to the fresh created server, @see https://docs.hetzner.cloud/#ssh-keys Node groups must be defined with the `--nodes=::::` flag. Multiple flags will create multiple node pools. For example: ``` @@ -16,6 +17,8 @@ Multiple flags will create multiple node pools. For example: --nodes=1:10:CX41:NBG1:pool3 ``` +You can find a deployment sample under [examples/cluster-autoscaler-run-on-master.yaml](examples/cluster-autoscaler-run-on-master.yaml). Please be aware that you should change the values within this deployment to reflect your cluster. + # Development Make sure you're inside the root path of the [autoscaler diff --git a/cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt b/cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt new file mode 100644 index 000000000000..7192d5f43a95 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt @@ -0,0 +1,35 @@ +#!/bin/bash +export DEBIAN_FRONTEND=noninteractive +export HOME=/root/ +apt-get update && apt-get install -y apt-transport-https ca-certificates curl software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +cat </etc/apt/sources.list.d/kubernetes.list +deb https://apt.kubernetes.io/ kubernetes-xenial main +EOF +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get upgrade -y +apt-get install -y kubelet=1.20.1-00 kubeadm=1.20.1-00 kubectl=1.20.1-00 +apt-mark hold kubelet kubeadm kubectl +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" +apt update +apt install -y docker-ce +cat < /etc/docker/daemon.json < --token --discovery-token-ca-cert-hash sha256: diff --git a/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml b/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml new file mode 100644 index 000000000000..6b0aa5e0a242 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml @@ -0,0 +1,190 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + # Node affinity is used to force cluster-autoscaler to stick + # to the master node. This allows the cluster to reliably downscale + # to zero worker nodes when needed. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - image: k8s.gcr.io/autoscaling/cluster-autoscaler:latest # or your custom image + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --cloud-provider=hetzner + - --stderrthreshold=info + - --nodes=1:10:CPX11:FSN1:pool1 + env: + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: hcloud + key: token + - name: HCLOUD_CLOUD_INIT + value: +# - name: HCLOUD_SSH_KEY +# value: +# - name: HCLOUD_NETWORK +# value: + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + imagePullSecrets: + - name: gitlab-registry + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go index cf6a9818221d..e76de37b6356 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go @@ -21,10 +21,11 @@ import ( "encoding/base64" "errors" "fmt" - apiv1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud" "os" "strings" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud" ) var ( @@ -39,6 +40,8 @@ type hetznerManager struct { apiCallContext context.Context cloudInit string image string + sshKey *hcloud.SSHKey + network *hcloud.Network } func newManager() (*hetznerManager, error) { @@ -64,11 +67,33 @@ func newManager() (*hetznerManager, error) { return nil, fmt.Errorf("failed to parse cloud init error: %s", err) } + var network *hcloud.Network + networkName := os.Getenv("HCLOUD_NETWORK") + + var sshKey *hcloud.SSHKey + sshKeyName := os.Getenv("HCLOUD_SSH_KEY") + if sshKeyName != "" { + sshKey, _, err = client.SSHKey.Get(ctx, sshKeyName) + if err != nil { + return nil, fmt.Errorf("failed to get ssh key error: %s", err) + } + } + + if networkName != "" { + network, _, err = client.Network.Get(ctx, networkName) + if err != nil { + return nil, fmt.Errorf("failed to get network error: %s", err) + } + + } + m := &hetznerManager{ client: client, nodeGroups: make(map[string]*hetznerNodeGroup), cloudInit: string(cloudInit), image: image, + sshKey: sshKey, + network: network, apiCallContext: ctx, } diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go index addcf9e199e5..a620f068abd8 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go @@ -24,7 +24,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud" "k8s.io/klog/v2" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" "math/rand" "strconv" "sync" @@ -344,7 +344,7 @@ func serverTypeAvailable(manager *hetznerManager, instanceType string, region st func createServer(n *hetznerNodeGroup) error { StartAfterCreate := true - serverCreateResult, _, err := n.manager.client.Server.Create(n.manager.apiCallContext, hcloud.ServerCreateOpts{ + opts := hcloud.ServerCreateOpts{ Name: newNodeName(n), UserData: n.manager.cloudInit, Location: &hcloud.Location{Name: n.region}, @@ -354,10 +354,17 @@ func createServer(n *hetznerNodeGroup) error { Labels: map[string]string{ nodeGroupLabel: n.id, }, - }) + } + if n.manager.sshKey != nil { + opts.SSHKeys = []*hcloud.SSHKey{n.manager.sshKey} + } + if n.manager.network != nil { + opts.Networks = []*hcloud.Network{n.manager.network} + } + serverCreateResult, _, err := n.manager.client.Server.Create(n.manager.apiCallContext, opts) if err != nil { - return fmt.Errorf("could not create server type %s in region %s", n.instanceType, n.region) + return fmt.Errorf("could not create server type %s in region %s: %v", n.instanceType, n.region, err) } server := serverCreateResult.Server