From 4b1e031496fc5b4dbd709cd90624f445f079cfb6 Mon Sep 17 00:00:00 2001 From: Shafeeque E S Date: Mon, 27 Dec 2021 15:24:04 +0530 Subject: [PATCH 1/2] Add cli flag for `leader-election-resource-lock` --- cmd/nginx/flags.go | 5 +++++ docs/deploy/rbac.md | 4 ++++ internal/ingress/controller/controller.go | 9 +++++---- internal/ingress/controller/nginx.go | 6 ++++-- internal/ingress/controller/status.go | 21 +++++++++++++-------- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go index 337a1f8276..33be1bc7b5 100644 --- a/cmd/nginx/flags.go +++ b/cmd/nginx/flags.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/pflag" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/controller" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" @@ -127,6 +128,9 @@ Requires setting the publish-service parameter to a valid Service reference.`) electionID = flags.String("election-id", "ingress-controller-leader", `Election id to use for Ingress status updates.`) + leaderElectionResourceLock = flags.String("leader-election-resource-lock", resourcelock.ConfigMapsResourceLock, + `Resourcelock to use for ingress-controller leader-election. Supported values are "configmaps", "configmapsleases", "leases".`) + updateStatusOnShutdown = flags.Bool("update-status-on-shutdown", true, `Update the load-balancer status of Ingress objects when the controller shuts down. Requires the update-status parameter.`) @@ -309,6 +313,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g KubeConfigFile: *kubeConfigFile, UpdateStatus: *updateStatus, ElectionID: *electionID, + LeaderElectionResourceLock: *leaderElectionResourceLock, EnableProfiling: *profiling, EnableMetrics: *enableMetrics, MetricsPerHost: *metricsPerHost, diff --git a/docs/deploy/rbac.md b/docs/deploy/rbac.md index 8c36d19a7a..04bda05bef 100644 --- a/docs/deploy/rbac.md +++ b/docs/deploy/rbac.md @@ -47,6 +47,8 @@ permissions are granted to the Role named `ingress-nginx` Furthermore to support leader-election, the ingress-nginx-controller needs to have access to a `configmap` using the resourceName `ingress-controller-leader-nginx` +and also a `lease` using the resourceName `ingress-controller-leader-nginx` +if `configmapsleases` or `leases` is used as default leader-election resource-lock. > Note that resourceNames can NOT be used to limit requests using the “create” > verb because authorizers only have access to information that can be obtained @@ -55,6 +57,8 @@ have access to a `configmap` using the resourceName `ingress-controller-leader-n * `configmaps`: get, update (for resourceName `ingress-controller-leader-nginx`) * `configmaps`: create +* `leases`: get, update (for resourceName `ingress-controller-leader-nginx`) +* `leases`: create This resourceName is the concatenation of the `election-id` and the `ingress-class` as defined by the ingress-controller, which defaults to: diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index f43c72d921..623c587248 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -83,10 +83,11 @@ type Configuration struct { PublishService string PublishStatusAddress string - UpdateStatus bool - UseNodeInternalIP bool - ElectionID string - UpdateStatusOnShutdown bool + UpdateStatus bool + UseNodeInternalIP bool + ElectionID string + LeaderElectionResourceLock string + UpdateStatusOnShutdown bool HealthCheckHost string ListenPorts *ngx_config.ListenPorts diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 7ff772a96f..3f0ef83bdb 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -265,8 +265,10 @@ func (n *NGINXController) Start() { electionID := n.cfg.ElectionID setupLeaderElection(&leaderElectionConfig{ - Client: n.cfg.Client, - ElectionID: electionID, + Client: n.cfg.Client, + ElectionID: electionID, + LeaderElectionResourceLock: n.cfg.LeaderElectionResourceLock, + OnStartedLeading: func(stopCh chan struct{}) { if n.syncStatus != nil { go n.syncStatus.Run(stopCh) diff --git a/internal/ingress/controller/status.go b/internal/ingress/controller/status.go index 7b90594a92..fb44317246 100644 --- a/internal/ingress/controller/status.go +++ b/internal/ingress/controller/status.go @@ -25,7 +25,6 @@ import ( "k8s.io/klog/v2" apiv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/leaderelection" @@ -36,7 +35,8 @@ import ( type leaderElectionConfig struct { Client clientset.Interface - ElectionID string + ElectionID string + LeaderElectionResourceLock string OnStartedLeading func(chan struct{}) OnStoppedLeading func() @@ -93,19 +93,24 @@ func setupLeaderElection(config *leaderElectionConfig) { Host: hostname, }) - lock := resourcelock.ConfigMapLock{ - ConfigMapMeta: metav1.ObjectMeta{Namespace: k8s.IngressPodDetails.Namespace, Name: config.ElectionID}, - Client: config.Client.CoreV1(), - LockConfig: resourcelock.ResourceLockConfig{ + lock, err := resourcelock.New(config.LeaderElectionResourceLock, + k8s.IngressPodDetails.Namespace, + config.ElectionID, + config.Client.CoreV1(), + config.Client.CoordinationV1(), + resourcelock.ResourceLockConfig{ Identity: k8s.IngressPodDetails.Name, EventRecorder: recorder, }, + ) + if err != nil { + klog.Fatalf("unexpected error creating resource lock: %v", err) } ttl := 30 * time.Second - elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ - Lock: &lock, + elector, err = leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: lock, LeaseDuration: ttl, RenewDeadline: ttl / 2, RetryPeriod: ttl / 4, From 2fdaefe45b7a1a3c3ad9d9ca9c5effcc94a68e2a Mon Sep 17 00:00:00 2001 From: Shafeeque E S Date: Mon, 3 Jan 2022 12:10:18 +0530 Subject: [PATCH 2/2] Update RBAC --- charts/ingress-nginx/templates/clusterrole.yaml | 7 +++++++ .../ingress-nginx/templates/controller-role.yaml | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index c093f048a0..0e725ec06c 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -29,6 +29,13 @@ rules: verbs: - list - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch {{- if and .Values.controller.scope.enabled .Values.controller.scope.namespace }} - apiGroups: - "" diff --git a/charts/ingress-nginx/templates/controller-role.yaml b/charts/ingress-nginx/templates/controller-role.yaml index 47bbc32d00..8e5f8a0d7b 100644 --- a/charts/ingress-nginx/templates/controller-role.yaml +++ b/charts/ingress-nginx/templates/controller-role.yaml @@ -73,6 +73,21 @@ rules: - configmaps verbs: - create + - apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - {{ .Values.controller.electionID }} + verbs: + - get + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create - apiGroups: - "" resources: