diff --git a/pkg/leaderelection/leader_election.go b/pkg/leaderelection/leader_election.go index ee4fcf4cbe..5cc253917a 100644 --- a/pkg/leaderelection/leader_election.go +++ b/pkg/leaderelection/leader_election.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "os" + "time" "k8s.io/apimachinery/pkg/util/uuid" coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" @@ -49,6 +50,12 @@ type Options struct { // LeaderElectionID determines the name of the resource that leader election // will use for holding the leader lock. LeaderElectionID string + + // RenewDeadline is the renew deadline for this leader election client. + // Must be set to ensure the resource lock has an appropriate client timeout. + // Without that, a single slow response from the API server can result + // in losing leadership. + RenewDeadline time.Duration } // NewResourceLock creates a new resource lock for use in a leader election loop. @@ -88,6 +95,20 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op // Construct clients for leader election rest.AddUserAgent(config, "leader-election") + + if options.RenewDeadline != 0 { + return resourcelock.NewFromKubeconfig(options.LeaderElectionResourceLock, + options.LeaderElectionNamespace, + options.LeaderElectionID, + resourcelock.ResourceLockConfig{ + Identity: id, + EventRecorder: recorderProvider.GetEventRecorderFor(id), + }, + config, + options.RenewDeadline, + ) + } + corev1Client, err := corev1client.NewForConfig(config) if err != nil { return nil, err @@ -97,7 +118,6 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op if err != nil { return nil, err } - return resourcelock.New(options.LeaderElectionResourceLock, options.LeaderElectionNamespace, options.LeaderElectionID, @@ -106,7 +126,8 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op resourcelock.ResourceLockConfig{ Identity: id, EventRecorder: recorderProvider.GetEventRecorderFor(id), - }) + }, + ) } func getInClusterNamespace() (string, error) { diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 3166f4818f..92906fe6ca 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -389,6 +389,7 @@ func New(config *rest.Config, options Options) (Manager, error) { LeaderElectionResourceLock: options.LeaderElectionResourceLock, LeaderElectionID: options.LeaderElectionID, LeaderElectionNamespace: options.LeaderElectionNamespace, + RenewDeadline: *options.RenewDeadline, }) if err != nil { return nil, err diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index c42d2f2ae7..6e5353e345 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -317,6 +317,28 @@ var _ = Describe("manger.Manager", func() { <-m2done }) + It("should default RenewDeadline for leader election config", func() { + var rl resourcelock.Interface + m1, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-id", + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + if options.RenewDeadline != 10*time.Second { + return nil, fmt.Errorf("expected RenewDeadline to be 10s, got %v", options.RenewDeadline) + } + var err error + rl, err = leaderelection.NewResourceLock(config, recorderProvider, options) + return rl, err + }, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m1).ToNot(BeNil()) + }) + It("should default ID to controller-runtime if ID is not set", func() { var rl resourcelock.Interface m1, err := New(cfg, Options{