diff --git a/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go b/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go
index 210062109..514d9ba0a 100644
--- a/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go
+++ b/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go
@@ -152,7 +152,7 @@ func (opts *completedProxyRunOptions) ProxyConfig() (*server.KubeRBACProxyConfig
 		return nil, err
 	}
 
-	if err := opts.ProxyOptions.ApplyTo(proxyConfig); err != nil {
+	if err := opts.ProxyOptions.ApplyTo(proxyConfig.KubeRBACProxyInfo, proxyConfig.DelegatingAuthentication, proxyConfig.SecureServing); err != nil {
 		return nil, err
 	}
 
diff --git a/cmd/kube-rbac-proxy/app/options/proxyoptions.go b/cmd/kube-rbac-proxy/app/options/proxyoptions.go
index d987676d0..ee5c7e2cd 100644
--- a/cmd/kube-rbac-proxy/app/options/proxyoptions.go
+++ b/cmd/kube-rbac-proxy/app/options/proxyoptions.go
@@ -28,6 +28,7 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/spf13/pflag"
 
+	serverconfig "k8s.io/apiserver/pkg/server"
 	"k8s.io/klog/v2"
 
 	"github.com/brancz/kube-rbac-proxy/pkg/authn/identityheaders"
@@ -120,30 +121,30 @@ func (o *ProxyOptions) Validate() []error {
 	return errs
 }
 
-func (o *ProxyOptions) ApplyTo(config *server.KubeRBACProxyConfig) error {
+func (o *ProxyOptions) ApplyTo(krpInfo *server.KubeRBACProxyInfo, authInfo *serverconfig.AuthenticationInfo, serving *serverconfig.SecureServingInfo) error {
 	var err error
 
-	config.KubeRBACProxyInfo.UpstreamURL, err = url.Parse(o.Upstream)
+	krpInfo.UpstreamURL, err = url.Parse(o.Upstream)
 	if err != nil {
 		return fmt.Errorf("failed to parse upstream URL: %w", err)
 	}
 
-	if err := config.KubeRBACProxyInfo.SetUpstreamTransport(o.UpstreamCAFile, o.UpstreamClientCertFile, o.UpstreamClientKeyFile); err != nil {
+	if err := krpInfo.SetUpstreamTransport(o.UpstreamCAFile, o.UpstreamClientCertFile, o.UpstreamClientKeyFile); err != nil {
 		return fmt.Errorf("failed to setup transport for upstream: %w", err)
 	}
 
 	if configFileName := o.ConfigFileName; len(configFileName) > 0 {
-		config.KubeRBACProxyInfo.Authorization, err = parseAuthorizationConfigFile(configFileName)
+		krpInfo.Authorization, err = parseAuthorizationConfigFile(configFileName)
 		if err != nil {
 			return fmt.Errorf("failed to read the config file: %w", err)
 		}
 	}
 
-	config.SecureServing.DisableHTTP2 = o.DisableHTTP2Serving
-	config.KubeRBACProxyInfo.UpstreamHeaders = o.UpstreamHeader
-	config.KubeRBACProxyInfo.IgnorePaths = o.IgnorePaths
-	config.KubeRBACProxyInfo.AllowPaths = o.AllowPaths
-	config.DelegatingAuthentication.APIAudiences = o.TokenAudiences
+	serving.DisableHTTP2 = o.DisableHTTP2Serving
+	krpInfo.UpstreamHeaders = o.UpstreamHeader
+	krpInfo.IgnorePaths = o.IgnorePaths
+	krpInfo.AllowPaths = o.AllowPaths
+	authInfo.APIAudiences = o.TokenAudiences
 
 	return nil
 }