diff --git a/cmd/main.go b/cmd/main.go
index 330b3cbc..ee1369f7 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -20,7 +20,6 @@ import (
 	"sigs.k8s.io/cli-utils/cmd/status"
 	"sigs.k8s.io/cli-utils/pkg/inventory"
 	"sigs.k8s.io/cli-utils/pkg/manifestreader"
-	"sigs.k8s.io/cli-utils/pkg/util/factory"
 
 	// This is here rather than in the libraries because of
 	// https://github.com/kubernetes-sigs/kustomize/issues/2060
@@ -42,9 +41,7 @@ func main() {
 	flags := cmd.PersistentFlags()
 	kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
 	kubeConfigFlags.AddFlags(flags)
-	matchVersionKubeConfigFlags := util.NewMatchVersionFlags(&factory.CachingRESTClientGetter{
-		Delegate: kubeConfigFlags,
-	})
+	matchVersionKubeConfigFlags := util.NewMatchVersionFlags(kubeConfigFlags)
 	matchVersionKubeConfigFlags.AddFlags(flags)
 	flags.AddGoFlagSet(flag.CommandLine)
 	f := util.NewFactory(matchVersionKubeConfigFlags)
diff --git a/pkg/apply/taskrunner/task.go b/pkg/apply/taskrunner/task.go
index a71284c0..8627632b 100644
--- a/pkg/apply/taskrunner/task.go
+++ b/pkg/apply/taskrunner/task.go
@@ -5,14 +5,11 @@ package taskrunner
 
 import (
 	"context"
-	"fmt"
-	"reflect"
 	"sync"
 	"time"
 
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/client-go/restmapper"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/cli-utils/pkg/apply/event"
 	"sigs.k8s.io/cli-utils/pkg/kstatus/status"
@@ -304,30 +301,5 @@ func (w *WaitTask) updateRESTMapper(taskContext *TaskContext) {
 	}
 
 	klog.V(5).Infof("resetting RESTMapper")
-	ddRESTMapper, err := extractDeferredDiscoveryRESTMapper(w.Mapper)
-	if err != nil {
-		if klog.V(4).Enabled() {
-			klog.Errorf("error resetting RESTMapper: %v", err)
-		}
-	}
-	ddRESTMapper.Reset()
-}
-
-// extractDeferredDiscoveryRESTMapper unwraps the provided RESTMapper
-// interface to get access to the underlying DeferredDiscoveryRESTMapper
-// that can be reset.
-func extractDeferredDiscoveryRESTMapper(mapper meta.RESTMapper) (
-	*restmapper.DeferredDiscoveryRESTMapper,
-	error,
-) {
-	val := reflect.ValueOf(mapper)
-	if val.Type().Kind() != reflect.Struct {
-		return nil, fmt.Errorf("unexpected RESTMapper type: %s", val.Type().String())
-	}
-	fv := val.FieldByName("RESTMapper")
-	ddRESTMapper, ok := fv.Interface().(*restmapper.DeferredDiscoveryRESTMapper)
-	if !ok {
-		return nil, fmt.Errorf("unexpected RESTMapper field type: %s", fv.Type())
-	}
-	return ddRESTMapper, nil
+	meta.MaybeResetRESTMapper(w.Mapper)
 }
diff --git a/pkg/util/factory/clientgetter.go b/pkg/util/factory/clientgetter.go
deleted file mode 100644
index b5b74065..00000000
--- a/pkg/util/factory/clientgetter.go
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2020 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package factory
-
-import (
-	"sync"
-
-	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
-	"k8s.io/client-go/discovery"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-)
-
-// CachingRESTClientGetter caches the RESTMapper so every call to
-// ToRESTMapper will get a reference to the same mapper.
-type CachingRESTClientGetter struct {
-	mx       sync.Mutex
-	Delegate genericclioptions.RESTClientGetter
-
-	mapper meta.RESTMapper
-}
-
-func (c *CachingRESTClientGetter) ToRESTConfig() (*rest.Config, error) {
-	return c.Delegate.ToRESTConfig()
-}
-
-func (c *CachingRESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
-	return c.Delegate.ToDiscoveryClient()
-}
-
-func (c *CachingRESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
-	c.mx.Lock()
-	defer c.mx.Unlock()
-	if c.mapper != nil {
-		return c.mapper, nil
-	}
-	var err error
-	c.mapper, err = c.Delegate.ToRESTMapper()
-	return c.mapper, err
-}
-
-func (c *CachingRESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
-	return c.Delegate.ToRawKubeConfigLoader()
-}
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index a5c0aeed..454708dd 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -24,7 +24,6 @@ import (
 	"sigs.k8s.io/cli-utils/pkg/apply"
 	"sigs.k8s.io/cli-utils/pkg/common"
 	"sigs.k8s.io/cli-utils/pkg/inventory"
-	"sigs.k8s.io/cli-utils/pkg/util/factory"
 	"sigs.k8s.io/cli-utils/test/e2e/customprovider"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -357,9 +356,7 @@ func newCustomInvDestroyer() *apply.Destroyer {
 
 func newFactory() util.Factory {
 	kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
-	matchVersionKubeConfigFlags := util.NewMatchVersionFlags(&factory.CachingRESTClientGetter{
-		Delegate: kubeConfigFlags,
-	})
+	matchVersionKubeConfigFlags := util.NewMatchVersionFlags(kubeConfigFlags)
 	return util.NewFactory(matchVersionKubeConfigFlags)
 }