From 37f2269808b1ac660d237db8f724d04529bcb187 Mon Sep 17 00:00:00 2001 From: Tanmay Satam Date: Mon, 30 Sep 2024 11:19:29 -0400 Subject: [PATCH 1/5] Add diagnostic data on MUO test failure --- test/e2e/operator.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 097ac9d149d..8edfb2d2fb8 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -472,6 +472,20 @@ var _ = Describe("ARO Operator - MUO Deployment", func() { managedUpgradeOperatorDeployment = "managed-upgrade-operator" ) + JustAfterEach(func(ctx context.Context) { + if CurrentSpecReport().Failed() { + deployment, err := clients.Kubernetes.AppsV1(). + Deployments(managedUpgradeOperatorDeployment). + Get(ctx, managedUpgradeOperatorDeployment, metav1.GetOptions{}) + + if err != nil { + AddReportEntry("muo-deployment-get-err", err) + } else { + AddReportEntry("muo-deployment-get", deployment) + } + } + }) + It("must be deployed by default with FIPS crypto mandated", func(ctx context.Context) { By("getting MUO pods") pods := ListK8sObjectWithRetry( From c92fbfe8abd3affc2a3e1264ece00d49505350ae Mon Sep 17 00:00:00 2001 From: Tanmay Satam Date: Mon, 30 Sep 2024 11:50:46 -0400 Subject: [PATCH 2/5] Add explicit step in E2E to validate MUO deployment is Available --- test/e2e/operator.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 8edfb2d2fb8..2b177111850 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -23,6 +23,7 @@ import ( cov1Helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/ugorji/go/codec" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -486,6 +487,27 @@ var _ = Describe("ARO Operator - MUO Deployment", func() { } }) + It("must be Available", func(ctx context.Context) { + By("getting MUO deployment and verifying its Available condition") + Eventually(func(g Gomega, ctx context.Context) { + deployment, err := clients.Kubernetes.AppsV1(). + Deployments(managedUpgradeOperatorNamespace). + Get(ctx, managedUpgradeOperatorDeployment, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + getAvailableConditionStatus := func(d *appsv1.Deployment) corev1.ConditionStatus { + for _, c := range d.Status.Conditions { + if c.Type == appsv1.DeploymentAvailable { + return c.Status + } + } + return corev1.ConditionUnknown + } + + g.Expect(deployment).To(WithTransform(getAvailableConditionStatus, Equal(corev1.ConditionTrue))) + }).WithContext(ctx).WithTimeout(DefaultTimeout).WithPolling(PollingInterval).Should(Succeed()) + }) + It("must be deployed by default with FIPS crypto mandated", func(ctx context.Context) { By("getting MUO pods") pods := ListK8sObjectWithRetry( From 5f9ad2a2e3e8770b63b8d2cae0186f41d56a0a49 Mon Sep 17 00:00:00 2001 From: Tanmay Satam Date: Mon, 30 Sep 2024 11:51:13 -0400 Subject: [PATCH 3/5] DROP: focus/repeat MUO E2E test --- test/e2e/operator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 2b177111850..48d56aa7d33 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -467,7 +467,7 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { }) }) -var _ = Describe("ARO Operator - MUO Deployment", func() { +var _ = Describe("ARO Operator - MUO Deployment", Focus, MustPassRepeatedly(10), func() { const ( managedUpgradeOperatorNamespace = "openshift-managed-upgrade-operator" managedUpgradeOperatorDeployment = "managed-upgrade-operator" From 0166b4d21190c6e90a5931f1804840175004a55e Mon Sep 17 00:00:00 2001 From: Tanmay Satam Date: Mon, 30 Sep 2024 15:42:00 -0400 Subject: [PATCH 4/5] Add logging/condition setting to MUO controller --- .../controllers/muo/muo_controller.go | 39 ++++-- .../controllers/muo/muo_controller_test.go | 127 ++++++++++++++++-- 2 files changed, 141 insertions(+), 25 deletions(-) diff --git a/pkg/operator/controllers/muo/muo_controller.go b/pkg/operator/controllers/muo/muo_controller.go index 0e05fe8314d..c4f356a7fb6 100644 --- a/pkg/operator/controllers/muo/muo_controller.go +++ b/pkg/operator/controllers/muo/muo_controller.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/ARO-RP/pkg/operator" arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/operator/controllers/base" "github.com/Azure/ARO-RP/pkg/operator/controllers/muo/config" "github.com/Azure/ARO-RP/pkg/operator/predicates" "github.com/Azure/ARO-RP/pkg/util/deployer" @@ -54,24 +55,24 @@ type MUODeploymentConfig struct { } type Reconciler struct { - log *logrus.Entry + base.AROController deployer deployer.Deployer - client client.Client - readinessPollTime time.Duration readinessTimeout time.Duration } func NewReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Interface) *Reconciler { return &Reconciler{ - log: log, + AROController: base.AROController{ + Log: log, + Client: client, + Name: ControllerName, + }, deployer: deployer.NewDeployer(client, dh, staticFiles, "staticresources"), - client: client, - readinessPollTime: 10 * time.Second, readinessTimeout: 5 * time.Minute, } @@ -79,17 +80,17 @@ func NewReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Int func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { instance := &arov1alpha1.Cluster{} - err := r.client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) + err := r.Client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) if err != nil { return reconcile.Result{}, err } if !instance.Spec.OperatorFlags.GetSimpleBoolean(operator.MuoEnabled) { - r.log.Debug("controller is disabled") + r.Log.Debug("controller is disabled") return reconcile.Result{}, nil } - r.log.Debug("running") + r.Log.Debug("running") managed := instance.Spec.OperatorFlags.GetWithDefault(operator.MuoManaged, "") @@ -103,8 +104,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. pullSpec = version.MUOImage(instance.Spec.ACRDomain) } - usePodSecurityAdmission, err := operator.ShouldUsePodSecurityStandard(ctx, r.client) + usePodSecurityAdmission, err := operator.ShouldUsePodSecurityStandard(ctx, r.Client) if err != nil { + r.Log.Error(err) + r.SetDegraded(ctx, err) + return reconcile.Result{}, err } @@ -117,7 +121,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. if !disableOCM { useOCM := func() bool { userSecret := &corev1.Secret{} - err = r.client.Get(ctx, pullSecretName, userSecret) + err = r.Client.Get(ctx, pullSecretName, userSecret) if err != nil { // if a pullsecret doesn't exist/etc, fallback to local return false @@ -146,6 +150,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. // Deploy the MUO manifests and config err = r.deployer.CreateOrUpdate(ctx, instance, config) if err != nil { + r.Log.Error(err) + r.SetDegraded(ctx, err) + return reconcile.Result{}, err } @@ -157,15 +164,23 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return r.deployer.IsReady(ctx, "openshift-managed-upgrade-operator", "managed-upgrade-operator") }, timeoutCtx.Done()) if err != nil { - return reconcile.Result{}, fmt.Errorf("managed Upgrade Operator deployment timed out on Ready: %w", err) + err = fmt.Errorf("managed Upgrade Operator deployment timed out on Ready: %w", err) + r.Log.Error(err) + r.SetDegraded(ctx, err) + + return reconcile.Result{}, err } } else if strings.EqualFold(managed, "false") { err := r.deployer.Remove(ctx, config.MUODeploymentConfig{}) if err != nil { + r.Log.Error(err) + r.SetDegraded(ctx, err) + return reconcile.Result{}, err } } + r.ClearConditions(ctx) return reconcile.Result{}, nil } diff --git a/pkg/operator/controllers/muo/muo_controller_test.go b/pkg/operator/controllers/muo/muo_controller_test.go index f61c98a160c..cd533343439 100644 --- a/pkg/operator/controllers/muo/muo_controller_test.go +++ b/pkg/operator/controllers/muo/muo_controller_test.go @@ -6,10 +6,13 @@ package muo import ( "context" "errors" + "fmt" "testing" "time" + "github.com/google/go-cmp/cmp/cmpopts" configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" "github.com/sirupsen/logrus" "go.uber.org/mock/gomock" corev1 "k8s.io/api/core/v1" @@ -19,13 +22,23 @@ import ( "github.com/Azure/ARO-RP/pkg/operator" arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/operator/controllers/base" "github.com/Azure/ARO-RP/pkg/operator/controllers/muo/config" + "github.com/Azure/ARO-RP/pkg/util/cmp" mock_deployer "github.com/Azure/ARO-RP/pkg/util/mocks/deployer" _ "github.com/Azure/ARO-RP/pkg/util/scheme" + utilconditions "github.com/Azure/ARO-RP/test/util/conditions" utilerror "github.com/Azure/ARO-RP/test/util/error" ) func TestMUOReconciler(t *testing.T) { + transitionTime := metav1.Time{Time: time.Now()} + defaultAvailable := utilconditions.ControllerDefaultAvailable(ControllerName) + defaultProgressing := utilconditions.ControllerDefaultProgressing(ControllerName) + defaultDegraded := utilconditions.ControllerDefaultDegraded(ControllerName) + + defaultConditions := []operatorv1.OperatorCondition{defaultAvailable, defaultProgressing, defaultDegraded} + tests := []struct { name string mocks func(*mock_deployer.MockDeployer, *arov1alpha1.Cluster) @@ -34,7 +47,8 @@ func TestMUOReconciler(t *testing.T) { // connected MUO -- cluster pullsecret pullsecret string // errors - wantErr string + wantErr string + wantConditions []operatorv1.OperatorCondition }{ { name: "disabled", @@ -43,6 +57,7 @@ func TestMUOReconciler(t *testing.T) { operator.MuoManaged: operator.FlagFalse, controllerPullSpec: "wonderfulPullspec", }, + wantConditions: defaultConditions, }, { name: "managed", @@ -57,9 +72,10 @@ func TestMUOReconciler(t *testing.T) { Pullspec: "wonderfulPullspec", EnableConnected: false, } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, no pullspec (uses default)", @@ -73,9 +89,10 @@ func TestMUOReconciler(t *testing.T) { Pullspec: "acrtest.example.com/app-sre/managed-upgrade-operator:v0.1.952-44b631a", EnableConnected: false, } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, OCM allowed but pull secret entirely missing", @@ -91,9 +108,10 @@ func TestMUOReconciler(t *testing.T) { Pullspec: "wonderfulPullspec", EnableConnected: false, } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, OCM allowed but empty pullsecret", @@ -110,9 +128,10 @@ func TestMUOReconciler(t *testing.T) { Pullspec: "wonderfulPullspec", EnableConnected: false, } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, OCM allowed but mangled pullsecret", @@ -129,9 +148,10 @@ func TestMUOReconciler(t *testing.T) { Pullspec: "wonderfulPullspec", EnableConnected: false, } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, OCM connected mode", @@ -149,9 +169,10 @@ func TestMUOReconciler(t *testing.T) { EnableConnected: true, OCMBaseURL: "https://api.openshift.com", } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, OCM connected mode, custom OCM URL", @@ -170,9 +191,10 @@ func TestMUOReconciler(t *testing.T) { EnableConnected: true, OCMBaseURL: "https://example.com", } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, pull secret exists, OCM disabled", @@ -189,9 +211,10 @@ func TestMUOReconciler(t *testing.T) { Pullspec: "wonderfulPullspec", EnableConnected: false, } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) }, + wantConditions: defaultConditions, }, { name: "managed, MUO does not become ready", @@ -207,10 +230,20 @@ func TestMUOReconciler(t *testing.T) { EnableConnected: false, SupportsPodSecurityAdmission: true, } - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, expectedConfig).Return(nil) + md.EXPECT().CreateOrUpdate(gomock.Any(), matchesCluster(cluster), expectedConfig).Return(nil) md.EXPECT().IsReady(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil) }, wantErr: "managed Upgrade Operator deployment timed out on Ready: timed out waiting for the condition", + wantConditions: []operatorv1.OperatorCondition{ + defaultAvailable, + defaultProgressing, + { + Type: ControllerName + "Controller" + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + LastTransitionTime: transitionTime, + Message: "managed Upgrade Operator deployment timed out on Ready: timed out waiting for the condition", + }, + }, }, { name: "managed, could not parse cluster version fails", @@ -220,6 +253,16 @@ func TestMUOReconciler(t *testing.T) { controllerPullSpec: "wonderfulPullspec", }, wantErr: `could not parse version ""`, + wantConditions: []operatorv1.OperatorCondition{ + defaultAvailable, + defaultProgressing, + { + Type: ControllerName + "Controller" + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + LastTransitionTime: transitionTime, + Message: `could not parse version ""`, + }, + }, }, { name: "managed, CreateOrUpdate() fails", @@ -230,9 +273,25 @@ func TestMUOReconciler(t *testing.T) { }, clusterVersion: "4.10.0", mocks: func(md *mock_deployer.MockDeployer, cluster *arov1alpha1.Cluster) { - md.EXPECT().CreateOrUpdate(gomock.Any(), cluster, gomock.AssignableToTypeOf(&config.MUODeploymentConfig{})).Return(errors.New("failed ensure")) + md.EXPECT(). + CreateOrUpdate( + gomock.Any(), + matchesCluster(cluster), + gomock.AssignableToTypeOf(&config.MUODeploymentConfig{}), + ). + Return(errors.New("failed ensure")) }, wantErr: "failed ensure", + wantConditions: []operatorv1.OperatorCondition{ + defaultAvailable, + defaultProgressing, + { + Type: ControllerName + "Controller" + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + LastTransitionTime: transitionTime, + Message: "failed ensure", + }, + }, }, { name: "managed=false (removal)", @@ -244,6 +303,7 @@ func TestMUOReconciler(t *testing.T) { mocks: func(md *mock_deployer.MockDeployer, cluster *arov1alpha1.Cluster) { md.EXPECT().Remove(gomock.Any(), gomock.Any()).Return(nil) }, + wantConditions: defaultConditions, }, { name: "managed=false (removal), Remove() fails", @@ -256,6 +316,16 @@ func TestMUOReconciler(t *testing.T) { md.EXPECT().Remove(gomock.Any(), gomock.Any()).Return(errors.New("failed delete")) }, wantErr: "failed delete", + wantConditions: []operatorv1.OperatorCondition{ + defaultAvailable, + defaultProgressing, + { + Type: ControllerName + "Controller" + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + LastTransitionTime: transitionTime, + Message: "failed delete", + }, + }, }, { name: "managed=blank (no action)", @@ -264,6 +334,7 @@ func TestMUOReconciler(t *testing.T) { operator.MuoManaged: "", controllerPullSpec: "wonderfulPullspec", }, + wantConditions: defaultConditions, }, } for _, tt := range tests { @@ -284,6 +355,9 @@ func TestMUOReconciler(t *testing.T) { OperatorFlags: tt.flags, ACRDomain: "acrtest.example.com", }, + Status: arov1alpha1.ClusterStatus{ + Conditions: defaultConditions, + }, } cv := &configv1.ClusterVersion{ @@ -319,14 +393,41 @@ func TestMUOReconciler(t *testing.T) { } r := &Reconciler{ - log: logrus.NewEntry(logrus.StandardLogger()), + AROController: base.AROController{ + Log: logrus.NewEntry(logrus.StandardLogger()), + Client: clientBuilder.Build(), + Name: ControllerName, + }, deployer: deployer, - client: clientBuilder.Build(), readinessTimeout: 0 * time.Second, readinessPollTime: 1 * time.Second, } _, err := r.Reconcile(ctx, reconcile.Request{}) utilerror.AssertErrorMessage(t, err, tt.wantErr) + utilconditions.AssertControllerConditions(t, ctx, r.AROController.Client, tt.wantConditions) }) } } + +type clusterMatcher struct { + expected *arov1alpha1.Cluster +} + +func (m clusterMatcher) Matches(actualRaw interface{}) bool { + if actual, ok := actualRaw.(*arov1alpha1.Cluster); !ok { + return false + } else { + if diff := cmp.Diff(m.expected, actual, cmpopts.EquateApproxTime(time.Second)); diff != "" { + return false + } + } + return true +} + +func (m clusterMatcher) String() string { + return fmt.Sprintf("matches clusterdoc %v", m.expected) +} + +func matchesCluster(expected *arov1alpha1.Cluster) clusterMatcher { + return clusterMatcher{expected: expected} +} From 3f30bd0815770ea8aa2056bec36acb7857b0aeaf Mon Sep 17 00:00:00 2001 From: Tanmay Satam Date: Mon, 30 Sep 2024 15:43:47 -0400 Subject: [PATCH 5/5] DROP: Log ARO operator on MUO e2e test failures --- .../controllers/muo/muo_controller.go | 3 ++- test/e2e/operator.go | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/operator/controllers/muo/muo_controller.go b/pkg/operator/controllers/muo/muo_controller.go index c4f356a7fb6..faee95de751 100644 --- a/pkg/operator/controllers/muo/muo_controller.go +++ b/pkg/operator/controllers/muo/muo_controller.go @@ -90,7 +90,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return reconcile.Result{}, nil } - r.Log.Debug("running") + // TODO: change me back to Debug + r.Log.Info("running") managed := instance.Spec.OperatorFlags.GetWithDefault(operator.MuoManaged, "") diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 48d56aa7d33..a5f4e9b0809 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -475,6 +475,26 @@ var _ = Describe("ARO Operator - MUO Deployment", Focus, MustPassRepeatedly(10), JustAfterEach(func(ctx context.Context) { if CurrentSpecReport().Failed() { + getOperatorFunc := clients.ConfigClient.ConfigV1().ClusterOperators().Get + operator := GetK8sObjectWithRetry(ctx, getOperatorFunc, "aro", metav1.GetOptions{}) + + operatorYaml, err := yaml.Marshal(operator) + + AddReportEntry("aro-operator", operatorYaml) + + aroOperatorMasterPods := ListK8sObjectWithRetry( + ctx, + clients.Kubernetes.CoreV1().Pods(aroOperatorNamespace).List, + metav1.ListOptions{LabelSelector: "app=aro-operator-master"}, + ) + Expect(aroOperatorMasterPods.Items).NotTo(BeEmpty()) + + logs := GetK8sPodLogsWithRetry( + ctx, aroOperatorNamespace, aroOperatorMasterPods.Items[0].Name, corev1.PodLogOptions{}, + ) + + AddReportEntry("aro-operator-logs", logs) + deployment, err := clients.Kubernetes.AppsV1(). Deployments(managedUpgradeOperatorDeployment). Get(ctx, managedUpgradeOperatorDeployment, metav1.GetOptions{})