Add support for cluster-scoped CRDs for kube-state-metrics (#2809)

Cluster-scoped CRD’s were not supported in the `pkg/kube-metrics` implementation since it was always looking for the GKV's in the List of namespaces informed which cause the issue "resource not found" since the CRD is not in any specific namespace. 

With the changes made here `pkg/kube-metrics` is able to List and Watch cluster-scoped CRDs.  Closes: #2707

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Added
 
+- Added support for generating kube-state-metrics metrics for cluster-scoped resources. Also added `pkg/kubemetrics.NewNamespacedMetricsStores` and `pkg/kubemetrics.NewClusterScopedMetricsStores` to support this new feature. ([#2809](https://github.com/operator-framework/operator-sdk/pull/2809))
 - Added the [`generate csv --deploy-dir --apis-dir --crd-dir`](website/content/en/docs/cli/operator-sdk_generate_csv.md#options) flags to allow configuring input locations for operator manifests and API types directories to the CSV generator in lieu of a config. See the CLI reference doc or `generate csv -h` help text for more details. ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511))
 - Added the [`generate csv --output-dir`](website/content/en/docs/cli/operator-sdk_generate_csv.md#options) flag to allow configuring the output location for the catalog directory. ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511))
 - The flag `--watch-namespace` and `--operator-namespace` was added to `operator-sdk run --local`, `operator-sdk test --local` and `operator-sdk cleanup` commands in order to replace the flag `--namespace` which was  deprecated.([#2617](https://github.com/operator-framework/operator-sdk/pull/2617))
@@ -30,6 +31,7 @@
 
 ### Deprecated
 
+- Deprecated `pkg/kubemetrics.NewMetricsStores`. Use `pkg/kubemetrics.NewNamespacedMetricsStores` instead. ([#2809](https://github.com/operator-framework/operator-sdk/pull/2809))
 - **Breaking Change:** The `--namespace` flag from `operator-sdk run --local`, `operator-sdk test --local` and `operator-sdk cleanup` command was deprecated and will be removed in the future versions. Use `--watch-namespace` and `--operator-namespace`  instead of. ([#2617](https://github.com/operator-framework/operator-sdk/pull/2617))
 - **Breaking Change:** The method `ctx.GetNamespace()` from the `pkg/test` is deprecated and will be removed in future versions. Use `ctx.GetOperatorNamespace()` and `ctx.GetWatchNamespace()` instead of. ([#2617](https://github.com/operator-framework/operator-sdk/pull/2617))
 - **Breaking Change:** package manifests are deprecated and new manifests are no longer generated; existing manifests are still updated by `operator-sdk generate csv`, but updates will not occur in future versions. Use [`operator-sdk bundle create`](./website/content/en/docs/cli/operator-sdk_bundle_create.md) to manage operator bundle metadata. ([#2755](https://github.com/operator-framework/operator-sdk/pull/2755))

pkg/kube-metrics/metrics.go

@@ -23,6 +23,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
 	ksmetric "k8s.io/kube-state-metrics/pkg/metric"
 	metricsstore "k8s.io/kube-state-metrics/pkg/metrics_store"
@@ -58,8 +59,18 @@ func GenerateAndServeCRMetrics(cfg *rest.Config,
 		if err != nil {
 			return err
 		}
+		namespaced, err := isNamespaced(gvk, cfg)
+		if err != nil {
+			return err
+		}
+		var gvkStores []*metricsstore.MetricsStore
+		if namespaced {
+			gvkStores = NewNamespacedMetricsStores(dclient, ns, apiVersion, kind, metricFamilies)
+		} else {
+			gvkStores = NewClusterScopedMetricsStores(dclient, apiVersion, kind, metricFamilies)
+		}
 		// Generate collector based on the group/version, kind and the metric families.
-		gvkStores := NewMetricsStores(dclient, ns, apiVersion, kind, metricFamilies)
+
 		allStores = append(allStores, gvkStores)
 	}
 	// Start serving metrics.
@@ -111,3 +122,22 @@ func GetNamespacesForMetrics(operatorNs string) ([]string, error) {
 	}
 	return ns, nil
 }
+
+func isNamespaced(gvk schema.GroupVersionKind, cfg *rest.Config) (bool, error) {
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(cfg)
+	if err != nil {
+		log.Error(err, "Unable to get discovery client")
+		return false, err
+	}
+	resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
+	if err != nil {
+		log.Error(err, "Unable to get resource list for", "apiversion", gvk.GroupVersion().String())
+		return false, err
+	}
+	for _, apiResource := range resourceList.APIResources {
+		if apiResource.Kind == gvk.Kind {
+			return apiResource.Namespaced, nil
+		}
+	}
+	return false, errors.New("unable to find type: " + gvk.String() + " in server")
+}

pkg/kube-metrics/store.go

@@ -27,10 +27,10 @@ import (
 	metricsstore "k8s.io/kube-state-metrics/pkg/metrics_store"
 )
 
-// NewMetricsStores returns collections of metrics in the namespaces provided, per the api/kind resource.
+// NewNamespacedMetricsStores returns collections of metrics in the namespaces provided, per the api/kind resource.
 // The metrics are registered in the custom metrics.FamilyGenerator that needs to be defined.
-func NewMetricsStores(dclient dynamic.NamespaceableResourceInterface, namespaces []string, api string, kind string,
-	metricFamily []metric.FamilyGenerator) []*metricsstore.MetricsStore {
+func NewNamespacedMetricsStores(dclient dynamic.NamespaceableResourceInterface, namespaces []string,
+	api string, kind string, metricFamily []metric.FamilyGenerator) []*metricsstore.MetricsStore {
 	namespaces = deduplicateNamespaces(namespaces)
 	var stores []*metricsstore.MetricsStore
 	// Generate collector per namespace.
@@ -44,6 +44,30 @@ func NewMetricsStores(dclient dynamic.NamespaceableResourceInterface, namespaces
 	return stores
 }
 
+// NewClusterScopedMetricsStores returns collections of metrics per the api/kind resource.
+// The metrics are registered in the custom metrics.FamilyGenerator that needs to be defined.
+func NewClusterScopedMetricsStores(dclient dynamic.NamespaceableResourceInterface, api string, kind string,
+	metricFamily []metric.FamilyGenerator) []*metricsstore.MetricsStore {
+	var stores []*metricsstore.MetricsStore
+	// Generate collector per cluster scoped resources.
+	composedMetricGenFuncs := metric.ComposeMetricGenFuncs(metricFamily)
+	headers := metric.ExtractMetricFamilyHeaders(metricFamily)
+	store := metricsstore.NewMetricsStore(headers, composedMetricGenFuncs)
+	reflectorPerClusterScoped(context.TODO(), dclient, &unstructured.Unstructured{}, store)
+	stores = append(stores, store)
+
+	return stores
+}
+
+// NewMetricsStores returns collections of metrics in the namespaces provided, per the api/kind resource.
+// The metrics are registered in the custom metrics.FamilyGenerator that needs to be defined.
+//
+// Deprecated: Use NewNamespacedMetricsStores instead.
+func NewMetricsStores(dclient dynamic.NamespaceableResourceInterface, namespaces []string,
+	api string, kind string, metricFamily []metric.FamilyGenerator) []*metricsstore.MetricsStore {
+	return NewNamespacedMetricsStores(dclient, namespaces, api, kind, metricFamily)
+}
+
 func deduplicateNamespaces(ns []string) (list []string) {
 	keys := make(map[string]struct{})
 	for _, entry := range ns {
@@ -55,19 +79,31 @@ func deduplicateNamespaces(ns []string) (list []string) {
 	return list
 }
 
+func reflectorPerClusterScoped(
+	ctx context.Context,
+	dynamicInterface dynamic.NamespaceableResourceInterface,
+	expectedType interface{},
+	store cache.Store,
+) {
+	lw := clusterScopeListWatchFunc(dynamicInterface)
+	reflector := cache.NewReflector(&lw, expectedType, store, 0)
+	go reflector.Run(ctx.Done())
+}
+
 func reflectorPerNamespace(
 	ctx context.Context,
 	dynamicInterface dynamic.NamespaceableResourceInterface,
 	expectedType interface{},
 	store cache.Store,
 	ns string,
 ) {
-	lw := listWatchFunc(dynamicInterface, ns)
+	lw := namespacedListWatchFunc(dynamicInterface, ns)
 	reflector := cache.NewReflector(&lw, expectedType, store, 0)
 	go reflector.Run(ctx.Done())
 }
 
-func listWatchFunc(dynamicInterface dynamic.NamespaceableResourceInterface, namespace string) cache.ListWatch {
+func namespacedListWatchFunc(dynamicInterface dynamic.NamespaceableResourceInterface,
+	namespace string) cache.ListWatch {
 	return cache.ListWatch{
 		ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
 			return dynamicInterface.Namespace(namespace).List(opts)
@@ -77,3 +113,14 @@ func listWatchFunc(dynamicInterface dynamic.NamespaceableResourceInterface, name
 		},
 	}
 }
+
+func clusterScopeListWatchFunc(dynamicInterface dynamic.NamespaceableResourceInterface) cache.ListWatch {
+	return cache.ListWatch{
+		ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+			return dynamicInterface.List(opts)
+		},
+		WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+			return dynamicInterface.Watch(opts)
+		},
+	}
+}