Auto-include node_stats metricset when xpack.enabled: true is set (el…

…astic#20613)

* Auto-include node_stats metricset when xpack.enabled: true is set

* Fixing some tests

* Try to fix python system test

* Fixing len check

* Fixing monitoring index type for node_stats metricset

* Account for node_stats docs being indexed into metricbeat-*

* Debugging

* More debugging

* Debugging

* Updating integration test

* Fixing test code

metricbeat/module/elasticsearch/elasticsearch.go

@@ -54,6 +54,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
 		"index_recovery",
 		"index_summary",
 		"ml_job",
+		"node_stats",
 		"shard",
 	}
 	return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName))

metricbeat/module/elasticsearch/elasticsearch_integration_test.go

@@ -133,7 +133,7 @@ func TestXPackEnabled(t *testing.T) {
 		"index_recovery": []string{"index_recovery"},
 		"index_summary":  []string{"indices_stats"},
 		"ml_job":         []string{"job_stats"},
-		"node_stats":     []string{"node_stats"},
+		"node_stats":     []string{}, // no longer indexed into .monitoring-es-*
 	}
 
 	config := getXPackConfig(host)
@@ -177,8 +177,19 @@ func TestXPackEnabled(t *testing.T) {
 			}
 
 			types := metricSetToTypesMap[metricSet.Name()]
-			require.Len(t, events, len(types))
+			numTypes := len(types)
 
+			// If no types are returned for this metricset, then it's because this metricset
+			// is no longer indexing events into .monitoring-* indices. But instead it is indexing
+			// events into the default Metricbeat indices.
+			if numTypes == 0 {
+				for _, event := range events {
+					require.Empty(t, event.Index)
+				}
+				return
+			}
+
+			require.Len(t, events, numTypes)
 			for i, event := range events {
 				require.Equal(t, types[i], event.RootFields["type"])
 				require.Regexp(t, `^.monitoring-es-\d-mb`, event.Index)

metricbeat/module/elasticsearch/elasticsearch_test.go

@@ -45,12 +45,12 @@ func TestXPackEnabledMetricsets(t *testing.T) {
 	}
 
 	metricSets := mbtest.NewReportingMetricSetV2Errors(t, config)
-	require.Len(t, metricSets, 8)
+	require.Len(t, metricSets, 9)
 	for _, ms := range metricSets {
 		name := ms.Name()
 		switch name {
 		case "ccr", "enrich", "cluster_stats", "index", "index_recovery",
-			"index_summary", "ml_job", "shard":
+			"index_summary", "ml_job", "node_stats", "shard":
 		default:
 			t.Errorf("unexpected metricset name = %v", name)
 		}

metricbeat/module/elasticsearch/test_elasticsearch.py

@@ -127,6 +127,7 @@ def test_xpack_cluster_stats(self):
                 "index_recovery",
                 "index_summary",
                 "ml_job",
+                "node_stats",
                 "shard"
             ],
             "hosts": self.get_hosts(),
@@ -144,6 +145,9 @@ def test_xpack_cluster_stats(self):
 
         docs = self.read_output_json()
         for doc in docs:
+            if "type" not in doc:
+                continue
+
             t = doc["type"]
             if t != "cluster_stats":
                 continue