Remove some metricsets from default metricbeat cfg

This is to avoid needing more permissions than the pod has through autodiscovery
Improve checks in e2e tests

pkg/controller/beat/metricbeat/config.go

@@ -14,7 +14,7 @@ var (
   autodiscover:
     providers:
     - hints:
-        default_config: null
+        default_config: {}
         enabled: "true"
       node: ${NODE_NAME}
       type: kubernetes
@@ -37,36 +37,7 @@ var (
   - module: system
     period: 1m
     metricsets:
-    - filesystem
     - fsstat
-    processors:
-    - drop_event:
-        when:
-          regexp:
-            system:
-              filesystem:
-                mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib)($|/)
-  - module: kubernetes
-    period: 10s
-    host: ${NODE_NAME}
-    hosts:
-    - https://${HOSTNAME}:10250
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    ssl:
-      verification_mode: none
-    metricsets:
-    - node
-    - system
-    - pod
-    - container
-    - volume
-  - module: kubernetes
-    period: 10s
-    host: ${NODE_NAME}
-    hosts:
-    - https://${HOSTNAME}:10249
-    metricsets:
-    - proxy
 processors:
 - add_cloud_metadata: {}
 `))

test/e2e/beat/config_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"testing"
 
+	v1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
 	"github.com/stretchr/testify/require"
 
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
@@ -29,7 +30,10 @@ func TestFilebeatDefaultConfig(t *testing.T) {
 
 	fbBuilder := beat.NewBuilder(name, filebeat.Type).
 		WithElasticsearchRef(esBuilder.Ref()).
-		WithESValidations(beat.HasEventFromPod(testPodBuilder.Pod.Name))
+		WithESValidations(
+			beat.HasEventFromBeat(filebeat.Type),
+			beat.HasEventFromPod(testPodBuilder.Pod.Name),
+			beat.HasMessageContaining(testPodBuilder.Logged))
 
 	test.Sequence(nil, test.EmptySteps, esBuilder, fbBuilder, testPodBuilder).RunSequential(t)
 }
@@ -44,7 +48,16 @@ func TestMetricbeatDefaultConfig(t *testing.T) {
 
 	mbBuilder := beat.NewBuilder(name, metricbeat.Type).
 		WithElasticsearchRef(esBuilder.Ref()).
-		WithESValidations(beat.HasEventFromBeat(metricbeat.Type))
+		WithESValidations(
+			beat.HasEventFromBeat(metricbeat.Type),
+			beat.HasEvent("event.dataset:system.cpu"),
+			beat.HasEvent("event.dataset:system.load"),
+			beat.HasEvent("event.dataset:system.memory"),
+			beat.HasEvent("event.dataset:system.network"),
+			beat.HasEvent("event.dataset:system.process"),
+			beat.HasEvent("event.dataset:system.process.summary"),
+			beat.HasEvent("event.dataset:system.fsstat"),
+		)
 
 	test.Sequence(nil, test.EmptySteps, esBuilder, mbBuilder, testPodBuilder).RunSequential(t)
 }
@@ -58,14 +71,16 @@ func TestHeartbeatConfig(t *testing.T) {
 	hbBuilder := beat.NewBuilder(name, "heartbeat").
 		WithElasticsearchRef(esBuilder.Ref()).
 		WithImage("docker.elastic.co/beats/heartbeat:7.7.0").
-		WithESValidations(beat.HasEventFromBeat("heartbeat"))
+		WithESValidations(
+			beat.HasEventFromBeat("heartbeat"),
+			beat.HasEvent("monitor.status:up"))
 
 	yaml := fmt.Sprintf(`
 heartbeat.monitors:
 - type: tcp
   schedule: '@every 5s'
   hosts: ["%s.%s.svc:9200"]
-`, esBuilder.Elasticsearch.Name, esBuilder.Elasticsearch.Namespace)
+`, v1.HTTPService(esBuilder.Elasticsearch.Name), esBuilder.Elasticsearch.Namespace)
 	hbBuilder = applyConfigYaml(t, hbBuilder, yaml)
 
 	test.Sequence(nil, test.EmptySteps, esBuilder, hbBuilder).RunSequential(t)

test/e2e/test/beat/checks.go

@@ -16,16 +16,24 @@ import (
 )
 
 func HasEventFromBeat(name beatcommon.Type) ValidationFunc {
-	return HasEvent(fmt.Sprintf("/*beat*/_search?q=agent.type:%s", name))
+	return HasEvent(fmt.Sprintf("agent.type:%s", name))
 }
 
 func HasEventFromPod(name string) ValidationFunc {
-	return HasEvent(fmt.Sprintf("/*beat*/_search?q=kubernetes.pod.name:%s", name))
+	return HasEvent(fmt.Sprintf("kubernetes.pod.name:%s", name))
+}
+
+func HasMessageContaining(message string) ValidationFunc {
+	return HasEvent(fmt.Sprintf("message:%s", message))
 }
 
 func HasEvent(query string) ValidationFunc {
+	return hasEvent(fmt.Sprintf("/*beat*/_search?q=%s", query))
+}
+
+func hasEvent(url string) ValidationFunc {
 	return func(esClient client.Client) error {
-		req, err := http.NewRequest(http.MethodGet, query, nil)
+		req, err := http.NewRequest(http.MethodGet, url, nil)
 		if err != nil {
 			return err
 		}
@@ -45,7 +53,7 @@ func HasEvent(query string) ValidationFunc {
 			return err
 		}
 		if len(results.Hits.Hits) == 0 {
-			return fmt.Errorf("hit count should be more than 0 for %s", query)
+			return fmt.Errorf("hit count should be more than 0 for %s", url)
 		}
 
 		return nil

test/e2e/test/beat/pod_builder.go

@@ -6,6 +6,7 @@ package beat
 
 import (
 	"fmt"
+	"math/rand"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -24,7 +25,8 @@ import (
 
 // Builder to create Beats
 type PodBuilder struct {
-	Pod corev1.Pod
+	Pod    corev1.Pod
+	Logged string
 }
 
 func NewPodBuilder(name string) PodBuilder {
@@ -34,6 +36,9 @@ func NewPodBuilder(name string) PodBuilder {
 		Labels:    map[string]string{run.TestNameLabel: name},
 	}
 
+	// inject random string into the logs to allow validating whether they end up in ES easily
+	loggedString := fmt.Sprintf("_%d_", rand.Int())
+
 	return PodBuilder{
 		Pod: corev1.Pod{
 			ObjectMeta: meta,
@@ -45,13 +50,14 @@ func NewPodBuilder(name string) PodBuilder {
 						Command: []string{
 							"bash",
 							"-c",
-							"while [ true ]; do echo \"$(date)\"; sleep 5; done",
+							fmt.Sprintf("while [ true ]; do echo \"$(date) - %s\"; sleep 5; done", loggedString),
 						},
 					},
 				},
 				SecurityContext: test.DefaultSecurityContext(),
 			},
 		},
+		Logged: loggedString,
 	}
 }