NETOBSERV-754: add pprof, improve memory footprint

.mk/development.mk

@@ -188,3 +188,19 @@ set-release-kind-downstream:
 	@echo -e "\n==> Redeploying..."
 	kubectl rollout status -n $(NAMESPACE) --timeout=60s deployment netobserv-controller-manager
 	kubectl wait -n $(NAMESPACE) --timeout=60s --for condition=Available=True deployment netobserv-controller-manager
+
+.PHONY: pprof
+pprof:
+	@echo -e "\n==> Enabling pprof... Check https://github.com/netobserv/network-observability-operator/blob/main/DEVELOPMENT.md#profiling for help."
+	kubectl -n $(NAMESPACE) set env deployment netobserv-controller-manager -c "manager" PROFILING_BIND_ADDRESS=:6060
+	@echo -e "\n==> Redeploying..."
+	kubectl rollout status -n $(NAMESPACE) --timeout=60s deployment netobserv-controller-manager
+	kubectl wait -n $(NAMESPACE) --timeout=60s --for condition=Available=True deployment netobserv-controller-manager
+	sleep 2
+	$(MAKE) pprof-pf
+
+.PHONY: pprof-pf
+pprof-pf:
+	@echo -e "\n==> Port-forwarding..."
+	oc get pods
+	kubectl port-forward -n $(NAMESPACE) $(shell kubectl get pod -l app=netobserv-operator -o jsonpath="{.items[0].metadata.name}") 6060

DEVELOPMENT.md

@@ -384,3 +384,18 @@ Remove the tag after you tested:
 git tag -d "0.0.0-rc0"
 git push --delete upstream 0.0.0-rc0
 ```
+
+## Profiling
+
+You can use `pprof` for profiling. Run `pprof` make target to start listening and port-forward on 6060: 
+
+```bash
+make pprof
+```
+
+In another terminal, run for instance:
+
+```bash
+curl "http://localhost:6060/debug/pprof/heap?gc" -o /tmp/heap
+go tool pprof -http localhost:3435 /tmp/heap
+```

bundle/manifests/netobserv-operator.clusterserviceversion.yaml

@@ -687,6 +687,7 @@ spec:
                 - --flowlogs-pipeline-image=$(RELATED_IMAGE_FLOWLOGS_PIPELINE)
                 - --console-plugin-image=$(RELATED_IMAGE_CONSOLE_PLUGIN)
                 - --downstream-deployment=$(DOWNSTREAM_DEPLOYMENT)
+                - --profiling-bind-address=$(PROFILING_BIND_ADDRESS)
                 command:
                 - /manager
                 env:
@@ -698,6 +699,7 @@ spec:
                   value: quay.io/netobserv/network-observability-console-plugin:v0.1.11
                 - name: DOWNSTREAM_DEPLOYMENT
                   value: "false"
+                - name: PROFILING_BIND_ADDRESS
                 image: quay.io/netobserv/network-observability-operator:1.0.4
                 imagePullPolicy: Always
                 livenessProbe:

config/manager/manager.yaml

@@ -28,6 +28,7 @@ spec:
         - --flowlogs-pipeline-image=$(RELATED_IMAGE_FLOWLOGS_PIPELINE)
         - --console-plugin-image=$(RELATED_IMAGE_CONSOLE_PLUGIN)
         - --downstream-deployment=$(DOWNSTREAM_DEPLOYMENT)
+        - --profiling-bind-address=$(PROFILING_BIND_ADDRESS)
         env:
           - name: RELATED_IMAGE_EBPF_AGENT
             value: quay.io/netobserv/netobserv-ebpf-agent:v0.3.2
@@ -37,6 +38,8 @@ spec:
             value: quay.io/netobserv/network-observability-console-plugin:v0.1.11
           - name: DOWNSTREAM_DEPLOYMENT
             value: "false"
+          - name: PROFILING_BIND_ADDRESS
+            value: ""
         image: controller:latest
         name: manager
         imagePullPolicy: Always

main.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"go.uber.org/zap/zapcore"
@@ -81,12 +83,14 @@ func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
+	var pprofAddr string
 	var versionFlag bool
 
 	config := operator.Config{}
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	flag.StringVar(&pprofAddr, "profiling-bind-address", "", "The address the profiling endpoint binds to, such as ':6060'. Leave unset to disable profiling.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -116,6 +120,14 @@ func main() {
 		os.Exit(1)
 	}
 
+	if pprofAddr != "" {
+		setupLog.WithValues("address", pprofAddr).Info("starting profiling server")
+		go func() {
+			err := http.ListenAndServe(pprofAddr, nil)
+			setupLog.Error(err, "stopped profiling server")
+		}()
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme: scheme,
 		Metrics: server.Options{