Add pprof port to bpfagent controller

Signed-off-by: Mohamed Mahmoud <[email protected]>

README.md

@@ -182,4 +182,53 @@ object to find references to the bpfMap pinpoints (`spec.maps`) to configure the
 ## Developer
 
 For more architecture details about `bpfman-operator`, refer to
-[Developing the bpfman-operator](https://bpfman.io/v0.4.1/developer-guide/develop-operator)
+[Developing the bpfman-operator](https://bpfman.io/v0.4.1/developer-guide/develop-operator)
+
+## Bpfman-agent profiling
+
+bpfman-agent process use port `6060` for Golang profiling to be able to get the different profiles
+
+1- Set port-forward rule in a different terminal 
+```shell
+oc get pods -n bpfman
+NAME                               READY   STATUS    RESTARTS   AGE
+bpfman-daemon-76v57                3/3     Running   0          14m
+bpfman-operator-7f67bc7c57-ww52z   2/2     Running   0          14m
+
+oc -n bpfman port-forward bpfman-daemon-76v57 6060
+```
+
+2- Download the required profiles:
+
+`curl -o <profile> http://localhost:6060/debug/pprof/<profile>`
+
+Where <profile> can be:
+
+| profile      | description                                                                   |
+|--------------|-------------------------------------------------------------------------------|
+| allocs       | A sampling of all memory allocations                                          |
+| block        | Stack traces that led to blocking on synchronization primitives               |
+| cmdline      | The command line invocation of the current program                            |
+| goroutine    | Stack traces of all current goroutines                                        |
+| heap         | A sampling of memory allocations of live objects.                             |
+|              | You can specify the gc GET parameter to run GC before taking the heap sample. |
+| mutex        | Stack traces of holders of contended mutexes                                  |
+| profile      | CPU profile.                                                                  |
+|              | You can specify the duration in the seconds GET parameter.                    |
+| threadcreate | Stack traces that led to the creation of new OS threads                       |
+| trace        | A trace of execution of the current program.                                  |
+|              | You can specify the duration in the seconds GET paramete.                     |
+| ------       | -----------------------------------------------                               |
+
+Example:
+
+```shell
+curl "http://localhost:6060/debug/pprof/trace?seconds=20" -o trace
+curl "http://localhost:6060/debug/pprof/profile?duration=20" -o cpu
+curl "http://localhost:6060/debug/pprof/heap?gc" -o heap
+curl "http://localhost:6060/debug/pprof/allocs" -o allocs
+curl "http://localhost:6060/debug/pprof/goroutine" -o goroutine
+```
+
+3- Use go tool pprof to dig into the profiles (go tool trace for the trace profile) or use 
+  web interface for example `go tool pprof -http=:8080 cpu`

cmd/bpfman-agent/main.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	//+kubebuilder:scaffold:imports
+	_ "net/http/pprof"
 )
 
 var (
@@ -62,10 +63,12 @@ func main() {
 	var probeAddr string
 	var opts zap.Options
 	var enableHTTP2 bool
+	var pprofAddr string
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8174", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8175", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", enableHTTP2, "If HTTP/2 should be enabled for the metrics and webhook servers.")
+	flag.StringVar(&pprofAddr, "profiling-bind-address", "", "The address the profiling endpoint binds to, such as ':6060'. Leave unset to disable profiling.")
 	flag.Parse()
 
 	// Get the Log level for bpfman deployment where this pod is running
@@ -109,6 +112,7 @@ func main() {
 			Port:    9443,
 			TLSOpts: []func(*tls.Config){disableHTTP2},
 		}),
+		PprofBindAddress:       pprofAddr,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         false,
 		// Specify that Secrets's should not be cached.

config/bpfman-deployment/daemonset.yaml

@@ -109,6 +109,7 @@ spec:
           args:
             - --health-probe-bind-address=:8175
             - --metrics-bind-address=127.0.0.1:8174
+            - --profiling-bind-address=:6060
           image: quay.io/bpfman/bpfman-agent:latest
           securityContext:
             privileged: true