NETOBSERV-1911 CLI metrics (#106)

* run in background * allow custom namespace * add metrics capture & cleanup readme * update e2e * Update README.md Co-authored-by: Joel Takvorian <[email protected]> * Update README.md Co-authored-by: Joel Takvorian <[email protected]> * NETOBSERV-2030 missing toleration on daemonset for metrics * skip dependencies check on help and update doc --------- Co-authored-by: Joel Takvorian <[email protected]>
netobserv · Jan 8, 2025 · b2b688c · b2b688c
1 parent 42be610
commit b2b688c
Show file tree

Hide file tree

Showing 12 changed files with 921 additions and 142 deletions.
diff --git a/README.md b/README.md
@@ -1,10 +1,12 @@
 # Network Observability CLI
 
-network-observability-cli is a lightweight Flow and Packet visualization tool.
+network-observability-cli is a lightweight Flow, Packet and Metrics visualization tool.
 It deploys [NetObserv eBPF agent](https://github.com/netobserv/netobserv-ebpf-agent) on your k8s cluster to collect flows or packets from nodes network interfaces
 and streams data to a local collector for analysis and visualization.
 Output files are generated under `output/flow` and `output/pcap` directories per host name
 
+On Openshift environments, you can also capture metrics in your monitoring stack and display a fully configured dashboard.
+
 ## Prerequisites
 
 To run this CLI, you will need:
@@ -44,7 +46,7 @@ USER=netobserv VERSION=dev make images
 Run the following command to start capturing flows, replacing `USER`, `VERSION` and `COMMAND_ARGS` accordingly:
 
 ```bash
-USER=netobserv VERSION=dev COMMAND_ARGS=br-ex make flows
+USER=netobserv VERSION=dev COMMAND_ARGS=--interfaces=br-ex make flows
 ```
 
 ![flows](./img/flow-table.png)
@@ -107,24 +109,34 @@ or `dbeaver`:
 Run the following command to start capturing packets, replacing `USER`, `VERSION` and `COMMAND_ARGS` accordingly:
 
 ```bash
-USER=netobserv VERSION=dev COMMAND_ARGS=tcp,80 make packets
+USER=netobserv VERSION=dev COMMAND_ARGS="--protocol=TCP --port=80" make packets
 ```
 
-![packets](./img/packet-table.png)
-
-It will display a table view with latest packets collected and write data under output/pcap directory.
+Similarly to flow capture, it will display a table view with latest flows. However, it will collect packets and write data under output/pcap directory.
 To stop capturing press Ctrl-C.
 
-This will write pcap into a single file located in `./output/pcap/<CAPTURE_DATE_TIME>.pcap` that can be opened with Wireshark for example:
+This will write [pcapng](https://wiki.wireshark.org/Development/PcapNg) into a single file located in `./output/pcap/<CAPTURE_DATE_TIME>.pcapng` that can be opened with Wireshark for example:
 
 ![wireshark](./img/wireshark.png)
 
+### Metrics dashboard (OCP only)
+
+Run the following command to start capturing metrics, replacing `USER`, `VERSION` and `COMMAND_ARGS` accordingly:
+```bash
+USER=netobserv VERSION=dev COMMAND_ARGS='--enable_pktdrop="true" --enable_dns="true" --enable_rtt="true"' make metrics
+```
+
+![metrics](./img/metrics-dashboard.png)
+
+It will generate a monitoring dashboard called "NetObserv / On Demand" in your Openshift cluster.
+The url to access it is automatically generated from the CLI. Simply click on the link to open the page.
+
 ### Cleanup
 
-The `cleanup` function will automatically remove the eBPF programs when the CLI exits. However you may need to run it manually if an error occurs.
+The `cleanup` function will automatically remove the eBPF programs when the CLI exits. However you may need to run it manually if running in background or an error occurs.
 
 ```bash
-./commands/netobserv-cleanup
+USER=netobserv VERSION=dev make cleanup
 ```
 
 ## Extending OpenShift or Kubernetes CLI with plugins

diff --git a/commands/netobserv b/commands/netobserv
@@ -41,133 +41,179 @@ command=""
 logLevel="info"
 
 # max time (default: 5min)
-maxTime="5m" 
+maxTime="5m"
 
 # max bytes (default: 50MB)
 maxBytes=50000000
 
 function flows() {
   case "$2" in
-    "help")
-      flows_usage
-      exit 0 ;;
-    *)
-      shift # remove first argument
-      options="$*"
-      # run flows command
-      command="flows" ;;
+  "help")
+    flows_usage
+    exit 0
+    ;;
+  *)
+    shift # remove first argument
+    options="$*"
+    # run flows command
+    command="flows"
+    ;;
   esac
 }
 
 function packets() {
   case "$2" in
-    "help")
-      packets_usage
-      exit 0 ;;
-    *)
-      shift # remove first argument
-      options="$*"
-      # run packets command
-      command="packets" ;;
+  "help")
+    packets_usage
+    exit 0
+    ;;
+  *)
+    shift # remove first argument
+    options="$*"
+    # run packets command
+    command="packets"
+    ;;
   esac
 }
 
-required_yq_version="v0.0.0"
-supported_archs=""
-check_dependencies "$required_yq_version" "$supported_archs"
+function metrics() {
+  case "$2" in
+  "help")
+    metrics_usage
+    exit 0
+    ;;
+  *)
+    shift # remove first argument
+    options="$*"
+    # run metrics command
+    command="metrics"
+    ;;
+  esac
+}
+
+if [[ ! "$*" =~ ^(.*)help|version(.*) ]]; then
+  required_yq_version="v0.0.0"
+  supported_archs=""
+  check_dependencies "$required_yq_version" "$supported_archs"
+fi
 
 case "$1" in
 "help")
-    # display Help
-    echo
-    echo "Netobserv allows you to capture flow and packets from your cluster."
-    echo "Find more information at: https://github.com/netobserv/network-observability-cli/"
-    echo
-    echo "Syntax: netobserv [flows|packets|cleanup] [options]"
-    echo
-    echo "commands:"
-    echo "  flows      Capture flows information in JSON format."
-    echo "        Options:"
-    flows_usage
-    echo "  packets    Capture packets information in pcap format."
-    echo "        Options:"
-    packets_usage
-    echo "  follow     Follow collector logs when running in background."
-    echo "  stop       Stop collection by removing agent daemonset."
-    echo "  copy       Copy generated files locally."
-    echo "  cleanup    Remove netobserv components."
-    echo "  version    Print software version."
-    echo
-    exit 0 ;;
+  # display Help
+  echo
+  echo "Netobserv allows you to capture flow, packets and metrics from your cluster."
+  echo "Find more information at: https://github.com/netobserv/network-observability-cli/"
+  echo
+  echo "Syntax: netobserv [flows|packets|metrics|follow|stop|copy|cleanup|version] [options]"
+  echo
+  echo "commands:"
+  echo "  flows      Capture flows information in JSON format using collector pod."
+  echo "        Options:"
+  flows_usage
+  echo "  packets    Capture packets information in pcap format using collector pod."
+  echo "        Options:"
+  packets_usage
+  echo "  metrics    Capture metrics information in Prometheus using a ServiceMonitor (OCP cluster only)."
+  echo "        Options:"
+  metrics_usage
+  echo "  follow     Follow collector logs when running in background."
+  echo "  stop       Stop collection by removing agent daemonset."
+  echo "  copy       Copy collector generated files locally."
+  echo "  cleanup    Remove netobserv components and configurations."
+  echo "  version    Print software version."
+  echo
+  exit 0
+  ;;
 "version")
-    # display version
-    echo "Netobserv CLI version $version"
-    exit 0 ;;
+  # display version
+  echo "Netobserv CLI version $version"
+  exit 0
+  ;;
 "flows")
-    flows $* ;;
+  flows $*
+  ;;
 "packets")
-    packets $* ;;
+  packets $*
+  ;;
+"metrics")
+  metrics $*
+  ;;
 "follow")
-    # run follow command
-    follow
-    exit 0 ;;
+  # run follow command
+  follow
+  exit 0
+  ;;
 "stop")
-    # run deleteDaemonset command
-    deleteDaemonset
-    exit 0 ;;
+  # run deleteDaemonset command
+  deleteDaemonset
+  exit 0
+  ;;
 "copy")
-    # run copy output command
-    copyOutput
-    exit 0 ;;
+  # run copy output command
+  copyOutput
+  exit 0
+  ;;
 "cleanup")
-    # run cleanup command
-    cleanup
-    exit 0 ;;
+  # run cleanup command
+  cleanup
+  exit 0
+  ;;
 *)
-    echo "Unknown command $1. Use 'netobserv help' to display options"
-    exit 1
+  echo "Unknown command $1. Use 'netobserv help' to display options"
+  exit 1
+  ;;
 esac
 
 trap cleanup EXIT
 
 setup $command $options
 
-# convert options to string
-optionStr="${options//--/}"
-optionStr="${optionStr// /|}"
-
-# prepare commands & args
-runCommand="sleep infinity"
-execCommand="/network-observability-cli get-$command ${optionStr:+"--options" "${optionStr}"} --loglevel $logLevel --maxtime $maxTime --maxbytes $maxBytes"
-if [[ "$runBackground" == "true" ]]; then
-  runCommand="$execCommand & $runCommand"
-  execCommand=""
-fi
-
-echo "Running network-observability-cli get-$command... "
-${K8S_CLI_BIN} run \
-  -n $namespace \
-  collector \
-  --image=$img\
-  --image-pull-policy='Always' \
-  --overrides='{ "spec": { "serviceAccount": "netobserv-cli" }  }' \
-  --restart='Never' \
-  --command -- $runCommand
-
-${K8S_CLI_BIN} wait \
-  -n $namespace \
-  --for=condition=Ready pod/collector || exit 1
-
-captureStarted=true
-
-if [ -n "${execCommand}" ]; then
-  ${K8S_CLI_BIN} exec -i --tty \
+if [[ "$command" == "flows" || "$command" == "packets" ]]; then
+  # convert options to string
+  optionStr="${options//--/}"
+  optionStr="${optionStr// /|}"
+
+  # prepare commands & args
+  runCommand="sleep infinity"
+  execCommand="/network-observability-cli get-$command ${optionStr:+"--options" "${optionStr}"} --loglevel $logLevel --maxtime $maxTime --maxbytes $maxBytes"
+  if [[ "$runBackground" == "true" ]]; then
+    runCommand="$execCommand & $runCommand"
+    execCommand=""
+  fi
+
+  echo "Running network-observability-cli get-$command... "
+  ${K8S_CLI_BIN} run \
     -n $namespace \
     collector \
-    -- $execCommand
-else 
-  echo "Background capture started. Use:"
-  echo " - '${K8S_CLI_BIN} netobserv follow' to see the capture progress"
-  echo " - '${K8S_CLI_BIN} netobserv copy' to copy the generated files locally"
-  echo " - '${K8S_CLI_BIN} netobserv cleanup' to remove the netobserv components"
+    --image=$img --image-pull-policy='Always' \
+    --overrides='{ "spec": { "serviceAccount": "netobserv-cli" }  }' \
+    --restart='Never' \
+    --command -- $runCommand
+
+  ${K8S_CLI_BIN} wait \
+    -n $namespace \
+    --for=condition=Ready pod/collector || exit 1
+
+  captureStarted=true
+
+  if [ -n "${execCommand}" ]; then
+    ${K8S_CLI_BIN} exec -i --tty \
+      -n $namespace \
+      collector \
+      -- $execCommand
+  else
+    echo "Background capture started. Use:"
+    echo " - '${K8S_CLI_BIN} netobserv follow' to see the capture progress"
+    echo " - '${K8S_CLI_BIN} netobserv copy' to copy the generated files locally"
+    echo " - '${K8S_CLI_BIN} netobserv cleanup' to remove the netobserv components"
+  fi
+elif [ "$command" = "metrics" ]; then
+  runBackground="true"
+  echo "Metrics capture started."
+  consoleURL="$(oc whoami --show-console)"
+  echo "Open ${consoleURL}/monitoring/dashboards/netobserv-cli to see generated metrics."
+  echo "Use 'oc netobserv stop' to stop the collection and 'oc netobserv cleanup' to remove everything."
+else
+  echo "Unexpected exception occured on $command"
+  exit 1
 fi