Support client monitoring by a remote service (e.g. beaconcha.in) (#5037

) * Initial client monitoring implementation * Add monitoring to beacon node * Add monitoring to validator * Ensure that monitoring endpoint is a valid URL * Improve validation of monitoring endpoint * Improve error handling and timeout of remote server request * Wait for pending request before sending next one * Update request error handling * Update monitoring endpoint parsing * Export monitoring package * Fix process cpu seconds total metric Use "process_cpu_user_seconds_total" instead of "process_cpu_seconds_total" as it properly reports the CPU usage of the process and more importantly also triggers the collect method of the metric which ensures that always the latest value is shown. * Add option to collect system stats * Define system stats * Improve logs when monitoring service is started * Use the term "remote service" instead of "remote server" * Move Client type to service * Add monitoring args to beacon node test * Update description of monitoring cli args * Update monitoring service * Add metrics for collecting and sending data * Add monitoring panels to VM + host dashboard * Update send data metric buckets * Print out machine when starting monitoring service * Refactoring * Add unit tests * Add metric values for sync_eth1_connected and sync_eth1_fallback_configured * Use setTimeout instead of sleep for initial delay * Use milliseconds instead of seconds for time values * Add description to client stats properties * Remove sinon spies after tests are finished * Document client monitoring usage * Add enum to check status of monitoring service * Reduce info log when monitoring service is started Only print out information which is configurable by non-hidden CLI options and properly documented.
ChainSafe · Feb 16, 2023 · e6eb3bd · e6eb3bd
1 parent 8beabbb
commit e6eb3bd
Show file tree

Hide file tree

Showing 24 changed files with 1,894 additions and 4 deletions.
diff --git a/dashboards/lodestar_vm_host.json b/dashboards/lodestar_vm_host.json
@@ -22,7 +22,7 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": 35,
-  "iteration": 1672991883959,
+  "iteration": 1674481074076,
   "links": [
     {
       "asDropdown": true,
@@ -2624,6 +2624,200 @@
         "x": 0,
         "y": 44
       },
+      "id": 527,
+      "panels": [
+        {
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 30,
+                "gradientMode": "opacity",
+                "hideFrom": {
+                  "graph": false,
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": true,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 45
+          },
+          "id": 529,
+          "options": {
+            "graph": {},
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "7.4.5",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${DS_PROMETHEUS}"
+              },
+              "exemplar": false,
+              "expr": "rate(lodestar_monitoring_collect_data_seconds_sum[$rate_interval])\r\n/\r\nrate(lodestar_monitoring_collect_data_seconds_count[$rate_interval])",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Collect data duration",
+          "type": "timeseries"
+        },
+        {
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 30,
+                "gradientMode": "opacity",
+                "hideFrom": {
+                  "graph": false,
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": true,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 45
+          },
+          "id": 531,
+          "options": {
+            "graph": {},
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "7.4.5",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${DS_PROMETHEUS}"
+              },
+              "exemplar": false,
+              "expr": "rate(lodestar_monitoring_send_data_seconds_sum[$rate_interval])\r\n/\r\nrate(lodestar_monitoring_send_data_seconds_count[$rate_interval])",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} - {{status}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Send data duration",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Monitoring",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 45
+      },
       "id": 86,
       "panels": [
         {
@@ -3158,7 +3352,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 45
+        "y": 46
       },
       "id": 164,
       "panels": [

diff --git a/docs/usage/client-monitoring.md b/docs/usage/client-monitoring.md
@@ -0,0 +1,49 @@
+# Client monitoring
+
+Lodestar has the ability to send client stats to a remote service for collection.
+At the moment, the main service offering remote monitoring is [beaconcha.in](https://beaconcha.in/).
+
+Instructions for setting up client monitoring with *beaconcha.in* can be found in their docs about
+[Mobile App <> Node Monitoring](https://kb.beaconcha.in/beaconcha.in-explorer/mobile-app-less-than-greater-than-beacon-node)
+and in your [account settings](https://beaconcha.in/user/settings#app).
+
+## Configuration
+
+Lodestar provides CLI options to configure monitoring on both the beacon node and validator client.
+
+### Remote endpoint URL
+
+Client monitoring can be enabled by setting the `--monitoring.endpoint` flag to a remote service endpoint URL.
+As monitoring relies on metrics data, it is required that metrics are also enabled by supplying the `--metrics` flag.
+
+```bash
+lodestar beacon --monitoring.endpoint "https://beaconcha.in/api/v1/client/metrics?apikey={apikey}&machine={machineName}" --metrics
+```
+
+In case of *beaconcha.in*, the API key can be found in your [account settings](https://beaconcha.in/user/settings#api).
+Setting the machine is optional but it is especially useful if you are monitoring multiple nodes.
+
+<!-- prettier-ignore-start -->
+!!! note
+    When sending data to a remote service you should be conscious about security:
+
+    - Only use a service that you trust as this will send information which may identify you
+      and associate your validators, IP address and other personal information.
+    - Always use a HTTPS connection (i.e. a URL starting with `https://`) to prevent the traffic
+      from being intercepted in transit and leaking information.
+<!-- prettier-ignore-end -->
+
+More details about the data sent to the remote service can be found in the [specification](https://docs.google.com/document/d/1qPWAVRjPCENlyAjUBwGkHMvz9qLdd_6u9DPZcNxDBpc).
+
+It is also possible to print out the data sent to the remote service by enabling debug logs which can be done by supplying the `--logLevel debug` flag.
+
+### Monitoring interval
+
+It is possible to adjust the interval between sending client stats to the remote service by
+setting the `--monitoring.interval` flag. It takes an integer value in milliseconds, the default is `60000` which means data is sent once a minute.
+
+```bash
+lodestar beacon --monitoring.interval 60000
+```
+
+Increasing the monitoring interval can be useful if you are running into rate limit errors when posting large amounts of data for multiple nodes.
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -76,6 +76,7 @@ nav:
       - Validator management: usage/validator-management.md
       - Prometheus & Grafana Setup: usage/prometheus-grafana.md
       - MEV Builder Integration: usage/mev-integration.md 
+      - Client monitoring: usage/client-monitoring.md
   - Reference:
       - Command line: reference/cli.md
   - Libraries: libraries/index.md

diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json
@@ -35,6 +35,9 @@
     "./metrics": {
       "import": "./lib/metrics/index.js"
     },
+    "./monitoring": {
+      "import": "./lib/monitoring/index.js"
+    },
     "./network": {
       "import": "./lib/network/index.js"
     },

diff --git a/packages/beacon-node/src/index.ts b/packages/beacon-node/src/index.ts
@@ -7,6 +7,9 @@ export * from "./node/index.js";
 // Export metrics utilities to de-duplicate validator metrics
 export {RegistryMetricCreator, collectNodeJSMetrics, HttpMetricsServer} from "./metrics/index.js";
 
+// Export monitoring service to make it usable by validator
+export {MonitoringService} from "./monitoring/index.js";
+
 // Export generic RestApi server for CLI
 export {RestApiServer, RestApiServerOpts, RestApiServerModules, RestApiServerMetrics} from "./api/rest/base.js";