From 820558c96c308fe6734bbdc38cc17a2ac8c48eeb Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Wed, 1 Feb 2023 13:35:35 -0500 Subject: [PATCH 01/12] Fetch OS cgroup stats in node-stats telemetry --- esrally/telemetry.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/esrally/telemetry.py b/esrally/telemetry.py index ce151e979..245ad30a0 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -816,6 +816,7 @@ def __init__(self, telemetry_params, cluster_name, client, metrics_store): self.include_network = telemetry_params.get("node-stats-include-network", True) self.include_process = telemetry_params.get("node-stats-include-process", True) self.include_mem_stats = telemetry_params.get("node-stats-include-mem", True) + self.include_cgroup_stats = telemetry_params.get("node-stats-include-cgroup", True) self.include_gc_stats = telemetry_params.get("node-stats-include-gc", True) self.include_indexing_pressure = telemetry_params.get("node-stats-include-indexing-pressure", True) self.client = client @@ -845,6 +846,8 @@ def record(self): if self.include_mem_stats: collected_node_stats.update(self.jvm_mem_stats(node_name, node_stats)) collected_node_stats.update(self.os_mem_stats(node_name, node_stats)) + if self.include_cgroup_stats: + collected_node_stats.update(self.os_cgroup_stats(node_name, node_stats)) if self.include_gc_stats: collected_node_stats.update(self.jvm_gc_stats(node_name, node_stats)) if self.include_network: @@ -906,6 +909,9 @@ def jvm_mem_stats(self, node_name, node_stats): def os_mem_stats(self, node_name, node_stats): return self.flatten_stats_fields(prefix="os_mem", stats=node_stats["os"]["mem"]) + + def os_cgroup_stats(self, node_name, node_stats): + return self.flatten_stats_fields(prefix="os_cgroup", stats=node_stats["os"]["cgroup"]) def jvm_gc_stats(self, node_name, node_stats): return self.flatten_stats_fields(prefix="jvm_gc", stats=node_stats["jvm"]["gc"]) From 1f3333d2a88b12ae2c93a066ed83a7dd4dede74f Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Wed, 1 Feb 2023 16:34:18 -0500 Subject: [PATCH 02/12] Convert os.cgroup.memory limits to int --- esrally/telemetry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 245ad30a0..ae1b4e55a 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -911,6 +911,10 @@ def os_mem_stats(self, node_name, node_stats): return self.flatten_stats_fields(prefix="os_mem", stats=node_stats["os"]["mem"]) def os_cgroup_stats(self, node_name, node_stats): + # Convert strings returned by the Node Stats API for os.cgroup.memory limits + # https://github.com/elastic/elasticsearch/issues/93429 + for k in ("limit_in_bytes", "usage_in_bytes"): + node_stats["os"]["cgroup"]["memory"].update({k: int(node_stats["os"]["cgroup"]["memory"].get(k))}) return self.flatten_stats_fields(prefix="os_cgroup", stats=node_stats["os"]["cgroup"]) def jvm_gc_stats(self, node_name, node_stats): From 89a430c1a24f21e8a202c733c6b71266983133a6 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Wed, 1 Feb 2023 18:20:02 -0500 Subject: [PATCH 03/12] Update tests --- tests/telemetry_test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/telemetry_test.py b/tests/telemetry_test.py index bc4f55226..a50aa997b 100644 --- a/tests/telemetry_test.py +++ b/tests/telemetry_test.py @@ -2089,6 +2089,14 @@ class TestNodeStatsRecorder: "os_mem_used_in_bytes": 57342185472, "os_mem_free_percent": 8, "os_mem_used_percent": 92, + "os_cgroup_cpuacct_usage_nanos": 1394207523870751, + "os_cgroup_cpu_cfs_period_micros": 100000, + "os_cgroup_cpu_cfs_quota_micros": 793162, + "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, + "os_cgroup_cpu_stat_number_of_times_throttled": 41890, + "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, + "os_cgroup_memory_limit_in_bytes": 62277025792, + "os_cgroup_memory_usage_in_bytes": 57342185472, "process_cpu_percent": 10, "process_cpu_total_in_millis": 56520, "breakers_parent_limit_size_in_bytes": 726571417, @@ -2481,6 +2489,14 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc): "os_mem_used_in_bytes": 57342185472, "os_mem_free_percent": 8, "os_mem_used_percent": 92, + "os_cgroup_cpuacct_usage_nanos": 1394207523870751, + "os_cgroup_cpu_cfs_period_micros": 100000, + "os_cgroup_cpu_cfs_quota_micros": 793162, + "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, + "os_cgroup_cpu_stat_number_of_times_throttled": 41890, + "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, + "os_cgroup_memory_limit_in_bytes": 62277025792, + "os_cgroup_memory_usage_in_bytes": 57342185472, "transport_rx_count": 77, "transport_rx_size_in_bytes": 98723498, "transport_server_open": 12, @@ -2794,6 +2810,14 @@ def test_stores_selected_indices_metrics_from_nodes_stats(self, metrics_store_pu "os_mem_used_in_bytes": 57342185472, "os_mem_free_percent": 8, "os_mem_used_percent": 92, + "os_cgroup_cpuacct_usage_nanos": 1394207523870751, + "os_cgroup_cpu_cfs_period_micros": 100000, + "os_cgroup_cpu_cfs_quota_micros": 793162, + "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, + "os_cgroup_cpu_stat_number_of_times_throttled": 41890, + "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, + "os_cgroup_memory_limit_in_bytes": 62277025792, + "os_cgroup_memory_usage_in_bytes": 57342185472, "transport_rx_count": 77, "transport_rx_size_in_bytes": 98723498, "transport_server_open": 12, From 4d7f02feb6c981d2000c3de4c8b2796e535253b6 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Wed, 1 Feb 2023 18:20:13 -0500 Subject: [PATCH 04/12] Fix black --- esrally/telemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esrally/telemetry.py b/esrally/telemetry.py index ae1b4e55a..550e2e1d8 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -909,7 +909,7 @@ def jvm_mem_stats(self, node_name, node_stats): def os_mem_stats(self, node_name, node_stats): return self.flatten_stats_fields(prefix="os_mem", stats=node_stats["os"]["mem"]) - + def os_cgroup_stats(self, node_name, node_stats): # Convert strings returned by the Node Stats API for os.cgroup.memory limits # https://github.com/elastic/elasticsearch/issues/93429 From 7faa79c83653a7d92c35d1e020e398823760ade0 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Wed, 1 Feb 2023 18:29:25 -0500 Subject: [PATCH 05/12] Add node-stats-include-cgroup to docs --- docs/telemetry.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/telemetry.rst b/docs/telemetry.rst index 87a3a595b..f0d529fd8 100644 --- a/docs/telemetry.rst +++ b/docs/telemetry.rst @@ -115,6 +115,7 @@ The node-stats telemetry device regularly calls the `cluster node-stats API Date: Mon, 6 Feb 2023 17:01:38 -0500 Subject: [PATCH 06/12] Refactor cgroup stats collection --- docs/telemetry.rst | 2 +- esrally/telemetry.py | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/telemetry.rst b/docs/telemetry.rst index f0d529fd8..80bbc79ec 100644 --- a/docs/telemetry.rst +++ b/docs/telemetry.rst @@ -133,7 +133,7 @@ Supported telemetry parameters: * ``node-stats-include-breakers`` (default: ``true``): A boolean indicating whether circuit breaker stats should be included. * ``node-stats-include-gc`` (default: ``true``): A boolean indicating whether JVM gc stats should be included. * ``node-stats-include-mem`` (default: ``true``): A boolean indicating whether both JVM heap, and OS mem stats should be included. -* ``node-stats-include-cgroup`` (default: ``true``): A boolean to include or exclude operating system cgroup stats. +* ``node-stats-include-cgroup`` (default: ``false``): A boolean to include or exclude operating system cgroup stats. * ``node-stats-include-network`` (default: ``true``): A boolean indicating whether network-related stats should be included. * ``node-stats-include-process`` (default: ``true``): A boolean indicating whether process cpu stats should be included. * ``node-stats-include-indexing-pressure`` (default: ``true``): A boolean indicating whether indexing pressuer stats should be included. diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 550e2e1d8..57a5fad83 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -777,6 +777,9 @@ def on_benchmark_stop(self): class NodeStatsRecorder: def __init__(self, telemetry_params, cluster_name, client, metrics_store): + self.logger = logging.getLogger(__name__) + + self.logger.info("node stats recorder") self.sample_interval = telemetry_params.get("node-stats-sample-interval", 1) if self.sample_interval <= 0: raise exceptions.SystemSetupError( @@ -816,7 +819,7 @@ def __init__(self, telemetry_params, cluster_name, client, metrics_store): self.include_network = telemetry_params.get("node-stats-include-network", True) self.include_process = telemetry_params.get("node-stats-include-process", True) self.include_mem_stats = telemetry_params.get("node-stats-include-mem", True) - self.include_cgroup_stats = telemetry_params.get("node-stats-include-cgroup", True) + self.include_cgroup_stats = telemetry_params.get("node-stats-include-cgroup", False) self.include_gc_stats = telemetry_params.get("node-stats-include-gc", True) self.include_indexing_pressure = telemetry_params.get("node-stats-include-indexing-pressure", True) self.client = client @@ -911,11 +914,17 @@ def os_mem_stats(self, node_name, node_stats): return self.flatten_stats_fields(prefix="os_mem", stats=node_stats["os"]["mem"]) def os_cgroup_stats(self, node_name, node_stats): - # Convert strings returned by the Node Stats API for os.cgroup.memory limits - # https://github.com/elastic/elasticsearch/issues/93429 - for k in ("limit_in_bytes", "usage_in_bytes"): - node_stats["os"]["cgroup"]["memory"].update({k: int(node_stats["os"]["cgroup"]["memory"].get(k))}) - return self.flatten_stats_fields(prefix="os_cgroup", stats=node_stats["os"]["cgroup"]) + cgroup_stats = {} + try: + # Convert strings returned by the Node Stats API for os.cgroup.memory limits + # https://github.com/elastic/elasticsearch/issues/93429 + for k in ("limit_in_bytes", "usage_in_bytes"): + node_stats["os"]["cgroup"]["memory"].update({k: int(node_stats["os"]["cgroup"]["memory"].get(k))}) + cgroup_stats = self.flatten_stats_fields(prefix="os_cgroup", stats=node_stats["os"]["cgroup"]) + except KeyError: + self.logger.warning("Node cgroup stats requested with none present.") + finally: + return cgroup_stats def jvm_gc_stats(self, node_name, node_stats): return self.flatten_stats_fields(prefix="jvm_gc", stats=node_stats["jvm"]["gc"]) From 9a5f45b30bfcc0379161aee3e1eb9fef7e64ca65 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Mon, 6 Feb 2023 17:29:29 -0500 Subject: [PATCH 07/12] Fix linting error --- esrally/telemetry.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 57a5fad83..a18b81427 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -923,8 +923,7 @@ def os_cgroup_stats(self, node_name, node_stats): cgroup_stats = self.flatten_stats_fields(prefix="os_cgroup", stats=node_stats["os"]["cgroup"]) except KeyError: self.logger.warning("Node cgroup stats requested with none present.") - finally: - return cgroup_stats + return cgroup_stats def jvm_gc_stats(self, node_name, node_stats): return self.flatten_stats_fields(prefix="jvm_gc", stats=node_stats["jvm"]["gc"]) From fabaa68c4f72d51d1d460c22c1c1991d8a3eed39 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Mon, 6 Feb 2023 18:23:19 -0500 Subject: [PATCH 08/12] Resolve test failures --- tests/telemetry_test.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/tests/telemetry_test.py b/tests/telemetry_test.py index a50aa997b..bc4f55226 100644 --- a/tests/telemetry_test.py +++ b/tests/telemetry_test.py @@ -2089,14 +2089,6 @@ class TestNodeStatsRecorder: "os_mem_used_in_bytes": 57342185472, "os_mem_free_percent": 8, "os_mem_used_percent": 92, - "os_cgroup_cpuacct_usage_nanos": 1394207523870751, - "os_cgroup_cpu_cfs_period_micros": 100000, - "os_cgroup_cpu_cfs_quota_micros": 793162, - "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, - "os_cgroup_cpu_stat_number_of_times_throttled": 41890, - "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, - "os_cgroup_memory_limit_in_bytes": 62277025792, - "os_cgroup_memory_usage_in_bytes": 57342185472, "process_cpu_percent": 10, "process_cpu_total_in_millis": 56520, "breakers_parent_limit_size_in_bytes": 726571417, @@ -2489,14 +2481,6 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc): "os_mem_used_in_bytes": 57342185472, "os_mem_free_percent": 8, "os_mem_used_percent": 92, - "os_cgroup_cpuacct_usage_nanos": 1394207523870751, - "os_cgroup_cpu_cfs_period_micros": 100000, - "os_cgroup_cpu_cfs_quota_micros": 793162, - "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, - "os_cgroup_cpu_stat_number_of_times_throttled": 41890, - "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, - "os_cgroup_memory_limit_in_bytes": 62277025792, - "os_cgroup_memory_usage_in_bytes": 57342185472, "transport_rx_count": 77, "transport_rx_size_in_bytes": 98723498, "transport_server_open": 12, @@ -2810,14 +2794,6 @@ def test_stores_selected_indices_metrics_from_nodes_stats(self, metrics_store_pu "os_mem_used_in_bytes": 57342185472, "os_mem_free_percent": 8, "os_mem_used_percent": 92, - "os_cgroup_cpuacct_usage_nanos": 1394207523870751, - "os_cgroup_cpu_cfs_period_micros": 100000, - "os_cgroup_cpu_cfs_quota_micros": 793162, - "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, - "os_cgroup_cpu_stat_number_of_times_throttled": 41890, - "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, - "os_cgroup_memory_limit_in_bytes": 62277025792, - "os_cgroup_memory_usage_in_bytes": 57342185472, "transport_rx_count": 77, "transport_rx_size_in_bytes": 98723498, "transport_server_open": 12, From 9a42ad7d877369d6036265202e0b265dd41b617e Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Mon, 6 Mar 2023 15:04:51 -0500 Subject: [PATCH 09/12] Simplify cgroup collection and add it to tests --- docs/telemetry.rst | 2 +- esrally/telemetry.py | 4 - tests/telemetry_test.py | 238 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 238 insertions(+), 6 deletions(-) diff --git a/docs/telemetry.rst b/docs/telemetry.rst index 80bbc79ec..3ce3356b8 100644 --- a/docs/telemetry.rst +++ b/docs/telemetry.rst @@ -133,7 +133,7 @@ Supported telemetry parameters: * ``node-stats-include-breakers`` (default: ``true``): A boolean indicating whether circuit breaker stats should be included. * ``node-stats-include-gc`` (default: ``true``): A boolean indicating whether JVM gc stats should be included. * ``node-stats-include-mem`` (default: ``true``): A boolean indicating whether both JVM heap, and OS mem stats should be included. -* ``node-stats-include-cgroup`` (default: ``false``): A boolean to include or exclude operating system cgroup stats. +* ``node-stats-include-cgroup`` (default: ``false``): A boolean to include operating system cgroup stats. Memory stats are omitted since Elasticsearch emits them as string values. Use ``os_mem_*`` fields instead. * ``node-stats-include-network`` (default: ``true``): A boolean indicating whether network-related stats should be included. * ``node-stats-include-process`` (default: ``true``): A boolean indicating whether process cpu stats should be included. * ``node-stats-include-indexing-pressure`` (default: ``true``): A boolean indicating whether indexing pressuer stats should be included. diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 0d7c46fac..401780cf5 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -916,10 +916,6 @@ def os_mem_stats(self, node_name, node_stats): def os_cgroup_stats(self, node_name, node_stats): cgroup_stats = {} try: - # Convert strings returned by the Node Stats API for os.cgroup.memory limits - # https://github.com/elastic/elasticsearch/issues/93429 - for k in ("limit_in_bytes", "usage_in_bytes"): - node_stats["os"]["cgroup"]["memory"].update({k: int(node_stats["os"]["cgroup"]["memory"].get(k))}) cgroup_stats = self.flatten_stats_fields(prefix="os_cgroup", stats=node_stats["os"]["cgroup"]) except KeyError: self.logger.warning("Node cgroup stats requested with none present.") diff --git a/tests/telemetry_test.py b/tests/telemetry_test.py index c77b2ef65..392a00a61 100644 --- a/tests/telemetry_test.py +++ b/tests/telemetry_test.py @@ -2397,7 +2397,7 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc): client = Client(nodes=SubClient(stats=node_stats_response)) cfg = create_config() metrics_store = metrics.EsMetricsStore(cfg) - telemetry_params = {"node-stats-include-indices": True} + telemetry_params = {"node-stats-include-indices": True, "node-stats-include-cgroup": True} recorder = telemetry.NodeStatsRecorder(telemetry_params, cluster_name="remote", client=client, metrics_store=metrics_store) recorder.record() @@ -2476,6 +2476,12 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc): "jvm_gc_collectors_young_collection_time_in_millis": 309, "jvm_gc_collectors_old_collection_count": 2, "jvm_gc_collectors_old_collection_time_in_millis": 229, + "os_cgroup_cpuacct_usage_nanos": 1394207523870751, + "os_cgroup_cpu_cfs_period_micros": 100000, + "os_cgroup_cpu_cfs_quota_micros": 793162, + "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, + "os_cgroup_cpu_stat_number_of_times_throttled": 41890, + "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, "os_mem_total_in_bytes": 62277025792, "os_mem_free_in_bytes": 4934840320, "os_mem_used_in_bytes": 57342185472, @@ -2833,6 +2839,236 @@ def test_exception_when_include_indices_metrics_not_valid(self): ): telemetry.NodeStatsRecorder(telemetry_params, cluster_name="remote", client=client, metrics_store=metrics_store) + @mock.patch("esrally.metrics.EsMetricsStore.put_doc") + def test_logs_warning_on_missing_cgroup_stats(self, metrics_store_put_doc): + node_stats_response = { + "cluster_name": "elasticsearch", + "nodes": { + "Zbl_e8EyRXmiR47gbHgPfg": { + "timestamp": 1524379617017, + "name": "rally0", + "transport_address": "127.0.0.1:9300", + "host": "127.0.0.1", + "ip": "127.0.0.1:9300", + "roles": [ + "master", + "data", + "ingest", + ], + "indices": { + "docs": { + "count": 76892364, + "deleted": 324530, + }, + "store": { + "size_in_bytes": 983409834, + }, + "indexing": { + "is_throttled": False, + "throttle_time_in_millis": 0, + }, + "search": { + "open_contexts": 0, + "query_total": 0, + "query_time_in_millis": 0, + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + }, + "refresh": { + "total": 747, + "total_time_in_millis": 277382, + "listeners": 0, + }, + "query_cache": { + "memory_size_in_bytes": 0, + "total_count": 0, + "hit_count": 0, + "miss_count": 0, + "cache_size": 0, + "cache_count": 0, + "evictions": 0, + }, + "fielddata": { + "memory_size_in_bytes": 6936, + "evictions": 17, + }, + "completion": { + "size_in_bytes": 0, + }, + "segments": { + "count": 0, + "memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -9223372036854775808, + "file_sizes": {}, + }, + "translog": { + "operations": 0, + "size_in_bytes": 0, + "uncommitted_operations": 0, + "uncommitted_size_in_bytes": 0, + }, + "request_cache": { + "memory_size_in_bytes": 0, + "evictions": 0, + "hit_count": 0, + "miss_count": 0, + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0, + }, + }, + "jvm": { + "buffer_pools": { + "mapped": { + "count": 7, + "used_in_bytes": 3120, + "total_capacity_in_bytes": 9999, + }, + "direct": { + "count": 6, + "used_in_bytes": 73868, + "total_capacity_in_bytes": 73867, + }, + }, + "classes": { + "current_loaded_count": 9992, + "total_loaded_count": 9992, + "total_unloaded_count": 0, + }, + "mem": { + "heap_used_in_bytes": 119073552, + "heap_used_percent": 19, + "heap_committed_in_bytes": 626393088, + "heap_max_in_bytes": 626393088, + "non_heap_used_in_bytes": 110250424, + "non_heap_committed_in_bytes": 118108160, + "pools": { + "young": { + "used_in_bytes": 66378576, + "max_in_bytes": 139591680, + "peak_used_in_bytes": 139591680, + "peak_max_in_bytes": 139591680, + }, + "survivor": { + "used_in_bytes": 358496, + "max_in_bytes": 17432576, + "peak_used_in_bytes": 17432576, + "peak_max_in_bytes": 17432576, + }, + "old": { + "used_in_bytes": 52336480, + "max_in_bytes": 469368832, + "peak_used_in_bytes": 52336480, + "peak_max_in_bytes": 469368832, + }, + }, + }, + "gc": { + "collectors": { + "young": { + "collection_count": 3, + "collection_time_in_millis": 309, + }, + "old": { + "collection_count": 2, + "collection_time_in_millis": 229, + }, + } + }, + }, + "process": { + "timestamp": 1526045135857, + "open_file_descriptors": 312, + "max_file_descriptors": 1048576, + "cpu": { + "percent": 10, + "total_in_millis": 56520, + }, + "mem": { + "total_virtual_in_bytes": 2472173568, + }, + }, + "os": { + "timestamp": 1655950949872, + "cpu": {"percent": 3, "load_average": {"1m": 3.38, "5m": 3.79, "15m": 3.84}}, + "mem": { + "total_in_bytes": 62277025792, + "free_in_bytes": 4934840320, + "used_in_bytes": 57342185472, + "free_percent": 8, + "used_percent": 92, + }, + "swap": {"total_in_bytes": 0, "free_in_bytes": 0, "used_in_bytes": 0}, + }, + "thread_pool": { + "generic": { + "threads": 4, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 4, + "completed": 8, + }, + }, + "transport": { + "server_open": 12, + "rx_count": 77, + "rx_size_in_bytes": 98723498, + "tx_count": 88, + "tx_size_in_bytes": 23879803, + }, + "breakers": { + "parent": { + "limit_size_in_bytes": 726571417, + "limit_size": "692.9mb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1.0, + "tripped": 0, + } + }, + "indexing_pressure": { + "memory": { + "current": { + "combined_coordinating_and_primary_in_bytes": 0, + "coordinating_in_bytes": 0, + "primary_in_bytes": 0, + "replica_in_bytes": 0, + "all_in_bytes": 0, + }, + "total": { + "combined_coordinating_and_primary_in_bytes": 0, + "coordinating_in_bytes": 0, + "primary_in_bytes": 0, + "replica_in_bytes": 0, + "all_in_bytes": 0, + "coordinating_rejections": 0, + "primary_rejections": 0, + "replica_rejections": 0, + }, + } + }, + } + }, + } + + client = Client(nodes=SubClient(stats=node_stats_response)) + cfg = create_config() + logger = logging.getLogger("esrally.telemetry") + metrics_store = metrics.EsMetricsStore(cfg) + telemetry_params = {"node-stats-include-cgroup": True} + recorder = telemetry.NodeStatsRecorder(telemetry_params, cluster_name="remote", client=client, metrics_store=metrics_store) + + with mock.patch.object(logger, "warning") as mocked_warning: + recorder.record() + mocked_warning.assert_called_once_with( + "Node cgroup stats requested with none present." + ) class TestTransformStats: def test_negative_sample_interval_forbidden(self): From 9f62d3a0430f21f2640096fe823f2daa0d4e2af4 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Mon, 6 Mar 2023 15:40:07 -0500 Subject: [PATCH 10/12] Fix formatting --- tests/telemetry_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/telemetry_test.py b/tests/telemetry_test.py index 392a00a61..ff4c7a736 100644 --- a/tests/telemetry_test.py +++ b/tests/telemetry_test.py @@ -3066,9 +3066,8 @@ def test_logs_warning_on_missing_cgroup_stats(self, metrics_store_put_doc): with mock.patch.object(logger, "warning") as mocked_warning: recorder.record() - mocked_warning.assert_called_once_with( - "Node cgroup stats requested with none present." - ) + mocked_warning.assert_called_once_with("Node cgroup stats requested with none present.") + class TestTransformStats: def test_negative_sample_interval_forbidden(self): From 7da6173cbdb91af755580dd7bec73a2c5bd3b24b Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Wed, 8 Mar 2023 16:37:48 -0500 Subject: [PATCH 11/12] Enable collection of os cgroup stats by default --- docs/telemetry.rst | 2 +- esrally/telemetry.py | 4 ++-- tests/telemetry_test.py | 18 +++++++++++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/telemetry.rst b/docs/telemetry.rst index 3ce3356b8..0b62e86ce 100644 --- a/docs/telemetry.rst +++ b/docs/telemetry.rst @@ -133,7 +133,7 @@ Supported telemetry parameters: * ``node-stats-include-breakers`` (default: ``true``): A boolean indicating whether circuit breaker stats should be included. * ``node-stats-include-gc`` (default: ``true``): A boolean indicating whether JVM gc stats should be included. * ``node-stats-include-mem`` (default: ``true``): A boolean indicating whether both JVM heap, and OS mem stats should be included. -* ``node-stats-include-cgroup`` (default: ``false``): A boolean to include operating system cgroup stats. Memory stats are omitted since Elasticsearch emits them as string values. Use ``os_mem_*`` fields instead. +* ``node-stats-include-cgroup`` (default: ``true``): A boolean to include operating system cgroup stats. Memory stats are omitted since Elasticsearch emits them as string values. Use ``os_mem_*`` fields instead. * ``node-stats-include-network`` (default: ``true``): A boolean indicating whether network-related stats should be included. * ``node-stats-include-process`` (default: ``true``): A boolean indicating whether process cpu stats should be included. * ``node-stats-include-indexing-pressure`` (default: ``true``): A boolean indicating whether indexing pressuer stats should be included. diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 401780cf5..87a72565f 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -819,7 +819,7 @@ def __init__(self, telemetry_params, cluster_name, client, metrics_store): self.include_network = telemetry_params.get("node-stats-include-network", True) self.include_process = telemetry_params.get("node-stats-include-process", True) self.include_mem_stats = telemetry_params.get("node-stats-include-mem", True) - self.include_cgroup_stats = telemetry_params.get("node-stats-include-cgroup", False) + self.include_cgroup_stats = telemetry_params.get("node-stats-include-cgroup", True) self.include_gc_stats = telemetry_params.get("node-stats-include-gc", True) self.include_indexing_pressure = telemetry_params.get("node-stats-include-indexing-pressure", True) self.client = client @@ -918,7 +918,7 @@ def os_cgroup_stats(self, node_name, node_stats): try: cgroup_stats = self.flatten_stats_fields(prefix="os_cgroup", stats=node_stats["os"]["cgroup"]) except KeyError: - self.logger.warning("Node cgroup stats requested with none present.") + self.logger.debug("Node cgroup stats requested with none present.") return cgroup_stats def jvm_gc_stats(self, node_name, node_stats): diff --git a/tests/telemetry_test.py b/tests/telemetry_test.py index ff4c7a736..b1935b704 100644 --- a/tests/telemetry_test.py +++ b/tests/telemetry_test.py @@ -2084,6 +2084,12 @@ class TestNodeStatsRecorder: "jvm_gc_collectors_young_collection_time_in_millis": 309, "jvm_gc_collectors_old_collection_count": 2, "jvm_gc_collectors_old_collection_time_in_millis": 229, + "os_cgroup_cpuacct_usage_nanos": 1394207523870751, + "os_cgroup_cpu_cfs_period_micros": 100000, + "os_cgroup_cpu_cfs_quota_micros": 793162, + "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, + "os_cgroup_cpu_stat_number_of_times_throttled": 41890, + "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, "os_mem_total_in_bytes": 62277025792, "os_mem_free_in_bytes": 4934840320, "os_mem_used_in_bytes": 57342185472, @@ -2795,6 +2801,12 @@ def test_stores_selected_indices_metrics_from_nodes_stats(self, metrics_store_pu "jvm_gc_collectors_young_collection_time_in_millis": 309, "jvm_gc_collectors_old_collection_count": 2, "jvm_gc_collectors_old_collection_time_in_millis": 229, + "os_cgroup_cpuacct_usage_nanos": 1394207523870751, + "os_cgroup_cpu_cfs_period_micros": 100000, + "os_cgroup_cpu_cfs_quota_micros": 793162, + "os_cgroup_cpu_stat_number_of_elapsed_periods": 41092415, + "os_cgroup_cpu_stat_number_of_times_throttled": 41890, + "os_cgroup_cpu_stat_time_throttled_nanos": 29380593023188, "os_mem_total_in_bytes": 62277025792, "os_mem_free_in_bytes": 4934840320, "os_mem_used_in_bytes": 57342185472, @@ -2840,7 +2852,7 @@ def test_exception_when_include_indices_metrics_not_valid(self): telemetry.NodeStatsRecorder(telemetry_params, cluster_name="remote", client=client, metrics_store=metrics_store) @mock.patch("esrally.metrics.EsMetricsStore.put_doc") - def test_logs_warning_on_missing_cgroup_stats(self, metrics_store_put_doc): + def test_logs_debug_on_missing_cgroup_stats(self, metrics_store_put_doc): node_stats_response = { "cluster_name": "elasticsearch", "nodes": { @@ -3064,9 +3076,9 @@ def test_logs_warning_on_missing_cgroup_stats(self, metrics_store_put_doc): telemetry_params = {"node-stats-include-cgroup": True} recorder = telemetry.NodeStatsRecorder(telemetry_params, cluster_name="remote", client=client, metrics_store=metrics_store) - with mock.patch.object(logger, "warning") as mocked_warning: + with mock.patch.object(logger, "debug") as mocked_debug: recorder.record() - mocked_warning.assert_called_once_with("Node cgroup stats requested with none present.") + mocked_debug.assert_called_once_with("Node cgroup stats requested with none present.") class TestTransformStats: From 91fda49ce44f82106e13d44cdb3a9c8efa6e30b1 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Wed, 8 Mar 2023 16:42:32 -0500 Subject: [PATCH 12/12] Remove extra line --- esrally/telemetry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 87a72565f..3ee8065e5 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -778,7 +778,6 @@ def on_benchmark_stop(self): class NodeStatsRecorder: def __init__(self, telemetry_params, cluster_name, client, metrics_store): self.logger = logging.getLogger(__name__) - self.logger.info("node stats recorder") self.sample_interval = telemetry_params.get("node-stats-sample-interval", 1) if self.sample_interval <= 0: