diff --git a/conftest.py b/conftest.py index ff4e56ee12..387da1ee43 100644 --- a/conftest.py +++ b/conftest.py @@ -698,3 +698,7 @@ def setup_test_colo( assert colo_model.colocated # Check to make sure that limit_db_cpus made it into the colo settings return colo_model + +@pytest.fixture +def config() -> smartsim._core.config.Config: + return CONFIG diff --git a/smartsim/_core/config/config.py b/smartsim/_core/config/config.py index af5ebe5084..e80b81946f 100644 --- a/smartsim/_core/config/config.py +++ b/smartsim/_core/config/config.py @@ -231,6 +231,9 @@ def telemetry_enabled(self) -> bool: def telemetry_cooldown(self) -> int: return int(os.environ.get("SMARTSIM_TELEMETRY_COOLDOWN", 90)) + @property + def telemetry_subdir(self) -> str: + return ".smartsim/telemetry" @lru_cache(maxsize=128, typed=False) def get_config() -> Config: diff --git a/smartsim/_core/control/manifest.py b/smartsim/_core/control/manifest.py index 62ab013e58..62e6e66345 100644 --- a/smartsim/_core/control/manifest.py +++ b/smartsim/_core/control/manifest.py @@ -29,6 +29,7 @@ from dataclasses import dataclass, field from ...database import Orchestrator +from ..config import CONFIG from ...entity import DBNode, Ensemble, EntitySequence, Model, SmartSimEntity from ...error import SmartSimError from ..utils import helpers as _helpers @@ -343,7 +344,7 @@ def finalize(self) -> LaunchedManifest[_T]: def _format_exp_telemetry_path( exp_path: t.Union[str, "os.PathLike[str]"] ) -> pathlib.Path: - return pathlib.Path(exp_path, _serialize.TELMON_SUBDIR) + return pathlib.Path(exp_path, CONFIG.telemetry_subdir) def _format_run_telemetry_path( diff --git a/smartsim/_core/entrypoints/telemetrymonitor.py b/smartsim/_core/entrypoints/telemetrymonitor.py index 86d6fe72fb..5023a7030f 100644 --- a/smartsim/_core/entrypoints/telemetrymonitor.py +++ b/smartsim/_core/entrypoints/telemetrymonitor.py @@ -57,7 +57,7 @@ from smartsim._core.launcher.slurm.slurmLauncher import SlurmLauncher from smartsim._core.launcher.stepInfo import StepInfo from smartsim._core.utils.helpers import get_ts -from smartsim._core.utils.serialize import MANIFEST_FILENAME, TELMON_SUBDIR +from smartsim._core.utils.serialize import MANIFEST_FILENAME from smartsim.error.errors import SmartSimError from smartsim.status import STATUS_COMPLETED, TERMINAL_STATUSES @@ -582,7 +582,7 @@ def main( poll for new jobs before attempting to shutdown :type cooldown_duration: int """ - manifest_relpath = pathlib.Path(TELMON_SUBDIR) / MANIFEST_FILENAME + manifest_relpath = pathlib.Path(CONFIG.telemetry_subdir) / MANIFEST_FILENAME manifest_path = experiment_dir / manifest_relpath monitor_pattern = str(manifest_relpath) @@ -667,7 +667,7 @@ def get_parser() -> argparse.ArgumentParser: log.setLevel(logging.DEBUG) log.propagate = False - log_path = os.path.join(args.exp_dir, TELMON_SUBDIR, "telemetrymonitor.log") + log_path = os.path.join(args.exp_dir, CONFIG.telemetry_subdir, "telemetrymonitor.log") fh = logging.FileHandler(log_path, "a") log.addHandler(fh) diff --git a/smartsim/_core/utils/serialize.py b/smartsim/_core/utils/serialize.py index 70d6b21bb1..2ae8ed764b 100644 --- a/smartsim/_core/utils/serialize.py +++ b/smartsim/_core/utils/serialize.py @@ -47,7 +47,7 @@ TStepLaunchMetaData = t.Tuple[ t.Optional[str], t.Optional[str], t.Optional[bool], str, str, Path ] -TELMON_SUBDIR: t.Final[str] = ".smartsim/telemetry" + MANIFEST_FILENAME: t.Final[str] = "manifest.json" _LOGGER = smartsim.log.get_logger(__name__) diff --git a/tests/test_configs/echo.py b/tests/test_configs/echo.py index 6523f4e4ff..ce96f82e1d 100644 --- a/tests/test_configs/echo.py +++ b/tests/test_configs/echo.py @@ -28,10 +28,10 @@ import time -def echo(message: str, sleep_time: int): +def echo(message, sleep_time): if sleep_time > 0: time.sleep(sleep_time) - print(f"Echoing: {message}") + print("Echoing:", message) if __name__ == "__main__": diff --git a/tests/test_indirect.py b/tests/test_indirect.py index f8af882668..56fc69ff41 100644 --- a/tests/test_indirect.py +++ b/tests/test_indirect.py @@ -31,9 +31,9 @@ import psutil import pytest +from smartsim._core.config import CONFIG from smartsim._core.entrypoints.indirect import cleanup, get_parser, get_ts, main from smartsim._core.utils.helpers import encode_cmd -from smartsim._core.utils.serialize import MANIFEST_FILENAME, TELMON_SUBDIR ALL_ARGS = { "+command", @@ -152,7 +152,7 @@ def test_indirect_main_dir_check(test_dir): cmd = ["echo", "unit-test"] encoded_cmd = encode_cmd(cmd) - status_path = exp_dir / TELMON_SUBDIR + status_path = exp_dir / CONFIG.telemetry_subdir # show that a missing status_path is created when missing main(encoded_cmd, "application", exp_dir, status_path) @@ -167,7 +167,7 @@ def test_indirect_main_cmd_check(capsys, test_dir, monkeypatch): captured = capsys.readouterr() # throw away existing output with monkeypatch.context() as ctx, pytest.raises(ValueError) as ex: ctx.setattr("smartsim._core.entrypoints.indirect.logger.error", print) - _ = main("", "application", exp_dir, exp_dir / TELMON_SUBDIR) + _ = main("", "application", exp_dir, exp_dir / CONFIG.telemetry_subdir) captured = capsys.readouterr() assert "Invalid cmd supplied" in ex.value.args[0] @@ -175,7 +175,7 @@ def test_indirect_main_cmd_check(capsys, test_dir, monkeypatch): # test with non-emptystring cmd with monkeypatch.context() as ctx, pytest.raises(ValueError) as ex: ctx.setattr("smartsim._core.entrypoints.indirect.logger.error", print) - _ = main(" \n \t ", "application", exp_dir, exp_dir / TELMON_SUBDIR) + _ = main(" \n \t ", "application", exp_dir, exp_dir / CONFIG.telemetry_subdir) captured = capsys.readouterr() assert "Invalid cmd supplied" in ex.value.args[0] @@ -190,13 +190,13 @@ def test_complete_process(fileutils, test_dir): raw_cmd = f"{sys.executable} {script} --time=1" cmd = encode_cmd(raw_cmd.split()) - rc = main(cmd, "application", exp_dir, exp_dir / TELMON_SUBDIR) + rc = main(cmd, "application", exp_dir, exp_dir / CONFIG.telemetry_subdir) assert rc == 0 assert exp_dir.exists() # NOTE: don't have a manifest so we're falling back to default event path - data_dir = exp_dir / TELMON_SUBDIR + data_dir = exp_dir / CONFIG.telemetry_subdir start_events = list(data_dir.rglob("start.json")) stop_events = list(data_dir.rglob("stop.json")) diff --git a/tests/test_serialize.py b/tests/test_serialize.py index 167e7e445f..93ec793697 100644 --- a/tests/test_serialize.py +++ b/tests/test_serialize.py @@ -37,7 +37,7 @@ from smartsim._core.utils import serialize from smartsim.database.orchestrator import Orchestrator -_REL_MANIFEST_PATH = f"{serialize.TELMON_SUBDIR}/{serialize.MANIFEST_FILENAME}" + _CFG_TM_ENABLED_ATTR = "telemetry_enabled" # The tests in this file belong to the group_b group @@ -54,10 +54,10 @@ def turn_on_tm(monkeypatch): yield -def test_serialize_creates_a_manifest_json_file_if_dne(test_dir): +def test_serialize_creates_a_manifest_json_file_if_dne(test_dir, config): lmb = LaunchedManifestBuilder("exp", test_dir, "launcher") serialize.save_launch_manifest(lmb.finalize()) - manifest_json = Path(test_dir) / _REL_MANIFEST_PATH + manifest_json = Path(test_dir) / config.telemetry_subdir / serialize.MANIFEST_FILENAME assert manifest_json.is_file() with open(manifest_json, "r") as f: @@ -69,7 +69,7 @@ def test_serialize_creates_a_manifest_json_file_if_dne(test_dir): def test_serialize_does_not_write_manifest_json_if_telemetry_monitor_is_off( - test_dir, monkeypatch + test_dir, monkeypatch, config ): monkeypatch.setattr( smartsim._core.config.config.Config, @@ -78,12 +78,12 @@ def test_serialize_does_not_write_manifest_json_if_telemetry_monitor_is_off( ) lmb = LaunchedManifestBuilder("exp", test_dir, "launcher") serialize.save_launch_manifest(lmb.finalize()) - manifest_json = Path(test_dir) / _REL_MANIFEST_PATH + manifest_json = Path(test_dir) / config.telemetry_subdir / serialize.MANIFEST_FILENAME assert not manifest_json.exists() -def test_serialize_appends_a_manifest_json_exists(test_dir): - manifest_json = Path(test_dir) / _REL_MANIFEST_PATH +def test_serialize_appends_a_manifest_json_exists(test_dir, config): + manifest_json = Path(test_dir) / config.telemetry_subdir / serialize.MANIFEST_FILENAME serialize.save_launch_manifest( LaunchedManifestBuilder("exp", test_dir, "launcher").finalize() ) @@ -102,8 +102,8 @@ def test_serialize_appends_a_manifest_json_exists(test_dir): assert len({run["run_id"] for run in manifest["runs"]}) == 3 -def test_serialize_overwites_file_if_not_json(test_dir): - manifest_json = Path(test_dir) / _REL_MANIFEST_PATH +def test_serialize_overwites_file_if_not_json(test_dir, config): + manifest_json = Path(test_dir) / config.telemetry_subdir / serialize.MANIFEST_FILENAME manifest_json.parent.mkdir(parents=True, exist_ok=True) with open(manifest_json, "w") as f: f.write("This is not a json\n") @@ -114,7 +114,7 @@ def test_serialize_overwites_file_if_not_json(test_dir): assert isinstance(json.load(f), dict) -def test_started_entities_are_serialized(test_dir): +def test_started_entities_are_serialized(test_dir, config): exp_name = "test-exp" test_dir = Path(test_dir) / exp_name test_dir.mkdir(parents=True) @@ -131,7 +131,7 @@ def test_started_entities_are_serialized(test_dir): exp.start(hello_world_model, spam_eggs_model, block=False) exp.start(hello_ensemble, block=False) - manifest_json = Path(exp.exp_path) / _REL_MANIFEST_PATH + manifest_json = Path(exp.exp_path) / config.telemetry_subdir / serialize.MANIFEST_FILENAME try: with open(manifest_json, "r") as f: manifest = json.load(f) diff --git a/tests/test_telemetry_monitor.py b/tests/test_telemetry_monitor.py index 3f804b077d..a4214945ee 100644 --- a/tests/test_telemetry_monitor.py +++ b/tests/test_telemetry_monitor.py @@ -26,6 +26,7 @@ import logging +import os import pathlib import sys import time @@ -93,8 +94,8 @@ def turn_on_tm(monkeypatch): yield -def snooze_nonblocking(test_dir: str, max_delay: int = 20, post_data_delay: int = 2): - telmon_subdir = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR +def snooze_nonblocking(test_dir: os.PathLike[str], max_delay: int = 20, post_data_delay: int = 2): + telmon_subdir = pathlib.Path(test_dir) # let the non-blocking experiment complete. for _ in range(max_delay): time.sleep(1) @@ -179,7 +180,7 @@ def test_track_event( assert expected_output.is_file() -def test_load_manifest(fileutils: FileUtils, test_dir: str): +def test_load_manifest(fileutils: FileUtils, test_dir: str, config: cfg.Config): """Ensure that the runtime manifest loads correctly""" sample_manifest_path = fileutils.get_test_conf_path("telemetry/telemetry.json") sample_manifest = pathlib.Path(sample_manifest_path) @@ -187,7 +188,7 @@ def test_load_manifest(fileutils: FileUtils, test_dir: str): test_manifest_path = fileutils.make_test_file( serialize.MANIFEST_FILENAME, - pathlib.Path(test_dir) / serialize.TELMON_SUBDIR, + pathlib.Path(test_dir) / config.telemetry_subdir, sample_manifest.read_text(), ) test_manifest = pathlib.Path(test_manifest_path) @@ -431,7 +432,7 @@ def is_alive(self) -> bool: assert observer.stop_count == 1 -def test_telemetry_single_model(fileutils, test_dir, wlmutils): +def test_telemetry_single_model(fileutils, test_dir, wlmutils, config): """Test that it is possible to create_database then colocate_db_uds/colocate_db_tcp with unique db_identifiers""" @@ -456,7 +457,7 @@ def test_telemetry_single_model(fileutils, test_dir, wlmutils): exp.start(smartsim_model, block=True) assert exp.get_status(smartsim_model)[0] == STATUS_COMPLETED - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -464,7 +465,7 @@ def test_telemetry_single_model(fileutils, test_dir, wlmutils): assert len(stop_events) == 1 -def test_telemetry_single_model_nonblocking(fileutils, test_dir, wlmutils, monkeypatch): +def test_telemetry_single_model_nonblocking(fileutils, test_dir, wlmutils, monkeypatch, config): """Ensure that the telemetry monitor logs exist when the experiment is non-blocking""" with monkeypatch.context() as ctx: @@ -490,11 +491,11 @@ def test_telemetry_single_model_nonblocking(fileutils, test_dir, wlmutils, monke exp.generate(smartsim_model) exp.start(smartsim_model) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=30) + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=30) assert exp.get_status(smartsim_model)[0] == STATUS_COMPLETED - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -502,7 +503,7 @@ def test_telemetry_single_model_nonblocking(fileutils, test_dir, wlmutils, monke assert len(stop_events) == 1 -def test_telemetry_serial_models(fileutils, test_dir, wlmutils, monkeypatch): +def test_telemetry_serial_models(fileutils, test_dir, wlmutils, monkeypatch, config): """ Test telemetry with models being run in serial (one after each other) """ @@ -534,7 +535,7 @@ def test_telemetry_serial_models(fileutils, test_dir, wlmutils, monkeypatch): [status == STATUS_COMPLETED for status in exp.get_status(*smartsim_models)] ) - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -543,7 +544,7 @@ def test_telemetry_serial_models(fileutils, test_dir, wlmutils, monkeypatch): def test_telemetry_serial_models_nonblocking( - fileutils, test_dir, wlmutils, monkeypatch + fileutils, test_dir, wlmutils, monkeypatch, config ): """ Test telemetry with models being run in serial (one after each other) @@ -574,13 +575,13 @@ def test_telemetry_serial_models_nonblocking( exp.generate(*smartsim_models) exp.start(*smartsim_models) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=10) + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=10) assert all( [status == STATUS_COMPLETED for status in exp.get_status(*smartsim_models)] ) - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -588,7 +589,7 @@ def test_telemetry_serial_models_nonblocking( assert len(stop_events) == 5 -def test_telemetry_db_only_with_generate(test_dir, wlmutils, monkeypatch): +def test_telemetry_db_only_with_generate(test_dir, wlmutils, monkeypatch, config): """ Test telemetry with only a database running """ @@ -609,12 +610,14 @@ def test_telemetry_db_only_with_generate(test_dir, wlmutils, monkeypatch): # create regular database orc = exp.create_database(port=test_port, interface=test_interface) exp.generate(orc) + + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir + try: exp.start(orc, block=True) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=10) + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=10) - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -622,7 +625,7 @@ def test_telemetry_db_only_with_generate(test_dir, wlmutils, monkeypatch): assert len(stop_events) <= 1 finally: exp.stop(orc) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=10) + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=10) assert exp.get_status(orc)[0] == STATUS_CANCELLED @@ -630,7 +633,7 @@ def test_telemetry_db_only_with_generate(test_dir, wlmutils, monkeypatch): assert len(stop_events) == 1 -def test_telemetry_db_only_without_generate(test_dir, wlmutils, monkeypatch): +def test_telemetry_db_only_without_generate(test_dir, wlmutils, monkeypatch, config): """ Test telemetry with only a non-generated database running """ @@ -651,13 +654,13 @@ def test_telemetry_db_only_without_generate(test_dir, wlmutils, monkeypatch): # create regular database orc = exp.create_database(port=test_port, interface=test_interface) orc.set_path(test_dir) + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir try: exp.start(orc) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=30) + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=30) - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -666,14 +669,14 @@ def test_telemetry_db_only_without_generate(test_dir, wlmutils, monkeypatch): finally: exp.stop(orc) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=10) + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=10) assert exp.get_status(orc)[0] == STATUS_CANCELLED stop_events = list(telemetry_output_path.rglob("stop.json")) assert len(stop_events) == 1 -def test_telemetry_db_and_model(fileutils, test_dir, wlmutils, monkeypatch): +def test_telemetry_db_and_model(fileutils, test_dir, wlmutils, monkeypatch, config): """ Test telemetry with only a database and a model running """ @@ -711,13 +714,12 @@ def test_telemetry_db_and_model(fileutils, test_dir, wlmutils, monkeypatch): finally: exp.stop(orc) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=30) + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=30) assert exp.get_status(orc)[0] == STATUS_CANCELLED assert exp.get_status(smartsim_model)[0] == STATUS_COMPLETED - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR - start_events = list(telemetry_output_path.rglob("database/**/start.json")) stop_events = list(telemetry_output_path.rglob("database/**/stop.json")) @@ -730,7 +732,7 @@ def test_telemetry_db_and_model(fileutils, test_dir, wlmutils, monkeypatch): assert len(stop_events) == 1 -def test_telemetry_ensemble(fileutils, test_dir, wlmutils, monkeypatch): +def test_telemetry_ensemble(fileutils, test_dir, wlmutils, monkeypatch, config): """ Test telemetry with only an ensemble """ @@ -757,8 +759,8 @@ def test_telemetry_ensemble(fileutils, test_dir, wlmutils, monkeypatch): exp.start(ens, block=True) assert all([status == STATUS_COMPLETED for status in exp.get_status(ens)]) - snooze_nonblocking(test_dir, max_delay=60, post_data_delay=30) - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir + snooze_nonblocking(telemetry_output_path, max_delay=60, post_data_delay=30) start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -766,7 +768,7 @@ def test_telemetry_ensemble(fileutils, test_dir, wlmutils, monkeypatch): assert len(stop_events) == 5 -def test_telemetry_colo(fileutils, test_dir, wlmutils, coloutils, monkeypatch): +def test_telemetry_colo(fileutils, test_dir, wlmutils, coloutils, monkeypatch, config): """ Test telemetry with only a colocated model running """ @@ -797,7 +799,7 @@ def test_telemetry_colo(fileutils, test_dir, wlmutils, coloutils, monkeypatch): [status == STATUS_COMPLETED for status in exp.get_status(smartsim_model)] ) - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir start_events = list(telemetry_output_path.rglob("start.json")) stop_events = list(telemetry_output_path.rglob("stop.json")) @@ -814,7 +816,7 @@ def test_telemetry_colo(fileutils, test_dir, wlmutils, coloutils, monkeypatch): pytest.param(1, 15, id="15s shutdown"), ], ) -def test_telemetry_autoshutdown(test_dir, wlmutils, monkeypatch, frequency, cooldown): +def test_telemetry_autoshutdown(test_dir, wlmutils, monkeypatch, frequency, cooldown, config): """ Ensure that the telemetry monitor process shuts down after the desired cooldown period @@ -837,7 +839,7 @@ def test_telemetry_autoshutdown(test_dir, wlmutils, monkeypatch, frequency, cool stop_time = start_time exp.start(block=False) - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir empty_mani = list(telemetry_output_path.rglob("manifest.json")) assert len(empty_mani) == 1, "an manifest.json should be created" @@ -867,8 +869,8 @@ def get_launch_cmd(self): @pytest.fixture -def mock_step_meta_dict(test_dir): - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR +def mock_step_meta_dict(test_dir, config): + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir yield { "entity_type": "mock", "status_dir": telemetry_output_path, @@ -958,6 +960,7 @@ def test_multistart_experiment( test_dir: str, monkeypatch: pytest.MonkeyPatch, run_command: str, + config: cfg.Config, ): """Run an experiment with multiple start calls to ensure that telemetry is saved correctly for each run @@ -1016,7 +1019,7 @@ def test_multistart_experiment( assert tm_pid == exp._control._telemetry_monitor.pid time.sleep(3) # time for telmon to write db stop event - telemetry_output_path = pathlib.Path(test_dir) / serialize.TELMON_SUBDIR + telemetry_output_path = pathlib.Path(test_dir) / config.telemetry_subdir db_start_events = list(telemetry_output_path.rglob("database/**/start.json")) db_stop_events = list(telemetry_output_path.rglob("database/**/stop.json"))