From 42f7ff1a04727ab276590656e89516bea7e5320f Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Mon, 17 Jun 2024 18:25:51 -0500 Subject: [PATCH 01/82] Builders are generic, have a format method --- smartsim/settings/builders/launch/dragon.py | 2 +- smartsim/settings/builders/launch/local.py | 2 +- smartsim/settings/builders/launch/lsf.py | 2 +- smartsim/settings/builders/launch/mpi.py | 2 +- smartsim/settings/builders/launch/pals.py | 2 +- smartsim/settings/builders/launch/slurm.py | 2 +- smartsim/settings/builders/launchArgBuilder.py | 16 +++++++++++++--- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 1ca0a244d..6043d6934 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) -class DragonArgBuilder(LaunchArgBuilder): +class DragonArgBuilder(LaunchArgBuilder[t.Any]): # TODO: come back and fix this def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Dragon.value diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index 595514f15..cdcd22a1c 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) -class LocalArgBuilder(LaunchArgBuilder): +class LocalArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Local.value diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index 2c72002e5..293f33281 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) -class JsrunArgBuilder(LaunchArgBuilder): +class JsrunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Lsf.value diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 1331be317..2fa68450c 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) -class _BaseMPIArgBuilder(LaunchArgBuilder): +class _BaseMPIArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wd", "wdir"} diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index 051409c29..77a907e7a 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) -class PalsMpiexecArgBuilder(LaunchArgBuilder): +class PalsMpiexecArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Pals.value diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 80d3d6be2..e0edd74dd 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -39,7 +39,7 @@ logger = get_logger(__name__) -class SlurmArgBuilder(LaunchArgBuilder): +class SlurmArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Slurm.value diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py index bb1f389f3..8a839a5c8 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/builders/launchArgBuilder.py @@ -37,7 +37,10 @@ logger = get_logger(__name__) -class LaunchArgBuilder(ABC): +_T = t.TypeVar("_T") + + +class LaunchArgBuilder(ABC, t.Generic[_T]): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the responsibility of child classes for each launcher to translate @@ -50,12 +53,14 @@ def __init__(self, launch_args: t.Dict[str, str | None] | None) -> None: @abstractmethod def launcher_str(self) -> str: """Get the string representation of the launcher""" - pass @abstractmethod def set(self, arg: str, val: str | None) -> None: """Set the launch arguments""" - pass + + @abstractmethod + def finalize(self, exe: ExecutableLike, env: dict[str, str | None]) -> _T: + """Prepare an entity for launch using the built options""" def format_launch_args(self) -> t.Union[t.List[str], None]: """Build formatted launch arguments""" @@ -90,3 +95,8 @@ def format_env_vars( def __str__(self) -> str: # pragma: no-cover string = f"\nLaunch Arguments:\n{fmt_dict(self._launch_args)}" return string + + +class ExecutableLike(t.Protocol): + @abstractmethod + def as_program_arguments(self) -> t.Sequence[str]: ... From f76ee4fe161a85ab4b5cf38d57e83111b3b08f79 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 18 Jun 2024 16:32:26 -0500 Subject: [PATCH 02/82] Impl slurm --- smartsim/settings/builders/launch/slurm.py | 13 +++++++ tests/temp_tests/test_settings/conftest.py | 38 +++++++++++++++++++ .../test_settings/test_slurmLauncher.py | 38 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 tests/temp_tests/test_settings/conftest.py diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index e0edd74dd..3210c6665 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -36,6 +36,9 @@ from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder +if t.TYPE_CHECKING: + from smartsim.settings.builders.launchArgBuilder import ExecutableLike + logger = get_logger(__name__) @@ -315,3 +318,13 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return ( + "srun", + *(self.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + ) diff --git a/tests/temp_tests/test_settings/conftest.py b/tests/temp_tests/test_settings/conftest.py new file mode 100644 index 000000000..ad1fa0f4a --- /dev/null +++ b/tests/temp_tests/test_settings/conftest.py @@ -0,0 +1,38 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +from smartsim.settings.builders import launchArgBuilder as launch + + +@pytest.fixture +def echo_executable_like(): + class _ExeLike(launch.ExecutableLike): + def as_program_arguments(self): + return ("echo", "hello", "world") + + return _ExeLike() diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index c5e9b5b62..bfa7dd9e1 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -4,6 +4,8 @@ from smartsim.settings.builders.launch.slurm import SlurmArgBuilder from smartsim.settings.launchCommand import LauncherType +pytestmark = pytest.mark.group_a + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" @@ -253,3 +255,39 @@ def test_set_het_groups(monkeypatch): assert slurmLauncher._arg_builder._launch_args["het-group"] == "3,2" with pytest.raises(ValueError): slurmLauncher.launch_args.set_het_group([4]) + + +@pytest.mark.parametrize( + "args, expected", + ( + pytest.param({}, ("srun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {"N": "1"}, + ("srun", "-N", "1", "--", "echo", "hello", "world"), + id="Short Arg", + ), + pytest.param( + {"nodes": "1"}, + ("srun", "--nodes=1", "--", "echo", "hello", "world"), + id="Long Arg", + ), + pytest.param( + {"v": None}, + ("srun", "-v", "--", "echo", "hello", "world"), + id="Short Arg (No Value)", + ), + pytest.param( + {"verbose": None}, + ("srun", "--verbose", "--", "echo", "hello", "world"), + id="Long Arg (No Value)", + ), + pytest.param( + {"nodes": "1", "n": "123"}, + ("srun", "--nodes=1", "-n", "123", "--", "echo", "hello", "world"), + id="Short and Long Args", + ), + ), +) +def test_formatting_launch_args(echo_executable_like, args, expected): + cmd = SlurmArgBuilder(args).finalize(echo_executable_like, {}) + assert tuple(cmd) == expected From 3027721ae9ad0aa8e74de677f4e3c9acb3ac4d38 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 18 Jun 2024 16:34:22 -0500 Subject: [PATCH 03/82] Impl local --- smartsim/settings/builders/launch/local.py | 8 ++++++++ tests/temp_tests/test_settings/test_localLauncher.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index cdcd22a1c..78e2dd28f 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -34,6 +34,9 @@ from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder +if t.TYPE_CHECKING: + from smartsim.settings.builders.launchArgBuilder import ExecutableLike + logger = get_logger(__name__) @@ -72,3 +75,8 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return exe.as_program_arguments() diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 1ee7b9d87..4eb314a8b 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -4,6 +4,8 @@ from smartsim.settings.builders.launch.local import LocalArgBuilder from smartsim.settings.launchCommand import LauncherType +pytestmark = pytest.mark.group_a + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" @@ -110,3 +112,8 @@ def test_format_env_vars(): localLauncher = LaunchSettings(launcher=LauncherType.Local, env_vars=env_vars) assert isinstance(localLauncher._arg_builder, LocalArgBuilder) assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] + + +def test_formatting_returns_original_exe(echo_executable_like): + cmd = LocalArgBuilder({}).finalize(echo_executable_like, {}) + assert tuple(cmd) == ("echo", "hello", "world") From 6266b759f243440f71577093771a23cfb36e5cbc Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 18 Jun 2024 17:31:01 -0500 Subject: [PATCH 04/82] Impl jsrun --- smartsim/settings/builders/launch/lsf.py | 13 +++++++ .../test_settings/test_lsfLauncher.py | 38 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index 293f33281..7fd3217ab 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -34,6 +34,9 @@ from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder +if t.TYPE_CHECKING: + from smartsim.settings.builders.launchArgBuilder import ExecutableLike + logger = get_logger(__name__) @@ -115,3 +118,13 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return ( + "jsrun", + *(self.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + ) diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 4c4260ac5..592c80ce7 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -4,6 +4,8 @@ from smartsim.settings.builders.launch.lsf import JsrunArgBuilder from smartsim.settings.launchCommand import LauncherType +pytestmark = pytest.mark.group_a + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" @@ -56,3 +58,39 @@ def test_launch_args(): "--np=100", ] assert formatted == result + + +@pytest.mark.parametrize( + "args, expected", + ( + pytest.param({}, ("jsrun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {"n": "1"}, + ("jsrun", "-n", "1", "--", "echo", "hello", "world"), + id="Short Arg", + ), + pytest.param( + {"nrs": "1"}, + ("jsrun", "--nrs=1", "--", "echo", "hello", "world"), + id="Long Arg", + ), + pytest.param( + {"v": None}, + ("jsrun", "-v", "--", "echo", "hello", "world"), + id="Short Arg (No Value)", + ), + pytest.param( + {"verbose": None}, + ("jsrun", "--verbose", "--", "echo", "hello", "world"), + id="Long Arg (No Value)", + ), + pytest.param( + {"tasks_per_rs": "1", "n": "123"}, + ("jsrun", "--tasks_per_rs=1", "-n", "123", "--", "echo", "hello", "world"), + id="Short and Long Args", + ), + ), +) +def test_formatting_launch_args(echo_executable_like, args, expected): + cmd = JsrunArgBuilder(args).finalize(echo_executable_like, {}) + assert tuple(cmd) == expected From bbb515297498985f929535b3475a52a8484527ab Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 18 Jun 2024 18:15:11 -0500 Subject: [PATCH 05/82] Impl mpi{run,exec}, orterun REBASEME: moar mpi REBASEME: fmt mpi --- smartsim/settings/builders/launch/mpi.py | 46 +++++++++++-------- smartsim/settings/builders/launch/pals.py | 13 ++++++ .../test_settings/test_mpiLauncher.py | 46 +++++++++++++++++++ .../test_settings/test_palsLauncher.py | 38 +++++++++++++++ 4 files changed, 125 insertions(+), 18 deletions(-) diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 2fa68450c..b012a4271 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -34,6 +34,9 @@ from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder +if t.TYPE_CHECKING: + from smartsim.settings.builders.launchArgBuilder import ExecutableLike + logger = get_logger(__name__) @@ -215,36 +218,43 @@ def set(self, key: str, value: str | None) -> None: class MpiArgBuilder(_BaseMPIArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) - def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpirun.value + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return ("mpirun", *self.format_launch_args(), "--", *exe.as_program_arguments()) -class MpiexecArgBuilder(_BaseMPIArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) +class MpiexecArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpiexec.value + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return ( + "mpiexec", + *self.format_launch_args(), + "--", + *exe.as_program_arguments(), + ) -class OrteArgBuilder(_BaseMPIArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) +class OrteArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Orterun.value + + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return ( + "orterun", + *self.format_launch_args(), + "--", + *exe.as_program_arguments(), + ) diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index 77a907e7a..af8cd7706 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -34,6 +34,9 @@ from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder +if t.TYPE_CHECKING: + from smartsim.settings.builders.launchArgBuilder import ExecutableLike + logger = get_logger(__name__) @@ -149,3 +152,13 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return ( + "mpiexec", + *(self.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + ) diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 815f0c5c1..9b651c220 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -10,6 +10,8 @@ ) from smartsim.settings.launchCommand import LauncherType +pytestmark = pytest.mark.group_a + @pytest.mark.parametrize( "launcher", @@ -205,3 +207,47 @@ def test_invalid_hostlist_format(launcher): mpiSettings.launch_args.set_hostlist([5]) with pytest.raises(TypeError): mpiSettings.launch_args.set_hostlist(5) + + +@pytest.mark.parametrize( + "cls, cmd", + ( + pytest.param(MpiArgBuilder, "mpirun", id="w/ mpirun"), + pytest.param(MpiexecArgBuilder, "mpiexec", id="w/ mpiexec"), + pytest.param(OrteArgBuilder, "orterun", id="w/ orterun"), + ), +) +@pytest.mark.parametrize( + "args, expected", + ( + pytest.param({}, ("--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {"n": "1"}, + ("--n", "1", "--", "echo", "hello", "world"), + id="Short Arg", + ), + pytest.param( + {"host": "myhost"}, + ("--host", "myhost", "--", "echo", "hello", "world"), + id="Long Arg", + ), + pytest.param( + {"v": None}, + ("--v", "--", "echo", "hello", "world"), + id="Short Arg (No Value)", + ), + pytest.param( + {"verbose": None}, + ("--verbose", "--", "echo", "hello", "world"), + id="Long Arg (No Value)", + ), + pytest.param( + {"n": "1", "host": "myhost"}, + ("--n", "1", "--host", "myhost", "--", "echo", "hello", "world"), + id="Short and Long Args", + ), + ), +) +def test_formatting_launch_args(echo_executable_like, cls, cmd, args, expected): + fmt = cls(args).finalize(echo_executable_like, {}) + assert tuple(fmt) == (cmd,) + expected diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 01cbea2ed..a0bc7821c 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -4,6 +4,8 @@ from smartsim.settings.builders.launch.pals import PalsMpiexecArgBuilder from smartsim.settings.launchCommand import LauncherType +pytestmark = pytest.mark.group_a + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" @@ -67,3 +69,39 @@ def test_invalid_hostlist_format(): palsLauncher.launch_args.set_hostlist([5]) with pytest.raises(TypeError): palsLauncher.launch_args.set_hostlist(5) + + +@pytest.mark.parametrize( + "args, expected", + ( + pytest.param({}, ("mpiexec", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {"n": "1"}, + ("mpiexec", "--n", "1", "--", "echo", "hello", "world"), + id="Short Arg", + ), + pytest.param( + {"host": "myhost"}, + ("mpiexec", "--host", "myhost", "--", "echo", "hello", "world"), + id="Long Arg", + ), + pytest.param( + {"v": None}, + ("mpiexec", "--v", "--", "echo", "hello", "world"), + id="Short Arg (No Value)", + ), + pytest.param( + {"verbose": None}, + ("mpiexec", "--verbose", "--", "echo", "hello", "world"), + id="Long Arg (No Value)", + ), + pytest.param( + {"n": "1", "host": "myhost"}, + ("mpiexec", "--n", "1", "--host", "myhost", "--", "echo", "hello", "world"), + id="Short and Long Args", + ), + ), +) +def test_formatting_launch_args(echo_executable_like, args, expected): + cmd = PalsMpiexecArgBuilder(args).finalize(echo_executable_like, {}) + assert tuple(cmd) == expected From b96a1605aef7b591f918d5f2d2ed0372b4ff3e9a Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 18 Jun 2024 18:32:24 -0500 Subject: [PATCH 06/82] Impl aprun --- smartsim/settings/builders/launch/alps.py | 15 +++++++- .../test_settings/test_alpsLauncher.py | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index a527cafac..538ddb00e 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -34,10 +34,13 @@ from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder +if t.TYPE_CHECKING: + from smartsim.settings.builders.launchArgBuilder import ExecutableLike + logger = get_logger(__name__) -class AprunArgBuilder(LaunchArgBuilder): +class AprunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wdir"} @@ -213,3 +216,13 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> t.Sequence[str]: + return ( + "aprun", + *(self.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + ) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 7f9a4c3b9..7fa95cb6d 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -4,6 +4,8 @@ from smartsim.settings.builders.launch.alps import AprunArgBuilder from smartsim.settings.launchCommand import LauncherType +pytestmark = pytest.mark.group_a + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" @@ -147,3 +149,39 @@ def test_invalid_exclude_hostlist_format(): alpsLauncher.launch_args.set_excluded_hosts([5]) with pytest.raises(TypeError): alpsLauncher.launch_args.set_excluded_hosts(5) + + +@pytest.mark.parametrize( + "args, expected", + ( + pytest.param({}, ("aprun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {"N": "1"}, + ("aprun", "-N", "1", "--", "echo", "hello", "world"), + id="Short Arg", + ), + pytest.param( + {"cpus-per-pe": "1"}, + ("aprun", "--cpus-per-pe=1", "--", "echo", "hello", "world"), + id="Long Arg", + ), + pytest.param( + {"q": None}, + ("aprun", "-q", "--", "echo", "hello", "world"), + id="Short Arg (No Value)", + ), + pytest.param( + {"quiet": None}, + ("aprun", "--quiet", "--", "echo", "hello", "world"), + id="Long Arg (No Value)", + ), + pytest.param( + {"N": "1", "cpus-per-pe": "123"}, + ("aprun", "-N", "1", "--cpus-per-pe=123", "--", "echo", "hello", "world"), + id="Short and Long Args", + ), + ), +) +def test_formatting_launch_args(echo_executable_like, args, expected): + cmd = AprunArgBuilder(args).finalize(echo_executable_like, {}) + assert tuple(cmd) == expected From dc1cf0ba60d9054c2ccb7b79386d4fdf2b7572dc Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Thu, 20 Jun 2024 14:18:10 -0500 Subject: [PATCH 07/82] Impl dragon --- smartsim/settings/builders/launch/dragon.py | 30 ++++++++++++++- .../test_settings/test_dragonLauncher.py | 38 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 6043d6934..bc5f2b528 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -26,18 +26,23 @@ from __future__ import annotations +import os import typing as t +from smartsim._core.schemas.dragonRequests import DragonRunRequest from smartsim.log import get_logger from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder +if t.TYPE_CHECKING: + from smartsim.settings.builders.launchArgBuilder import ExecutableLike + logger = get_logger(__name__) -class DragonArgBuilder(LaunchArgBuilder[t.Any]): # TODO: come back and fix this +class DragonArgBuilder(LaunchArgBuilder[DragonRunRequest]): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Dragon.value @@ -54,7 +59,7 @@ def set_tasks_per_node(self, tasks_per_node: int) -> None: :param tasks_per_node: number of tasks per node """ - self.set("tasks-per-node", str(tasks_per_node)) + self.set("tasks_per_node", str(tasks_per_node)) def set(self, key: str, value: str | None) -> None: """Set the launch arguments""" @@ -62,3 +67,24 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + + def finalize( + self, exe: ExecutableLike, env: dict[str, str | None] + ) -> DragonRunRequest: + exe_, *args = exe.as_program_arguments() + return DragonRunRequest( + exe=exe_, + exe_args=args, + path=os.getcwd(), # FIXME: Currently this is hard coded because + # the schema requires it, but in future, + # it is almost certainly necessary that + # this will need to be injected by the + # user or by us to have the command + # execute next to any generated files. A + # similar problem exists for the other + # settings. + # TODO: Find a way to inject this path + env=env, + current_env=dict(os.environ), + **self._launch_args, + ) diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index d21a21c59..004090eef 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -1,9 +1,12 @@ import pytest +from smartsim._core.schemas.dragonRequests import DragonRunRequest from smartsim.settings import LaunchSettings from smartsim.settings.builders.launch.dragon import DragonArgBuilder from smartsim.settings.launchCommand import LauncherType +pytestmark = pytest.mark.group_a + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" @@ -16,7 +19,7 @@ def test_launcher_str(): [ pytest.param("set_nodes", (2,), "2", "nodes", id="set_nodes"), pytest.param( - "set_tasks_per_node", (2,), "2", "tasks-per-node", id="set_tasks_per_node" + "set_tasks_per_node", (2,), "2", "tasks_per_node", id="set_tasks_per_node" ), ], ) @@ -25,3 +28,36 @@ def test_dragon_class_methods(function, value, flag, result): assert isinstance(dragonLauncher._arg_builder, DragonArgBuilder) getattr(dragonLauncher.launch_args, function)(*value) assert dragonLauncher.launch_args._launch_args[flag] == result + + +NOT_SET = object() + + +@pytest.mark.parametrize("nodes", (NOT_SET, 20, 40)) +@pytest.mark.parametrize("tasks_per_node", (NOT_SET, 1, 20)) +def test_formatting_launch_args_into_request( + echo_executable_like, nodes, tasks_per_node +): + builder = DragonArgBuilder({}) + if nodes is not NOT_SET: + builder.set_nodes(nodes) + if tasks_per_node is not NOT_SET: + builder.set_tasks_per_node(tasks_per_node) + req = builder.finalize(echo_executable_like, {}) + + args = dict( + (k, v) + for k, v in ( + ("nodes", nodes), + ("tasks_per_node", tasks_per_node), + ) + if v is not NOT_SET + ) + expected = DragonRunRequest( + exe="echo", exe_args=["hello", "world"], path="/tmp", env={}, **args + ) + + assert req.nodes == expected.nodes + assert req.tasks_per_node == expected.tasks_per_node + assert req.hostlist == expected.hostlist + assert req.pmi_enabled == expected.pmi_enabled From e5553426e9084d1584e3d8a0d75ccb301736476e Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Thu, 20 Jun 2024 16:09:39 -0500 Subject: [PATCH 08/82] Type errors supressed for now --- smartsim/entity/entity.py | 3 +++ smartsim/settings/launchSettings.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/smartsim/entity/entity.py b/smartsim/entity/entity.py index 2f4b651f9..5304d2c29 100644 --- a/smartsim/entity/entity.py +++ b/smartsim/entity/entity.py @@ -26,6 +26,9 @@ import typing as t +if t.TYPE_CHECKING: + import smartsim.settings.base.RunSettings + class TelemetryConfiguration: """A base class for configuraing telemetry production behavior on diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index a9e5e8103..f512fcc34 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -67,8 +67,15 @@ def launcher(self) -> str: return self._launcher.value @property - def launch_args(self) -> LaunchArgBuilder: + def launch_args(self) -> LaunchArgBuilder[t.Any]: """Return the launch argument translator.""" + # FIXME: We _REALLY_ need to make the `LaunchSettings` class generic at + # on `_arg_builder` if we are expecting users to call specific + # `set_*` methods defined specificially on each of the + # subclasses. Otherwise we have no way of showing what methods + # are available at intellisense/static analysis/compile time. + # This whole object basically resolves to being one step removed + # from `Any` typed!! return self._arg_builder @launch_args.setter @@ -88,7 +95,7 @@ def env_vars(self, value: t.Dict[str, str]) -> None: """Set the environment variables.""" self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder: + def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder[t.Any]: """Map the Launcher to the LaunchArgBuilder""" if self._launcher == LauncherType.Slurm: return SlurmArgBuilder(launch_args) From 91611258ad9790269db0dadb2d69c4f93d03deda Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Sat, 22 Jun 2024 17:20:53 -0500 Subject: [PATCH 09/82] Add a dispatcher class to send built settings to a launcher --- smartsim/settings/builders/launch/alps.py | 2 + smartsim/settings/builders/launch/local.py | 2 + smartsim/settings/builders/launch/lsf.py | 2 + smartsim/settings/builders/launch/mpi.py | 4 + smartsim/settings/builders/launch/pals.py | 2 + smartsim/settings/builders/launch/slurm.py | 3 +- smartsim/settings/dispatch.py | 164 ++++++++++++++++++ tests/temp_tests/test_settings/conftest.py | 30 +++- .../temp_tests/test_settings/test_dispatch.py | 142 +++++++++++++++ 9 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 smartsim/settings/dispatch.py create mode 100644 tests/temp_tests/test_settings/test_dispatch.py diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index 538ddb00e..97e83a6c1 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -29,6 +29,7 @@ import typing as t from smartsim.log import get_logger +from smartsim.settings.dispatch import default_dispatcher, ShellLauncher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -40,6 +41,7 @@ logger = get_logger(__name__) +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class AprunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index 78e2dd28f..2cab0df75 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -29,6 +29,7 @@ import typing as t from smartsim.log import get_logger +from smartsim.settings.dispatch import default_dispatcher, ShellLauncher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -40,6 +41,7 @@ logger = get_logger(__name__) +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class LocalArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index 7fd3217ab..118946ca2 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -29,6 +29,7 @@ import typing as t from smartsim.log import get_logger +from smartsim.settings.dispatch import default_dispatcher, ShellLauncher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -40,6 +41,7 @@ logger = get_logger(__name__) +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class JsrunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index b012a4271..d2a1f3170 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -29,6 +29,7 @@ import typing as t from smartsim.log import get_logger +from smartsim.settings.dispatch import default_dispatcher, ShellLauncher from ...common import set_check_input from ...launchCommand import LauncherType @@ -217,6 +218,7 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class MpiArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" @@ -228,6 +230,7 @@ def finalize( return ("mpirun", *self.format_launch_args(), "--", *exe.as_program_arguments()) +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class MpiexecArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" @@ -244,6 +247,7 @@ def finalize( ) +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class OrteArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index af8cd7706..ff1755fb6 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -29,6 +29,7 @@ import typing as t from smartsim.log import get_logger +from smartsim.settings.dispatch import default_dispatcher, ShellLauncher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -40,6 +41,7 @@ logger = get_logger(__name__) +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class PalsMpiexecArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 3210c6665..1cdbea3f7 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -31,6 +31,7 @@ import typing as t from smartsim.log import get_logger +from smartsim.settings.dispatch import default_dispatcher, ShellLauncher from ...common import set_check_input from ...launchCommand import LauncherType @@ -41,7 +42,7 @@ logger = get_logger(__name__) - +@default_dispatcher.dispatch(to_launcher=ShellLauncher) class SlurmArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py new file mode 100644 index 000000000..27b247593 --- /dev/null +++ b/smartsim/settings/dispatch.py @@ -0,0 +1,164 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import annotations + +import subprocess as sp +import uuid +import typing as t + +from smartsim.error import errors + +if t.TYPE_CHECKING: + from typing_extensions import Self + from smartsim.experiment import Experiment + from smartsim.settings.builders import LaunchArgBuilder + +_T = t.TypeVar("_T") +_T_contra = t.TypeVar("_T_contra", contravariant=True) + +JobID = t.NewType("JobID", uuid.UUID) + + +def create_job_id() -> JobID: + return JobID(uuid.uuid4()) + + +class LauncherLike(t.Protocol[_T_contra]): + def start(self, launchable: _T_contra) -> JobID: ... + @classmethod + def create(cls, exp: Experiment) -> Self: ... + + +@t.final +class Dispatcher: + """A class capable of deciding which launcher type should be used to launch + a given settings builder type. + """ + + def __init__( + self, + *, + dispatch_registry: ( + t.Mapping[type[LaunchArgBuilder[t.Any]], type[LauncherLike[t.Any]]] | None + ) = None, + ) -> None: + self._dispatch_registry = ( + dict(dispatch_registry) if dispatch_registry is not None else {} + ) + + def copy(self) -> Self: + """Create a shallow copy of the Dispatcher""" + return type(self)(dispatch_registry=self._dispatch_registry) + + @t.overload + def dispatch( + self, + args: None = ..., + *, + to_launcher: type[LauncherLike[_T]], + allow_overwrite: bool = ..., + ) -> t.Callable[[type[LaunchArgBuilder[_T]]], type[LaunchArgBuilder[_T]]]: ... + @t.overload + def dispatch( + self, + args: type[LaunchArgBuilder[_T]], + *, + to_launcher: type[LauncherLike[_T]], + allow_overwrite: bool = ..., + ) -> None: ... + def dispatch( + self, + args: type[LaunchArgBuilder[_T]] | None = None, + *, + to_launcher: type[LauncherLike[_T]], + allow_overwrite: bool = False, + ) -> t.Callable[[type[LaunchArgBuilder[_T]]], type[LaunchArgBuilder[_T]]] | None: + """A type safe way to add a mapping of settings builder to launcher to + handle the settings at launch time. + """ + + def register( + args_: type[LaunchArgBuilder[_T]], / + ) -> type[LaunchArgBuilder[_T]]: + if args_ in self._dispatch_registry and not allow_overwrite: + launcher_type = self._dispatch_registry[args_] + raise TypeError( + f"{args_.__name__} has already been registered to be " + f"launched with {launcher_type}" + ) + self._dispatch_registry[args_] = to_launcher + return args_ + + if args is not None: + register(args) + return None + return register + + def get_launcher_for( + self, args: LaunchArgBuilder[_T] | type[LaunchArgBuilder[_T]], / + ) -> type[LauncherLike[_T]]: + """Find a type of launcher that is registered as being able to launch + the output of the provided builder + """ + if not isinstance(args, type): + args = type(args) + launcher_type = self._dispatch_registry.get(args, None) + if launcher_type is None: + raise TypeError( + f"{type(self).__name__} {self} has no launcher type to " + f"dispatch to for argument builder of type {args}" + ) + # Note the sleight-of-hand here: we are secretly casting a type of + # `LauncherLike[Any]` to `LauncherLike[_T]`. This is safe to do if all + # entries in the mapping were added using a type safe method (e.g. + # `Dispatcher.dispatch`), but if a user were to supply a custom + # dispatch registry or otherwise modify the registry THIS IS NOT + # NECESSARILY 100% TYPE SAFE!! + return launcher_type + + +default_dispatcher: t.Final = Dispatcher() + + +class ShellLauncher: + """Mock launcher for launching/tracking simple shell commands + + TODO: this is probably all we need for a "local" launcher, but probably + best to move this to a `smartsim._core.launcher` module/submodule + """ + + def __init__(self) -> None: + self._launched: dict[JobID, sp.Popen[bytes]] = {} + + def start(self, launchable: t.Sequence[str]) -> JobID: + id_ = create_job_id() + self._launched[id_] = sp.Popen(launchable) + return id_ + + @classmethod + def create(cls, exp: Experiment) -> Self: + return cls() diff --git a/tests/temp_tests/test_settings/conftest.py b/tests/temp_tests/test_settings/conftest.py index ad1fa0f4a..334431fce 100644 --- a/tests/temp_tests/test_settings/conftest.py +++ b/tests/temp_tests/test_settings/conftest.py @@ -25,8 +25,10 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pytest +from unittest.mock import Mock from smartsim.settings.builders import launchArgBuilder as launch +from smartsim.settings import dispatch @pytest.fixture @@ -35,4 +37,30 @@ class _ExeLike(launch.ExecutableLike): def as_program_arguments(self): return ("echo", "hello", "world") - return _ExeLike() + yield _ExeLike() + + +@pytest.fixture +def settings_builder(): + class _SettingsBuilder(launch.LaunchArgBuilder): + def launcher_str(self): + return "Mock Settings Builder" + + def set(self, arg, val): ... + def finalize(self, exe, env): + return Mock() + + yield _SettingsBuilder({}) + + +@pytest.fixture +def launcher_like(): + class _LuancherLike(dispatch.LauncherLike): + def start(self, launchable): + return dispatch.create_job_id() + + @classmethod + def create(cls, exp): + return cls() + + yield _LuancherLike() diff --git a/tests/temp_tests/test_settings/test_dispatch.py b/tests/temp_tests/test_settings/test_dispatch.py new file mode 100644 index 000000000..4dbd25c3f --- /dev/null +++ b/tests/temp_tests/test_settings/test_dispatch.py @@ -0,0 +1,142 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest +from smartsim.settings import dispatch + +import contextlib + + +def test_declaritive_form_dispatch_declaration(launcher_like, settings_builder): + d = dispatch.Dispatcher() + assert type(settings_builder) == d.dispatch(to_launcher=type(launcher_like))( + type(settings_builder) + ) + assert d._dispatch_registry == {type(settings_builder): type(launcher_like)} + + +def test_imperative_form_dispatch_declaration(launcher_like, settings_builder): + d = dispatch.Dispatcher() + assert None == d.dispatch(type(settings_builder), to_launcher=type(launcher_like)) + assert d._dispatch_registry == {type(settings_builder): type(launcher_like)} + + +def test_dispatchers_from_same_registry_do_not_cross_polute( + launcher_like, settings_builder +): + some_starting_registry = {} + d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) + d2 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) + assert ( + d1._dispatch_registry == d2._dispatch_registry == some_starting_registry == {} + ) + assert ( + d1._dispatch_registry is not d2._dispatch_registry is not some_starting_registry + ) + + d2.dispatch(type(settings_builder), to_launcher=type(launcher_like)) + assert d1._dispatch_registry == {} + assert d2._dispatch_registry == {type(settings_builder): type(launcher_like)} + + +def test_copied_dispatchers_do_not_cross_pollute(launcher_like, settings_builder): + some_starting_registry = {} + d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) + d2 = d1.copy() + assert ( + d1._dispatch_registry == d2._dispatch_registry == some_starting_registry == {} + ) + assert ( + d1._dispatch_registry is not d2._dispatch_registry is not some_starting_registry + ) + + d2.dispatch(type(settings_builder), to_launcher=type(launcher_like)) + assert d1._dispatch_registry == {} + assert d2._dispatch_registry == {type(settings_builder): type(launcher_like)} + + +@pytest.mark.parametrize( + "add_dispatch, expected_ctx", + ( + pytest.param( + lambda d, s, l: d.dispatch(s, to_launcher=l), + pytest.raises(TypeError, match="has already been registered"), + id="Imperative -- Disallowed implicitly", + ), + pytest.param( + lambda d, s, l: d.dispatch(s, to_launcher=l, allow_overwrite=True), + contextlib.nullcontext(), + id="Imperative -- Allowed with flag", + ), + pytest.param( + lambda d, s, l: d.dispatch(to_launcher=l)(s), + pytest.raises(TypeError, match="has already been registered"), + id="Declarative -- Disallowed implicitly", + ), + pytest.param( + lambda d, s, l: d.dispatch(to_launcher=l, allow_overwrite=True)(s), + contextlib.nullcontext(), + id="Declarative -- Allowed with flag", + ), + ), +) +def test_dispatch_overwriting( + add_dispatch, expected_ctx, launcher_like, settings_builder +): + registry = {type(settings_builder): type(launcher_like)} + d = dispatch.Dispatcher(dispatch_registry=registry) + with expected_ctx: + add_dispatch(d, type(settings_builder), type(launcher_like)) + + +@pytest.mark.parametrize( + "map_settings", + ( + pytest.param(type, id="From settings type"), + pytest.param(lambda s: s, id="From settings instance"), + ), +) +def test_dispatch_can_retrieve_launcher_to_dispatch_to( + map_settings, launcher_like, settings_builder +): + registry = {type(settings_builder): type(launcher_like)} + d = dispatch.Dispatcher(dispatch_registry=registry) + assert type(launcher_like) == d.get_launcher_for(map_settings(settings_builder)) + + +@pytest.mark.parametrize( + "map_settings", + ( + pytest.param(type, id="From settings type"), + pytest.param(lambda s: s, id="From settings instance"), + ), +) +def test_dispatch_raises_if_settings_type_not_registered( + map_settings, launcher_like, settings_builder +): + d = dispatch.Dispatcher(dispatch_registry={}) + with pytest.raises(TypeError, match="no launcher type to dispatch to"): + d.get_launcher_for(map_settings(settings_builder)) From d765718ea5c491581dc49bf7d49cda860d5efac3 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Mon, 24 Jun 2024 12:11:23 -0500 Subject: [PATCH 10/82] Isort/Black --- smartsim/settings/builders/launch/alps.py | 2 +- smartsim/settings/builders/launch/dragon.py | 19 ++++++++++--------- smartsim/settings/builders/launch/local.py | 2 +- smartsim/settings/builders/launch/lsf.py | 2 +- smartsim/settings/builders/launch/mpi.py | 2 +- smartsim/settings/builders/launch/pals.py | 2 +- smartsim/settings/builders/launch/slurm.py | 3 ++- smartsim/settings/dispatch.py | 5 ++--- smartsim/settings/launchSettings.py | 4 +++- tests/temp_tests/test_settings/conftest.py | 5 +++-- .../temp_tests/test_settings/test_dispatch.py | 5 ++++- 11 files changed, 29 insertions(+), 22 deletions(-) diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index 97e83a6c1..8f425dacc 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import default_dispatcher, ShellLauncher +from smartsim.settings.dispatch import ShellLauncher, default_dispatcher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index bc5f2b528..1c8f1ac78 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -75,15 +75,16 @@ def finalize( return DragonRunRequest( exe=exe_, exe_args=args, - path=os.getcwd(), # FIXME: Currently this is hard coded because - # the schema requires it, but in future, - # it is almost certainly necessary that - # this will need to be injected by the - # user or by us to have the command - # execute next to any generated files. A - # similar problem exists for the other - # settings. - # TODO: Find a way to inject this path + # FIXME: Currently this is hard coded because + # the schema requires it, but in future, + # it is almost certainly necessary that + # this will need to be injected by the + # user or by us to have the command + # execute next to any generated files. A + # similar problem exists for the other + # settings. + # TODO: Find a way to inject this path + path=os.getcwd(), env=env, current_env=dict(os.environ), **self._launch_args, diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index 2cab0df75..23c5d75f0 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import default_dispatcher, ShellLauncher +from smartsim.settings.dispatch import ShellLauncher, default_dispatcher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index 118946ca2..e7b22cb47 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import default_dispatcher, ShellLauncher +from smartsim.settings.dispatch import ShellLauncher, default_dispatcher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index d2a1f3170..6bcde18da 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import default_dispatcher, ShellLauncher +from smartsim.settings.dispatch import ShellLauncher, default_dispatcher from ...common import set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index ff1755fb6..c8bdf2432 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import default_dispatcher, ShellLauncher +from smartsim.settings.dispatch import ShellLauncher, default_dispatcher from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 1cdbea3f7..11e9a7b15 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -31,7 +31,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import default_dispatcher, ShellLauncher +from smartsim.settings.dispatch import ShellLauncher, default_dispatcher from ...common import set_check_input from ...launchCommand import LauncherType @@ -42,6 +42,7 @@ logger = get_logger(__name__) + @default_dispatcher.dispatch(to_launcher=ShellLauncher) class SlurmArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 27b247593..db2094030 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -27,13 +27,12 @@ from __future__ import annotations import subprocess as sp -import uuid import typing as t - -from smartsim.error import errors +import uuid if t.TYPE_CHECKING: from typing_extensions import Self + from smartsim.experiment import Experiment from smartsim.settings.builders import LaunchArgBuilder diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index f512fcc34..17dbfa7a2 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -95,7 +95,9 @@ def env_vars(self, value: t.Dict[str, str]) -> None: """Set the environment variables.""" self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder[t.Any]: + def _get_arg_builder( + self, launch_args: StringArgument | None + ) -> LaunchArgBuilder[t.Any]: """Map the Launcher to the LaunchArgBuilder""" if self._launcher == LauncherType.Slurm: return SlurmArgBuilder(launch_args) diff --git a/tests/temp_tests/test_settings/conftest.py b/tests/temp_tests/test_settings/conftest.py index 334431fce..ebf361e97 100644 --- a/tests/temp_tests/test_settings/conftest.py +++ b/tests/temp_tests/test_settings/conftest.py @@ -24,11 +24,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest from unittest.mock import Mock -from smartsim.settings.builders import launchArgBuilder as launch +import pytest + from smartsim.settings import dispatch +from smartsim.settings.builders import launchArgBuilder as launch @pytest.fixture diff --git a/tests/temp_tests/test_settings/test_dispatch.py b/tests/temp_tests/test_settings/test_dispatch.py index 4dbd25c3f..ccd1e81cd 100644 --- a/tests/temp_tests/test_settings/test_dispatch.py +++ b/tests/temp_tests/test_settings/test_dispatch.py @@ -24,10 +24,13 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import contextlib + import pytest + from smartsim.settings import dispatch -import contextlib +pytestmark = pytest.mark.group_a def test_declaritive_form_dispatch_declaration(launcher_like, settings_builder): From 01393c7602688e0a99993563127064a4959674fa Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 26 Jun 2024 16:38:27 -0500 Subject: [PATCH 11/82] Env: dict -> Mapping --- smartsim/settings/builders/launch/alps.py | 2 +- smartsim/settings/builders/launch/dragon.py | 2 +- smartsim/settings/builders/launch/local.py | 2 +- smartsim/settings/builders/launch/lsf.py | 2 +- smartsim/settings/builders/launch/mpi.py | 6 +++--- smartsim/settings/builders/launch/pals.py | 2 +- smartsim/settings/builders/launch/slurm.py | 2 +- smartsim/settings/builders/launchArgBuilder.py | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index 8f425dacc..f1a196e7c 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -220,7 +220,7 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return ( "aprun", diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 1c8f1ac78..0d0062bf2 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -69,7 +69,7 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> DragonRunRequest: exe_, *args = exe.as_program_arguments() return DragonRunRequest( diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index 23c5d75f0..64770f696 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -79,6 +79,6 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return exe.as_program_arguments() diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index e7b22cb47..e1a03ef3b 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -122,7 +122,7 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return ( "jsrun", diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 6bcde18da..6eac12a24 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -225,7 +225,7 @@ def launcher_str(self) -> str: return LauncherType.Mpirun.value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return ("mpirun", *self.format_launch_args(), "--", *exe.as_program_arguments()) @@ -237,7 +237,7 @@ def launcher_str(self) -> str: return LauncherType.Mpiexec.value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return ( "mpiexec", @@ -254,7 +254,7 @@ def launcher_str(self) -> str: return LauncherType.Orterun.value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return ( "orterun", diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index c8bdf2432..d21edc8bd 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -156,7 +156,7 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return ( "mpiexec", diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 11e9a7b15..1125c2611 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -322,7 +322,7 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: dict[str, str | None] + self, exe: ExecutableLike, env: t.Mapping[str, str | None] ) -> t.Sequence[str]: return ( "srun", diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py index 8a839a5c8..b125046cd 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/builders/launchArgBuilder.py @@ -59,7 +59,7 @@ def set(self, arg: str, val: str | None) -> None: """Set the launch arguments""" @abstractmethod - def finalize(self, exe: ExecutableLike, env: dict[str, str | None]) -> _T: + def finalize(self, exe: ExecutableLike, env: t.Mapping[str, str | None]) -> _T: """Prepare an entity for launch using the built options""" def format_launch_args(self) -> t.Union[t.List[str], None]: From e504ccfcee2e2fcb7e21c365d8d2c0c631631682 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 26 Jun 2024 16:44:19 -0500 Subject: [PATCH 12/82] Organize dispatch file, call out work to do --- smartsim/settings/dispatch.py | 36 +++++++++++++++++++++-------------- smartsim/types.py | 32 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 smartsim/types.py diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index db2094030..013ae6319 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -30,6 +30,8 @@ import typing as t import uuid +from smartsim.types import LaunchedJobID + if t.TYPE_CHECKING: from typing_extensions import Self @@ -39,18 +41,6 @@ _T = t.TypeVar("_T") _T_contra = t.TypeVar("_T_contra", contravariant=True) -JobID = t.NewType("JobID", uuid.UUID) - - -def create_job_id() -> JobID: - return JobID(uuid.uuid4()) - - -class LauncherLike(t.Protocol[_T_contra]): - def start(self, launchable: _T_contra) -> JobID: ... - @classmethod - def create(cls, exp: Experiment) -> Self: ... - @t.final class Dispatcher: @@ -143,6 +133,21 @@ def get_launcher_for( default_dispatcher: t.Final = Dispatcher() +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# TODO: move these to a common module under `smartsim._core.launcher` +# ----------------------------------------------------------------------------- + + +def create_job_id() -> LaunchedJobID: + return LaunchedJobID(uuid.uuid4()) + + +class LauncherLike(t.Protocol[_T_contra]): + def start(self, launchable: _T_contra) -> LaunchedJobID: ... + @classmethod + def create(cls, exp: Experiment) -> Self: ... + + class ShellLauncher: """Mock launcher for launching/tracking simple shell commands @@ -151,9 +156,9 @@ class ShellLauncher: """ def __init__(self) -> None: - self._launched: dict[JobID, sp.Popen[bytes]] = {} + self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - def start(self, launchable: t.Sequence[str]) -> JobID: + def start(self, launchable: t.Sequence[str]) -> LaunchedJobID: id_ = create_job_id() self._launched[id_] = sp.Popen(launchable) return id_ @@ -161,3 +166,6 @@ def start(self, launchable: t.Sequence[str]) -> JobID: @classmethod def create(cls, exp: Experiment) -> Self: return cls() + + +# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/smartsim/types.py b/smartsim/types.py new file mode 100644 index 000000000..84eb31a85 --- /dev/null +++ b/smartsim/types.py @@ -0,0 +1,32 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import annotations + +import typing as t +import uuid + +LaunchedJobID = t.NewType("LaunchedJobID", uuid.UUID) From 910b2d903f3c7d0b6f5a9ab7591f5a10dee5ea5d Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 26 Jun 2024 17:30:52 -0500 Subject: [PATCH 13/82] Wire up dispatch dragon builder to dragon launcher --- smartsim/_core/config/config.py | 5 +- .../_core/launcher/dragon/dragonConnector.py | 25 +++-- .../_core/launcher/dragon/dragonLauncher.py | 66 +++++++++---- smartsim/_core/utils/helpers.py | 8 +- smartsim/experiment.py | 95 ++++++++++++------- smartsim/settings/__init__.py | 2 + smartsim/settings/builders/launch/dragon.py | 26 ++--- smartsim/settings/dispatch.py | 6 +- smartsim/types.py | 3 +- 9 files changed, 155 insertions(+), 81 deletions(-) diff --git a/smartsim/_core/config/config.py b/smartsim/_core/config/config.py index 374457f3a..1012129e9 100644 --- a/smartsim/_core/config/config.py +++ b/smartsim/_core/config/config.py @@ -161,10 +161,7 @@ def dragon_dotenv(self) -> Path: @property def dragon_server_path(self) -> t.Optional[str]: - return os.getenv( - "SMARTSIM_DRAGON_SERVER_PATH", - os.getenv("SMARTSIM_DRAGON_SERVER_PATH_EXP", None), - ) + return os.getenv("SMARTSIM_DRAGON_SERVER_PATH", None) @property def dragon_server_timeout(self) -> int: diff --git a/smartsim/_core/launcher/dragon/dragonConnector.py b/smartsim/_core/launcher/dragon/dragonConnector.py index 0cd68c24e..6ce0e257f 100644 --- a/smartsim/_core/launcher/dragon/dragonConnector.py +++ b/smartsim/_core/launcher/dragon/dragonConnector.py @@ -57,6 +57,11 @@ ) from ...utils.network import find_free_port, get_best_interface_and_address +if t.TYPE_CHECKING: + from typing_extensions import Self + + from smartsim.experiment import Experiment + logger = get_logger(__name__) _SchemaT = t.TypeVar("_SchemaT", bound=t.Union[DragonRequest, DragonResponse]) @@ -69,21 +74,29 @@ class DragonConnector: to start a Dragon server and communicate with it. """ - def __init__(self) -> None: + def __init__(self, path: str | os.PathLike[str]) -> None: self._context: zmq.Context[t.Any] = zmq.Context.instance() self._context.setsockopt(zmq.REQ_CORRELATE, 1) self._context.setsockopt(zmq.REQ_RELAXED, 1) self._authenticator: t.Optional[zmq.auth.thread.ThreadAuthenticator] = None config = get_config() self._reset_timeout(config.dragon_server_timeout) + + # TODO: We should be able to make these "non-optional" + # by simply moving the impl of + # `DragonConnectior.connect_to_dragon` to this method. This is + # fine as we expect the that method should only be called once + # without hitting a guard clause. self._dragon_head_socket: t.Optional[zmq.Socket[t.Any]] = None self._dragon_head_process: t.Optional[subprocess.Popen[bytes]] = None # Returned by dragon head, useful if shutdown is to be requested # but process was started by another connector self._dragon_head_pid: t.Optional[int] = None - self._dragon_server_path = config.dragon_server_path + self._dragon_server_path = _resolve_dragon_path(path) logger.debug(f"Dragon Server path was set to {self._dragon_server_path}") self._env_vars: t.Dict[str, str] = {} + + # TODO: Remove! in theory this is unreachable if self._dragon_server_path is None: raise SmartSimError( "DragonConnector could not find the dragon server path. " @@ -293,8 +306,7 @@ def connect_to_dragon(self) -> None: "Establishing connection with Dragon server or starting a new one..." ) - path = _resolve_dragon_path(self._dragon_server_path) - + path = self._dragon_server_path self._connect_to_existing_server(path) if self.is_connected: return @@ -520,8 +532,9 @@ def _dragon_cleanup( def _resolve_dragon_path(fallback: t.Union[str, "os.PathLike[str]"]) -> Path: - dragon_server_path = get_config().dragon_server_path or os.path.join( - fallback, ".smartsim", "dragon" + config = get_config() + dragon_server_path = config.dragon_server_path or os.path.join( + fallback, config.dragon_default_subdir ) dragon_server_paths = dragon_server_path.split(":") if len(dragon_server_paths) > 1: diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 17b47e309..829828224 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -29,6 +29,8 @@ import os import typing as t +from smartsim.types import LaunchedJobID + from ...._core.launcher.stepMapping import StepMap from ....error import LauncherError, SmartSimError from ....log import get_logger @@ -42,6 +44,7 @@ from ....status import SmartSimStatus from ...schemas import ( DragonRunRequest, + DragonRunRequestView, DragonRunResponse, DragonStopRequest, DragonStopResponse, @@ -55,6 +58,11 @@ from ..stepInfo import StepInfo from .dragonConnector import DragonConnector, _SchemaT +if t.TYPE_CHECKING: + from typing_extensions import Self + + from smartsim.experiment import Experiment + logger = get_logger(__name__) @@ -72,9 +80,9 @@ class DragonLauncher(WLMLauncher): the Job Manager to interact with it. """ - def __init__(self) -> None: + def __init__(self, server_path: str | os.PathLike[str]) -> None: super().__init__() - self._connector = DragonConnector() + self._connector = DragonConnector(server_path) """Connector used to start and interact with the Dragon server""" self._slurm_launcher = SlurmLauncher() """Slurm sub-launcher, used only for batch jobs""" @@ -119,6 +127,19 @@ def add_step_to_mapping_table(self, name: str, step_map: StepMap) -> None: ) sublauncher.add_step_to_mapping_table(name, sublauncher_step_map) + @classmethod + def create(cls, exp: Experiment) -> Self: + self = cls(exp.exp_path) + self._connector.connect_to_dragon() # TODO: protected access + return self + + def start(self, req_args: DragonRunRequestView) -> LaunchedJobID: + self._connector.load_persisted_env() + merged_env = self._connector.merge_persisted_env(os.environ.copy()) + req = DragonRunRequest(**dict(req_args), current_env=merged_env) + res = _assert_schema_type(self._connector.send_request(req), DragonRunResponse) + return LaunchedJobID(res.step_id) + def run(self, step: Step) -> t.Optional[str]: """Run a job step through Slurm @@ -165,27 +186,21 @@ def run(self, step: Step) -> t.Optional[str]: run_args = step.run_settings.run_args req_env = step.run_settings.env_vars self._connector.load_persisted_env() - merged_env = self._connector.merge_persisted_env(os.environ.copy()) nodes = int(run_args.get("nodes", None) or 1) tasks_per_node = int(run_args.get("tasks-per-node", None) or 1) - response = _assert_schema_type( - self._connector.send_request( - DragonRunRequest( - exe=cmd[0], - exe_args=cmd[1:], - path=step.cwd, - name=step.name, - nodes=nodes, - tasks_per_node=tasks_per_node, - env=req_env, - current_env=merged_env, - output_file=out, - error_file=err, - ) - ), - DragonRunResponse, + step_id = self.start( + DragonRunRequestView( + exe=cmd[0], + exe_args=cmd[1:], + path=step.cwd, + name=step.name, + nodes=nodes, + tasks_per_node=tasks_per_node, + env=req_env, + output_file=out, + error_file=err, + ) ) - step_id = str(response.step_id) else: # pylint: disable-next=consider-using-with out_strm = open(out, "w+", encoding="utf-8") @@ -319,3 +334,14 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: if not isinstance(obj, typ): raise TypeError(f"Expected schema of type `{typ}`, but got {type(obj)}") return obj + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# TODO: Remove this registry and move back to builder file after fixing +# circular import +# ----------------------------------------------------------------------------- +from smartsim.settings.dispatch import default_dispatcher +from smartsim.settings.builders.launch.dragon import DragonArgBuilder + +default_dispatcher.dispatch(DragonArgBuilder, to_launcher=DragonLauncher) +# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index 70f52bc4e..646b086dd 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -27,6 +27,8 @@ """ A file of helper functions for SmartSim """ +from __future__ import annotations + import base64 import collections.abc import os @@ -45,6 +47,7 @@ from types import FrameType +_T = t.TypeVar("_T") _TSignalHandlerFn = t.Callable[[int, t.Optional["FrameType"]], object] @@ -122,7 +125,6 @@ def expand_exe_path(exe: str) -> str: # which returns none if not found in_path = which(exe) - print(f"hmm what is this: {in_path}") if not in_path: if os.path.isfile(exe) and os.access(exe, os.X_OK): return os.path.abspath(exe) @@ -412,6 +414,10 @@ def is_crayex_platform() -> bool: return result.is_cray +def first(predicate: t.Callable[[_T], bool], iterable: t.Iterable[_T]) -> _T | None: + return next((item for item in iterable if predicate(item)), None) + + @t.final class SignalInterceptionStack(collections.abc.Collection[_TSignalHandlerFn]): """Registers a stack of callables to be called when a signal is diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 5ffc6102e..07095461a 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -26,15 +26,21 @@ # pylint: disable=too-many-lines +from __future__ import annotations + +import itertools import os import os.path as osp +import textwrap import typing as t from os import environ, getcwd from tabulate import tabulate from smartsim._core.config import CONFIG +from smartsim._core.utils.helpers import first from smartsim.error.errors import SSUnsupportedError +from smartsim.settings.dispatch import default_dispatcher from smartsim.status import SmartSimStatus from ._core import Controller, Generator, Manifest, previewrenderer @@ -49,7 +55,12 @@ from .error import SmartSimError from .log import ctx_exp_path, get_logger, method_contextualizer from .settings import BatchSettings, Container, RunSettings -from .wlm import detect_launcher + +if t.TYPE_CHECKING: + from smartsim.launchable.job import Job + from smartsim.settings.builders import LaunchArgBuilder + from smartsim.settings.dispatch import Dispatcher, LauncherLike + from smartsim.types import LaunchedJobID logger = get_logger(__name__) @@ -101,8 +112,9 @@ class Experiment: def __init__( self, name: str, - exp_path: t.Optional[str] = None, - launcher: str = "local", + exp_path: str | None = None, + *, + settings_dispatcher: Dispatcher = default_dispatcher, ): """Initialize an Experiment instance. @@ -110,7 +122,7 @@ def __init__( local launcher, which will start all Experiment created instances on the localhost. - Example of initializing an Experiment with the local launcher + Example of initializing an Experiment .. highlight:: python .. code-block:: python @@ -143,10 +155,6 @@ def __init__( :param name: name for the ``Experiment`` :param exp_path: path to location of ``Experiment`` directory - :param launcher: type of launcher being used, options are "slurm", "pbs", - "lsf", "sge", or "local". If set to "auto", - an attempt will be made to find an available launcher - on the system. """ self.name = name if exp_path: @@ -160,28 +168,45 @@ def __init__( self.exp_path = exp_path - self._launcher = launcher.lower() - - if self._launcher == "auto": - self._launcher = detect_launcher() - if self._launcher == "cobalt": - raise SSUnsupportedError("Cobalt launcher is no longer supported.") - - if launcher == "dragon": - self._set_dragon_server_path() - - self._control = Controller(launcher=self._launcher) + # TODO: Remove this! The contoller is becoming obsolete + self._control = Controller(launcher="local") + self._dispatcher = settings_dispatcher + self._active_launchers: set[LauncherLike[t.Any]] = set() self.fs_identifiers: t.Set[str] = set() self._telemetry_cfg = ExperimentTelemetryConfiguration() - def _set_dragon_server_path(self) -> None: - """Set path for dragon server through environment varialbes""" - if not "SMARTSIM_DRAGON_SERVER_PATH" in environ: - environ["SMARTSIM_DRAGON_SERVER_PATH_EXP"] = osp.join( - self.exp_path, CONFIG.dragon_default_subdir + def start_jobs(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: + """WIP: replacemnt method to launch jobs using the new API""" + + if not jobs: + raise TypeError( + f"{type(self).__name__}.start_jobs() missing at least 1 required " + "positional argument" ) + def _start(job: Job) -> LaunchedJobID: + builder = job.launch_settings.launch_args + launcher_type = self._dispatcher.get_launcher_for(builder) + launcher = first( + lambda launcher: type(launcher) is launcher_type, + self._active_launchers, + ) + if launcher is None: + launcher = launcher_type.create(self) + self._active_launchers.add(launcher) + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # FIXME: Opting out of type check here. Fix this later!! + # TODO: Very much dislike that we have to pass in attrs off of `job` + # into `builder`, which is itself an attr of an attr of `job`. + # Why is `Job` not generic based on launch arg builder? + # --------------------------------------------------------------------- + finalized = builder.finalize(job.entity, job.launch_settings.env_vars) + # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + return launcher.start(finalized) + + return tuple(map(_start, jobs)) + @_contextualize def start( self, @@ -477,7 +502,7 @@ def preview( """ # Retrieve any active feature store jobs - active_fsjobs = self._control.active_active_feature_store_jobs + active_fsjobs = self._control.active_feature_store_jobs preview_manifest = Manifest(*args) @@ -490,10 +515,6 @@ def preview( active_fsjobs, ) - @property - def launcher(self) -> str: - return self._launcher - @_contextualize def summary(self, style: str = "github") -> str: """Return a summary of the ``Experiment`` @@ -551,11 +572,19 @@ def _launch_summary(self, manifest: Manifest) -> None: :param manifest: Manifest of deployables. """ + launcher_list = "\n".join(str(launcher) for launcher in self._active_launchers) + # ^^^^^^^^^^^^^ + # TODO: make this a nicer string + summary = textwrap.dedent(f"""\ + + + === Launch Summary === + Experiment: {self.name} + Experiment Path: {self.exp_path} + Launchers: + {textwrap.indent(" - ", launcher_list)} + """) - summary = "\n\n=== Launch Summary ===\n" - summary += f"Experiment: {self.name}\n" - summary += f"Experiment Path: {self.exp_path}\n" - summary += f"Launcher: {self._launcher}\n" if manifest.applications: summary += f"Applications: {len(manifest.applications)}\n" diff --git a/smartsim/settings/__init__.py b/smartsim/settings/__init__.py index e0313f341..e904240af 100644 --- a/smartsim/settings/__init__.py +++ b/smartsim/settings/__init__.py @@ -24,6 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import typing as t + from .baseSettings import BaseSettings from .batchSettings import BatchSettings from .launchSettings import LaunchSettings diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 0d0062bf2..8fb2ebc48 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -29,7 +29,7 @@ import os import typing as t -from smartsim._core.schemas.dragonRequests import DragonRunRequest +from smartsim._core.schemas.dragonRequests import DragonRunRequestView from smartsim.log import get_logger from ...common import StringArgument, set_check_input @@ -42,7 +42,7 @@ logger = get_logger(__name__) -class DragonArgBuilder(LaunchArgBuilder[DragonRunRequest]): +class DragonArgBuilder(LaunchArgBuilder[DragonRunRequestView]): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Dragon.value @@ -70,22 +70,22 @@ def set(self, key: str, value: str | None) -> None: def finalize( self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> DragonRunRequest: + ) -> DragonRunRequestView: exe_, *args = exe.as_program_arguments() - return DragonRunRequest( + return DragonRunRequestView( exe=exe_, exe_args=args, - # FIXME: Currently this is hard coded because - # the schema requires it, but in future, - # it is almost certainly necessary that - # this will need to be injected by the - # user or by us to have the command - # execute next to any generated files. A - # similar problem exists for the other - # settings. + # FIXME: Currently this is hard coded because the schema requires + # it, but in future, it is almost certainly necessary that + # this will need to be injected by the user or by us to have + # the command execute next to any generated files. A similar + # problem exists for the other settings. # TODO: Find a way to inject this path path=os.getcwd(), env=env, - current_env=dict(os.environ), + # TODO: Not sure how this info is injected + name=None, + output_file=None, + error_file=None, **self._launch_args, ) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 013ae6319..c731d0940 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -30,6 +30,7 @@ import typing as t import uuid +from smartsim._core.utils import helpers from smartsim.types import LaunchedJobID if t.TYPE_CHECKING: @@ -139,7 +140,7 @@ def get_launcher_for( def create_job_id() -> LaunchedJobID: - return LaunchedJobID(uuid.uuid4()) + return LaunchedJobID(str(uuid.uuid4())) class LauncherLike(t.Protocol[_T_contra]): @@ -160,7 +161,8 @@ def __init__(self) -> None: def start(self, launchable: t.Sequence[str]) -> LaunchedJobID: id_ = create_job_id() - self._launched[id_] = sp.Popen(launchable) + exe, *rest = launchable + self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest)) return id_ @classmethod diff --git a/smartsim/types.py b/smartsim/types.py index 84eb31a85..e806d8130 100644 --- a/smartsim/types.py +++ b/smartsim/types.py @@ -27,6 +27,5 @@ from __future__ import annotations import typing as t -import uuid -LaunchedJobID = t.NewType("LaunchedJobID", uuid.UUID) +LaunchedJobID = t.NewType("LaunchedJobID", str) From f4ebada3981bdc6cc1fbcab588f5da483cb893e4 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 2 Jul 2024 12:13:19 -0500 Subject: [PATCH 14/82] Import sort --- smartsim/_core/launcher/dragon/dragonLauncher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 829828224..020f7682e 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -340,8 +340,8 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: # TODO: Remove this registry and move back to builder file after fixing # circular import # ----------------------------------------------------------------------------- -from smartsim.settings.dispatch import default_dispatcher from smartsim.settings.builders.launch.dragon import DragonArgBuilder +from smartsim.settings.dispatch import default_dispatcher default_dispatcher.dispatch(DragonArgBuilder, to_launcher=DragonLauncher) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< From 9d2901f1e100f36e57a88b2f75be70558442fe46 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 2 Jul 2024 14:14:40 -0500 Subject: [PATCH 15/82] Remove old TODOs --- smartsim/_core/launcher/dragon/dragonConnector.py | 10 ---------- smartsim/_core/launcher/dragon/dragonLauncher.py | 2 +- smartsim/experiment.py | 2 -- smartsim/settings/dispatch.py | 6 +----- smartsim/settings/launchSettings.py | 2 +- 5 files changed, 3 insertions(+), 19 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonConnector.py b/smartsim/_core/launcher/dragon/dragonConnector.py index 6ce0e257f..ca721eeaa 100644 --- a/smartsim/_core/launcher/dragon/dragonConnector.py +++ b/smartsim/_core/launcher/dragon/dragonConnector.py @@ -96,16 +96,6 @@ def __init__(self, path: str | os.PathLike[str]) -> None: logger.debug(f"Dragon Server path was set to {self._dragon_server_path}") self._env_vars: t.Dict[str, str] = {} - # TODO: Remove! in theory this is unreachable - if self._dragon_server_path is None: - raise SmartSimError( - "DragonConnector could not find the dragon server path. " - "This should not happen if the Connector was started by an " - "experiment.\nIf the DragonConnector was started manually, " - "then the environment variable SMARTSIM_DRAGON_SERVER_PATH " - "should be set to an existing directory." - ) - @property def is_connected(self) -> bool: """Whether the Connector established a connection to the server diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 020f7682e..ce11ed91f 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -130,7 +130,7 @@ def add_step_to_mapping_table(self, name: str, step_map: StepMap) -> None: @classmethod def create(cls, exp: Experiment) -> Self: self = cls(exp.exp_path) - self._connector.connect_to_dragon() # TODO: protected access + self._connector.connect_to_dragon() # pylint: disable=protected-access return self def start(self, req_args: DragonRunRequestView) -> LaunchedJobID: diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 07095461a..182d4663a 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -573,8 +573,6 @@ def _launch_summary(self, manifest: Manifest) -> None: :param manifest: Manifest of deployables. """ launcher_list = "\n".join(str(launcher) for launcher in self._active_launchers) - # ^^^^^^^^^^^^^ - # TODO: make this a nicer string summary = textwrap.dedent(f"""\ diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index c731d0940..9928bad91 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -150,11 +150,7 @@ def create(cls, exp: Experiment) -> Self: ... class ShellLauncher: - """Mock launcher for launching/tracking simple shell commands - - TODO: this is probably all we need for a "local" launcher, but probably - best to move this to a `smartsim._core.launcher` module/submodule - """ + """Mock launcher for launching/tracking simple shell commands""" def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 17dbfa7a2..9718523d2 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -75,7 +75,7 @@ def launch_args(self) -> LaunchArgBuilder[t.Any]: # subclasses. Otherwise we have no way of showing what methods # are available at intellisense/static analysis/compile time. # This whole object basically resolves to being one step removed - # from `Any` typed!! + # from `Any` typed (worse even, as type checkers will error)!! return self._arg_builder @launch_args.setter From c413b8f1c25ba425a6f6beee50809d7483fa2047 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 3 Jul 2024 12:35:41 -0500 Subject: [PATCH 16/82] textwrap.dedent fix --- smartsim/experiment.py | 6 +----- smartsim/settings/__init__.py | 2 -- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 182d4663a..2ce3c287b 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -574,8 +574,6 @@ def _launch_summary(self, manifest: Manifest) -> None: """ launcher_list = "\n".join(str(launcher) for launcher in self._active_launchers) summary = textwrap.dedent(f"""\ - - === Launch Summary === Experiment: {self.name} Experiment Path: {self.exp_path} @@ -593,9 +591,7 @@ def _launch_summary(self, manifest: Manifest) -> None: else: summary += "Feature Store Status: inactive\n" - summary += f"\n{str(manifest)}" - - logger.info(summary) + logger.info(f"\n\n{summary}\n{manifest}") def _create_entity_dir(self, start_manifest: Manifest) -> None: def create_entity_dir( diff --git a/smartsim/settings/__init__.py b/smartsim/settings/__init__.py index e904240af..e0313f341 100644 --- a/smartsim/settings/__init__.py +++ b/smartsim/settings/__init__.py @@ -24,8 +24,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import typing as t - from .baseSettings import BaseSettings from .batchSettings import BatchSettings from .launchSettings import LaunchSettings From 35e686c6ea554b963deedb23114bfdb740dbe410 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 3 Jul 2024 20:48:17 -0500 Subject: [PATCH 17/82] Add doc strs --- smartsim/_core/utils/helpers.py | 18 ++++++++++++++++++ smartsim/experiment.py | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index 646b086dd..d193b6604 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -415,6 +415,24 @@ def is_crayex_platform() -> bool: def first(predicate: t.Callable[[_T], bool], iterable: t.Iterable[_T]) -> _T | None: + """Return the first instance of an iterable that meets some precondition. + Any elements of the iterable that do not meet the precondition will be + forgotten. If no item in the iterable is found that meets the predicate, + `None` is returned. This is roughly equivalent to + + .. highlight:: python + .. code-block:: python + + next(filter(predicate, iterable), None) + + but does not require the predicate to be a type guard to type check. + + :param predicate: A function that returns `True` or `False` given a element + of the iterable + :param iterable: An iterable that yields elements to evealuate + :returns: The first element of the iterable to make the the `predicate` + return `True` + """ return next((item for item in iterable if predicate(item)), None) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 2ce3c287b..3b5690f55 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -113,7 +113,7 @@ def __init__( self, name: str, exp_path: str | None = None, - *, + *, # Keyword arguments only settings_dispatcher: Dispatcher = default_dispatcher, ): """Initialize an Experiment instance. @@ -155,6 +155,9 @@ def __init__( :param name: name for the ``Experiment`` :param exp_path: path to location of ``Experiment`` directory + :param settings_dispatcher: The dispatcher the experiment will use to + figure determine how to launch a job. If none is provided, the + experiment will use the default dispatcher. """ self.name = name if exp_path: From 9b2c2b75bbfe6a730faacd357fd04c6714290514 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 3 Jul 2024 20:49:18 -0500 Subject: [PATCH 18/82] Make dispatching settings more concise --- smartsim/_core/launcher/dragon/dragonLauncher.py | 4 ++-- smartsim/settings/builders/launch/alps.py | 4 ++-- smartsim/settings/builders/launch/local.py | 4 ++-- smartsim/settings/builders/launch/lsf.py | 4 ++-- smartsim/settings/builders/launch/mpi.py | 8 ++++---- smartsim/settings/builders/launch/pals.py | 4 ++-- smartsim/settings/builders/launch/slurm.py | 4 ++-- smartsim/settings/dispatch.py | 1 + 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index ce11ed91f..e137037c1 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -341,7 +341,7 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: # circular import # ----------------------------------------------------------------------------- from smartsim.settings.builders.launch.dragon import DragonArgBuilder -from smartsim.settings.dispatch import default_dispatcher +from smartsim.settings.dispatch import dispatch -default_dispatcher.dispatch(DragonArgBuilder, to_launcher=DragonLauncher) +dispatch(DragonArgBuilder, to_launcher=DragonLauncher) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index f1a196e7c..de2d7b91d 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, default_dispatcher +from smartsim.settings.dispatch import ShellLauncher, dispatch from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -41,7 +41,7 @@ logger = get_logger(__name__) -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class AprunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index 64770f696..aeb018bc4 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, default_dispatcher +from smartsim.settings.dispatch import ShellLauncher, dispatch from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -41,7 +41,7 @@ logger = get_logger(__name__) -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class LocalArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index e1a03ef3b..bec63a802 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, default_dispatcher +from smartsim.settings.dispatch import ShellLauncher, dispatch from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -41,7 +41,7 @@ logger = get_logger(__name__) -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class JsrunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 6eac12a24..7ce79fbc3 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, default_dispatcher +from smartsim.settings.dispatch import ShellLauncher, dispatch from ...common import set_check_input from ...launchCommand import LauncherType @@ -218,7 +218,7 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class MpiArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" @@ -230,7 +230,7 @@ def finalize( return ("mpirun", *self.format_launch_args(), "--", *exe.as_program_arguments()) -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class MpiexecArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" @@ -247,7 +247,7 @@ def finalize( ) -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class OrteArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index d21edc8bd..1b2ed17bf 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -29,7 +29,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, default_dispatcher +from smartsim.settings.dispatch import ShellLauncher, dispatch from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType @@ -41,7 +41,7 @@ logger = get_logger(__name__) -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class PalsMpiexecArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 1125c2611..db2d673cb 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -31,7 +31,7 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, default_dispatcher +from smartsim.settings.dispatch import ShellLauncher, dispatch from ...common import set_check_input from ...launchCommand import LauncherType @@ -43,7 +43,7 @@ logger = get_logger(__name__) -@default_dispatcher.dispatch(to_launcher=ShellLauncher) +@dispatch(to_launcher=ShellLauncher) class SlurmArgBuilder(LaunchArgBuilder[t.Sequence[str]]): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 9928bad91..f2c945f3a 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -132,6 +132,7 @@ def get_launcher_for( default_dispatcher: t.Final = Dispatcher() +dispatch: t.Final = default_dispatcher.dispatch # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> From 219d481bb71514e0b46e54db9fb4c07922d5d1d8 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Mon, 8 Jul 2024 19:09:28 -0500 Subject: [PATCH 19/82] Address reviewer feedback --- smartsim/experiment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index c6e86e366..cd38454a2 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -177,7 +177,9 @@ def __init__( # TODO: Remove this! The contoller is becoming obsolete self._control = Controller(launcher="local") self._dispatcher = settings_dispatcher + self._active_launchers: set[LauncherLike[t.Any]] = set() + """The active launchers created, used, and reused by the experiment""" self.fs_identifiers: t.Set[str] = set() self._telemetry_cfg = ExperimentTelemetryConfiguration() @@ -192,7 +194,7 @@ def start_jobs(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: ) def _start(job: Job) -> LaunchedJobID: - builder = job.launch_settings.launch_args + builder: LaunchArgBuilder[t.Any] = job.launch_settings.launch_args launcher_type = self._dispatcher.get_launcher_for(builder) launcher = first( lambda launcher: type(launcher) is launcher_type, From 979744266eea868472d5d77a523025e98bdfee62 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Tue, 16 Jul 2024 12:18:11 -0500 Subject: [PATCH 20/82] Remove stale FIXME comment --- smartsim/settings/launchSettings.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 24ee8993c..9078a04d9 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -69,13 +69,6 @@ def launcher(self) -> str: @property def launch_args(self) -> LaunchArgBuilder[t.Any]: """Return the launch argument translator.""" - # FIXME: We _REALLY_ need to make the `LaunchSettings` class generic at - # on `_arg_builder` if we are expecting users to call specific - # `set_*` methods defined specificially on each of the - # subclasses. Otherwise we have no way of showing what methods - # are available at intellisense/static analysis/compile time. - # This whole object basically resolves to being one step removed - # from `Any` typed (worse even, as type checkers will error)!! return self._arg_builder @launch_args.setter From 2a667f2654e0f9e806f73c07121f473d8fb0a95e Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 9 Jul 2024 17:38:25 -0500 Subject: [PATCH 21/82] Path injection and Generator class refactor --- smartsim/_core/generation/generator.py | 447 +++++++++--------- smartsim/_core/generation/modelwriter.py | 158 ------- smartsim/_core/utils/helpers.py | 2 +- smartsim/entity/ensemble.py | 3 +- smartsim/entity/entity.py | 3 +- smartsim/entity/model.py | 3 +- smartsim/experiment.py | 28 +- smartsim/launchable/job.py | 16 + smartsim/launchable/jobGroup.py | 7 + smartsim/settings/builders/launch/alps.py | 9 +- smartsim/settings/builders/launch/dragon.py | 7 +- smartsim/settings/builders/launch/lsf.py | 9 +- smartsim/settings/builders/launch/mpi.py | 32 +- smartsim/settings/builders/launch/pals.py | 9 +- smartsim/settings/builders/launch/slurm.py | 9 +- .../settings/builders/launchArgBuilder.py | 2 +- smartsim/settings/dispatch.py | 8 +- tests/temp_tests/test_launchable.py | 6 +- .../test_settings/test_alpsLauncher.py | 5 +- .../test_settings/test_dragonLauncher.py | 7 +- .../test_settings/test_lsfLauncher.py | 5 +- .../test_settings/test_mpiLauncher.py | 7 +- .../test_settings/test_palsLauncher.py | 5 +- .../test_settings/test_slurmLauncher.py | 5 +- tests/test_ensemble.py | 6 - tests/test_generator/test_generator.py | 107 +++++ 26 files changed, 449 insertions(+), 456 deletions(-) delete mode 100644 smartsim/_core/generation/modelwriter.py create mode 100644 tests/test_generator/test_generator.py diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index b1d241416..c2e287d3e 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -24,6 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import base64 +import os import pathlib import shutil import typing as t @@ -36,10 +38,10 @@ from tabulate import tabulate from ...database import FeatureStore -from ...entity import Application, Ensemble, TaggedFilesHierarchy +from ...entity import Application, TaggedFilesHierarchy +from ...launchable import Job, JobGroup from ...log import get_logger -from ..control import Manifest -from .modelwriter import ApplicationWriter +from ..utils.helpers import create_short_id_str logger = get_logger(__name__) logger.propagate = False @@ -47,30 +49,89 @@ class Generator: """The primary job of the generator is to create the file structure - for a SmartSim experiment. The Generator is responsible for reading - and writing into configuration files as well. + for a SmartSim experiment. The Generator is also responsible for reading + and writing into configuration files. """ - def __init__( - self, gen_path: str, overwrite: bool = False, verbose: bool = True - ) -> None: + def __init__(self, gen_path: str, run_ID: str, job: Job) -> None: """Initialize a generator object - if overwrite is true, replace any existing - configured applications within an ensemble if there - is a name collision. Also replace any and all directories - for the experiment with fresh copies. Otherwise, if overwrite - is false, raises EntityExistsError when there is a name - collision between entities. + The Generator class is responsible for creating Job directories. + It ensures that paths adhere to SmartSim path standards. Additionally, + it creates a log directory for telemetry data and handles symlinking, + configuration, and file copying within the job directory. :param gen_path: Path in which files need to be generated - :param overwrite: toggle entity replacement - :param verbose: Whether generation information should be logged to std out + :param job: Reference to a SmartSimEntity and LaunchSettings """ - self._writer = ApplicationWriter() - self.gen_path = gen_path - self.overwrite = overwrite - self.log_level = DEBUG if not verbose else INFO + self.job = job + self.path = self._generate_job_path(job, gen_path, run_ID) + self.log_path = self._generate_log_path(gen_path) + + def _generate_log_path(self, gen_path: str) -> str: + """ + Generates the path for logs. + + :param gen_path: The base path for job generation + :returns str: The generated path for the log directory. + """ + log_path = os.path.join(gen_path, "log") + return log_path + + def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: + """ + Generates the path for a job based on its type and ensemble name (if applicable). + + :param job: The Job object + :param gen_path: The base path for job generation + :param run_ID: The unique run ID + :returns str: The generated path for the job. + """ + if job._ensemble_name is None: + job_type = f"{job.__class__.__name__.lower()}s" + entity_type = f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" + path = os.path.join( + gen_path, + run_ID, + job_type, + f"{job.name}-{create_short_id_str()}", + entity_type, + "run", + ) + else: + job_type = "ensembles" + entity_type = f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" + path = os.path.join( + gen_path, + run_ID, + job_type, + job._ensemble_name, + f"{job.name}", + entity_type, + "run", + ) + return path + + @property + def log_level(self) -> int: + """Determines the log level based on the value of the environment + variable SMARTSIM_LOG_LEVEL. + + If the environment variable is set to "debug", returns the log level DEBUG. + Otherwise, returns the default log level INFO. + + :return: Log level (DEBUG or INFO) + """ + # Get the value of the environment variable SMARTSIM_LOG_LEVEL + env_log_level = os.getenv("SMARTSIM_LOG_LEVEL") + + # Set the default log level to INFO + default_log_level = INFO + + if env_log_level == "debug": + return DEBUG + else: + return default_log_level @property def log_file(self) -> str: @@ -80,9 +141,9 @@ def log_file(self) -> str: :returns: path to file with parameter settings """ - return join(self.gen_path, "smartsim_params.txt") + return join(self.path, "smartsim_params.txt") - def generate_experiment(self, *args: t.Any) -> None: + def generate_experiment(self) -> str: """Run ensemble and experiment file structure generation Generate the file structure for a SmartSim experiment. This @@ -102,49 +163,8 @@ def generate_experiment(self, *args: t.Any) -> None: e.g. ``THERMO=;90;`` """ - generator_manifest = Manifest(*args) - - self._gen_exp_dir() - self._gen_feature_store_dir(generator_manifest.fss) - self._gen_entity_list_dir(generator_manifest.ensembles) - self._gen_entity_dirs(generator_manifest.applications) - - def set_tag(self, tag: str, regex: t.Optional[str] = None) -> None: - """Set the tag used for tagging input files - - Set a tag or a regular expression for the - generator to look for when configuring new applications. - - For example, a tag might be ``;`` where the - expression being replaced in the application configuration - file would look like ``;expression;`` - - A full regular expression might tag specific - application configurations such that the configuration - files don't need to be tagged manually. - - :param tag: A string of characters that signify - the string to be changed. Defaults to ``;`` - :param regex: full regex for the applicationwriter to search for - """ - self._writer.set_tag(tag, regex) - - def _gen_exp_dir(self) -> None: - """Create the directory for an experiment if it does not - already exist. - """ - - if path.isfile(self.gen_path): - raise FileExistsError( - f"Experiment directory could not be created. {self.gen_path} exists" - ) - if not path.isdir(self.gen_path): - # keep exists ok for race conditions on NFS - pathlib.Path(self.gen_path).mkdir(exist_ok=True, parents=True) - else: - logger.log( - level=self.log_level, msg="Working in previously created experiment" - ) + pathlib.Path(self.path).mkdir(exist_ok=True, parents=True) + pathlib.Path(self.log_path).mkdir(exist_ok=True, parents=True) # The log_file only keeps track of the last generation # this is to avoid gigantic files in case the user repeats @@ -154,81 +174,47 @@ def _gen_exp_dir(self) -> None: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - def _gen_feature_store_dir(self, feature_store_list: t.List[FeatureStore]) -> None: - """Create the directory that will hold the error, output and - configuration files for the feature store. - - :param featurestore: FeatureStore instance + # TODO update this to execute the file operations when entrypoint is merged in + # if isinstance(Application, type(self.job.entity)): + # file_operation_list = self.build_operations() + # self.execute_file_operations(file_operation_list) + return self.path + + # TODO update this to execute the file operations when entrypoint is merged in + def execute_file_operations( + self, file_ops: t.Sequence[t.Sequence[str]] + ) -> None: ... + + def build_operations(self) -> t.Sequence[t.Sequence[str]]: + """This method generates file system operations based on the provided application. + It processes three types of operations: to_copy, to_symlink, and to_configure. + For each type, it calls the corresponding private methods and appends the results + to the `file_operation_list`. + + :param app: The application for which operations are generated. + :return: A list of lists containing file system operations. """ - # Loop through feature stores - for featurestore in feature_store_list: - feature_store_path = path.join(self.gen_path, featurestore.name) - - featurestore.set_path(feature_store_path) - # Always remove featurestore files if present. - if path.isdir(feature_store_path): - shutil.rmtree(feature_store_path, ignore_errors=True) - pathlib.Path(feature_store_path).mkdir( - exist_ok=self.overwrite, parents=True - ) - - def _gen_entity_list_dir(self, entity_lists: t.List[Ensemble]) -> None: - """Generate directories for Ensemble instances - - :param entity_lists: list of Ensemble instances - """ - - if not entity_lists: - return - - for elist in entity_lists: - elist_dir = path.join(self.gen_path, elist.name) - if path.isdir(elist_dir): - if self.overwrite: - shutil.rmtree(elist_dir) - mkdir(elist_dir) - else: - mkdir(elist_dir) - elist.path = elist_dir - - def _gen_entity_dirs( - self, - entities: t.List[Application], - entity_list: t.Optional[Ensemble] = None, - ) -> None: - """Generate directories for Entity instances - - :param entities: list of Application instances - :param entity_list: Ensemble instance - :raises EntityExistsError: if a directory already exists for an - entity by that name - """ - if not entities: - return - - for entity in entities: - if entity_list: - dst = path.join(self.gen_path, entity_list.name, entity.name) - else: - dst = path.join(self.gen_path, entity.name) - - if path.isdir(dst): - if self.overwrite: - shutil.rmtree(dst) - else: - error = ( - f"Directory for entity {entity.name} " - f"already exists in path {dst}" - ) - raise FileExistsError(error) - pathlib.Path(dst).mkdir(exist_ok=True) - entity.path = dst - - self._copy_entity_files(entity) - self._link_entity_files(entity) - self._write_tagged_entity_files(entity) - - def _write_tagged_entity_files(self, entity: Application) -> None: + application_files = self.job.entity.files + file_operation_list: t.List[t.Sequence[str]] = [] + # Generate copy file system operations + file_operation_list.extend( + self._get_copy_file_system_operation(file_copy) + for file_copy in application_files.copy + ) + # Generate symlink file system operations + file_operation_list.extend( + self._get_symlink_file_system_operation(file_link) + for file_link in application_files.link + ) + # Generate configure file system operations + file_operation_list.extend( + self._write_tagged_entity_files(file_configure) + for file_configure in application_files.tagged + ) + return file_operation_list + + # TODO update this to execute the file operations when entrypoint is merged in + def _write_tagged_entity_files(self, configure_file: str) -> t.Sequence[str]: """Read, configure and write the tagged input files for a Application instance within an ensemble. This function specifically deals with the tagged files attached to @@ -236,103 +222,100 @@ def _write_tagged_entity_files(self, entity: Application) -> None: :param entity: a Application instance """ - if entity.files: - to_write = [] - - def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: - """Using a TaggedFileHierarchy, reproduce the tagged file - directory structure - - :param tagged: a TaggedFileHierarchy to be built as a - directory structure - """ - for file in tagged.files: - dst_path = path.join(entity.path, tagged.base, path.basename(file)) - shutil.copyfile(file, dst_path) - to_write.append(dst_path) - - for tagged_dir in tagged.dirs: - mkdir( - path.join( - entity.path, tagged.base, path.basename(tagged_dir.base) - ) - ) - _build_tagged_files(tagged_dir) - - if entity.files.tagged_hierarchy: - _build_tagged_files(entity.files.tagged_hierarchy) - - # write in changes to configurations - if isinstance(entity, Application): - files_to_params = self._writer.configure_tagged_application_files( - to_write, entity.params - ) - self._log_params(entity, files_to_params) - - def _log_params( - self, entity: Application, files_to_params: t.Dict[str, t.Dict[str, str]] - ) -> None: - """Log which files were modified during generation - - and what values were set to the parameters - - :param entity: the application being generated - :param files_to_params: a dict connecting each file to its parameter settings - """ - used_params: t.Dict[str, str] = {} - file_to_tables: t.Dict[str, str] = {} - for file, params in files_to_params.items(): - used_params.update(params) - table = tabulate(params.items(), headers=["Name", "Value"]) - file_to_tables[relpath(file, self.gen_path)] = table - - if used_params: - used_params_str = ", ".join( - [f"{name}={value}" for name, value in used_params.items()] - ) - logger.log( - level=self.log_level, - msg=f"Configured application {entity.name} with params {used_params_str}", - ) - file_table = tabulate( - file_to_tables.items(), - headers=["File name", "Parameters"], - ) - log_entry = f"Application name: {entity.name}\n{file_table}\n\n" - with open(self.log_file, mode="a", encoding="utf-8") as logfile: - logfile.write(log_entry) - with open( - join(entity.path, "smartsim_params.txt"), mode="w", encoding="utf-8" - ) as local_logfile: - local_logfile.write(log_entry) - - else: - logger.log( - level=self.log_level, - msg=f"Configured application {entity.name} with no parameters", - ) - + # if entity.files: + # to_write = [] + + # def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: + # """Using a TaggedFileHierarchy, reproduce the tagged file + # directory structure + + # :param tagged: a TaggedFileHierarchy to be built as a + # directory structure + # """ + # for file in tagged.files: + # dst_path = path.join(entity.path, tagged.base, path.basename(file)) + # shutil.copyfile(file, dst_path) + # to_write.append(dst_path) + + # for tagged_dir in tagged.dirs: + # mkdir( + # path.join( + # entity.path, tagged.base, path.basename(tagged_dir.base) + # ) + # ) + # _build_tagged_files(tagged_dir) + + # if entity.files.tagged_hierarchy: + # _build_tagged_files(entity.files.tagged_hierarchy) + + # # write in changes to configurations + # if isinstance(entity, Application): + # files_to_params = self._writer.configure_tagged_application_files( + # to_write, entity.params + # ) + # self._log_params(entity, files_to_params) + return ["temporary", "config"] + + # TODO replace with entrypoint operation @staticmethod - def _copy_entity_files(entity: Application) -> None: - """Copy the entity files and directories attached to this entity. + def _get_copy_file_system_operation(copy_file: str) -> t.Sequence[str]: + """Get copy file system operation for a file. - :param entity: Application + :param linked_file: The file to be copied. + :return: A list of copy file system operations. """ - if entity.files: - for to_copy in entity.files.copy: - dst_path = path.join(entity.path, path.basename(to_copy)) - if path.isdir(to_copy): - dir_util.copy_tree(to_copy, entity.path) - else: - shutil.copyfile(to_copy, dst_path) + return ["temporary", "copy"] + # TODO replace with entrypoint operation @staticmethod - def _link_entity_files(entity: Application) -> None: - """Symlink the entity files attached to this entity. + def _get_symlink_file_system_operation(linked_file: str) -> t.Sequence[str]: + """Get symlink file system operation for a file. - :param entity: Application + :param linked_file: The file to be symlinked. + :return: A list of symlink file system operations. """ - if entity.files: - for to_link in entity.files.link: - dst_path = path.join(entity.path, path.basename(to_link)) - symlink(to_link, dst_path) + return ["temporary", "link"] + + # TODO to be refactored in ticket 723 + # def _log_params( + # self, entity: Application, files_to_params: t.Dict[str, t.Dict[str, str]] + # ) -> None: + # """Log which files were modified during generation + + # and what values were set to the parameters + + # :param entity: the application being generated + # :param files_to_params: a dict connecting each file to its parameter settings + # """ + # used_params: t.Dict[str, str] = {} + # file_to_tables: t.Dict[str, str] = {} + # for file, params in files_to_params.items(): + # used_params.update(params) + # table = tabulate(params.items(), headers=["Name", "Value"]) + # file_to_tables[relpath(file, self.gen_path)] = table + + # if used_params: + # used_params_str = ", ".join( + # [f"{name}={value}" for name, value in used_params.items()] + # ) + # logger.log( + # level=self.log_level, + # msg=f"Configured application {entity.name} with params {used_params_str}", + # ) + # file_table = tabulate( + # file_to_tables.items(), + # headers=["File name", "Parameters"], + # ) + # log_entry = f"Application name: {entity.name}\n{file_table}\n\n" + # with open(self.log_file, mode="a", encoding="utf-8") as logfile: + # logfile.write(log_entry) + # with open( + # join(entity.path, "smartsim_params.txt"), mode="w", encoding="utf-8" + # ) as local_logfile: + # local_logfile.write(log_entry) + + # else: + # logger.log( + # level=self.log_level, + # msg=f"Configured application {entity.name} with no parameters", + # ) diff --git a/smartsim/_core/generation/modelwriter.py b/smartsim/_core/generation/modelwriter.py deleted file mode 100644 index a22bc029a..000000000 --- a/smartsim/_core/generation/modelwriter.py +++ /dev/null @@ -1,158 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2024, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import collections -import re -import typing as t - -from smartsim.error.errors import SmartSimError - -from ...error import ParameterWriterError -from ...log import get_logger - -logger = get_logger(__name__) - - -class ApplicationWriter: - def __init__(self) -> None: - self.tag = ";" - self.regex = "(;[^;]+;)" - self.lines: t.List[str] = [] - - def set_tag(self, tag: str, regex: t.Optional[str] = None) -> None: - """Set the tag for the applicationwriter to search for within - tagged files attached to an entity. - - :param tag: tag for the applicationwriter to search for, - defaults to semi-colon e.g. ";" - :param regex: full regex for the applicationwriter to search for, - defaults to "(;.+;)" - """ - if regex: - self.regex = regex - else: - self.tag = tag - self.regex = "".join(("(", tag, ".+", tag, ")")) - - def configure_tagged_application_files( - self, - tagged_files: t.List[str], - params: t.Dict[str, str], - make_missing_tags_fatal: bool = False, - ) -> t.Dict[str, t.Dict[str, str]]: - """Read, write and configure tagged files attached to a Application - instance. - - :param tagged_files: list of paths to tagged files - :param params: application parameters - :param make_missing_tags_fatal: raise an error if a tag is missing - :returns: A dict connecting each file to its parameter settings - """ - files_to_tags: t.Dict[str, t.Dict[str, str]] = {} - for tagged_file in tagged_files: - self._set_lines(tagged_file) - used_tags = self._replace_tags(params, make_missing_tags_fatal) - self._write_changes(tagged_file) - files_to_tags[tagged_file] = used_tags - - return files_to_tags - - def _set_lines(self, file_path: str) -> None: - """Set the lines for the applicationwriter to iterate over - - :param file_path: path to the newly created and tagged file - :raises ParameterWriterError: if the newly created file cannot be read - """ - try: - with open(file_path, "r+", encoding="utf-8") as file_stream: - self.lines = file_stream.readlines() - except (IOError, OSError) as e: - raise ParameterWriterError(file_path) from e - - def _write_changes(self, file_path: str) -> None: - """Write the ensemble-specific changes - - :raises ParameterWriterError: if the newly created file cannot be read - """ - try: - with open(file_path, "w+", encoding="utf-8") as file_stream: - for line in self.lines: - file_stream.write(line) - except (IOError, OSError) as e: - raise ParameterWriterError(file_path, read=False) from e - - def _replace_tags( - self, params: t.Dict[str, str], make_fatal: bool = False - ) -> t.Dict[str, str]: - """Replace the tagged parameters within the file attached to this - application. The tag defaults to ";" - - :param application: The application instance - :param make_fatal: (Optional) Set to True to force a fatal error - if a tag is not matched - :returns: A dict of parameter names and values set for the file - """ - edited = [] - unused_tags: t.DefaultDict[str, t.List[int]] = collections.defaultdict(list) - used_params: t.Dict[str, str] = {} - for i, line in enumerate(self.lines, 1): - while search := re.search(self.regex, line): - tagged_line = search.group(0) - previous_value = self._get_prev_value(tagged_line) - if self._is_ensemble_spec(tagged_line, params): - new_val = str(params[previous_value]) - line = re.sub(self.regex, new_val, line, 1) - used_params[previous_value] = new_val - - # if a tag is found but is not in this application's configurations - # put in placeholder value - else: - tag = tagged_line.split(self.tag)[1] - unused_tags[tag].append(i) - line = re.sub(self.regex, previous_value, line) - break - edited.append(line) - - for tag, value in unused_tags.items(): - missing_tag_message = f"Unused tag {tag} on line(s): {str(value)}" - if make_fatal: - raise SmartSimError(missing_tag_message) - logger.warning(missing_tag_message) - self.lines = edited - return used_params - - def _is_ensemble_spec( - self, tagged_line: str, application_params: t.Dict[str, str] - ) -> bool: - split_tag = tagged_line.split(self.tag) - prev_val = split_tag[1] - if prev_val in application_params.keys(): - return True - return False - - def _get_prev_value(self, tagged_line: str) -> str: - split_tag = tagged_line.split(self.tag) - return split_tag[1] diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index d193b6604..0e58d7a78 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -74,7 +74,7 @@ def unpack_colo_fs_identifier(fs_id: str) -> str: return "_" + fs_id if fs_id else "" -def create_short_id_str() -> str: +def create_short_id_str() -> str: # here return str(uuid.uuid4())[:7] diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index 517d33161..883b69f9a 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -97,7 +97,6 @@ def _create_applications(self) -> tuple[Application, ...]: # ^^^^^^^^^^^^^^^^^^^^^^^ # FIXME: remove this constructor arg! It should not exist!! exe_args=self.exe_args, - path=os.path.join(self.path, self.name), files=self.files, params=permutation.params, params_as_args=permutation.exe_args, # type: ignore[arg-type] @@ -111,4 +110,4 @@ def as_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: apps = self._create_applications() if not apps: raise ValueError("There are no members as part of this ensemble") - return tuple(Job(app, settings) for app in apps) + return tuple(Job(app, settings, f"job_{i}", ensemble_name=self.name) for i, app in enumerate(apps, 1)) diff --git a/smartsim/entity/entity.py b/smartsim/entity/entity.py index 6416a8b2b..8f0ca73b8 100644 --- a/smartsim/entity/entity.py +++ b/smartsim/entity/entity.py @@ -98,7 +98,7 @@ def _on_disable(self) -> None: class SmartSimEntity: - def __init__(self, name: str, path: str, run_settings: "RunSettings") -> None: + def __init__(self, name: str, run_settings: "RunSettings") -> None: """Initialize a SmartSim entity. Each entity must have a name, path, and @@ -110,7 +110,6 @@ def __init__(self, name: str, path: str, run_settings: "RunSettings") -> None: """ self.name = name self.run_settings = run_settings - self.path = path @property def type(self) -> str: diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index 4304ee95b..93e07577d 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -64,7 +64,6 @@ def __init__( run_settings: "RunSettings", params: t.Optional[t.Dict[str, str]] = None, exe_args: t.Optional[t.List[str]] = None, - path: t.Optional[str] = getcwd(), params_as_args: t.Optional[t.List[str]] = None, batch_settings: t.Optional["BatchSettings"] = None, files: t.Optional[EntityFiles] = None, @@ -85,7 +84,7 @@ def __init__( application as a batch job :param files: Files to have available to the application """ - super().__init__(name, str(path), run_settings) + super().__init__(name, run_settings) self.exe = [expand_exe_path(exe)] # self.exe = [exe] if run_settings.container else [expand_exe_path(exe)] self.exe_args = exe_args or [] diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 8a4ed42f6..26ce08bbe 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -28,6 +28,7 @@ from __future__ import annotations +import datetime import itertools import os import os.path as osp @@ -173,6 +174,12 @@ def __init__( exp_path = osp.join(getcwd(), name) self.exp_path = exp_path + self.run_ID = ( + "run-" + + datetime.datetime.now().strftime("%H:%M:%S") + + "-" + + datetime.datetime.now().strftime("%Y-%m-%d") + ) # TODO: Remove this! The contoller is becoming obsolete self._control = Controller(launcher="local") @@ -203,6 +210,7 @@ def _start(job: Job) -> LaunchedJobID: if launcher is None: launcher = launcher_type.create(self) self._active_launchers.add(launcher) + job_execution_path = self._generate(job) # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # FIXME: Opting out of type check here. Fix this later!! # TODO: Very much dislike that we have to pass in attrs off of `job` @@ -212,7 +220,9 @@ def _start(job: Job) -> LaunchedJobID: # to protocol # --------------------------------------------------------------------- exe_like = t.cast("ExecutableLike", job.entity) - finalized = builder.finalize(exe_like, job.launch_settings.env_vars) + finalized = builder.finalize( + exe_like, job.launch_settings.env_vars, job_execution_path + ) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< return launcher.start(finalized) @@ -326,13 +336,10 @@ def stop( raise @_contextualize - def generate( + def _generate( self, - *args: t.Union[SmartSimEntity, EntitySequence[SmartSimEntity]], - tag: t.Optional[str] = None, - overwrite: bool = False, - verbose: bool = False, - ) -> None: + job: Job, + ) -> str: """Generate the file structure for an ``Experiment`` ``Experiment.generate`` creates directories for each entity @@ -351,10 +358,9 @@ def generate( :param verbose: log parameter settings to std out """ try: - generator = Generator(self.exp_path, overwrite=overwrite, verbose=verbose) - if tag: - generator.set_tag(tag) - generator.generate_experiment(*args) + generator = Generator(self.exp_path, self.run_ID, job) + job_path = generator.generate_experiment() + return job_path except SmartSimError as e: logger.error(e) raise diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index f440ead0b..8608beddc 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -32,6 +32,10 @@ from smartsim._core.commands.launchCommands import LaunchCommands from smartsim.launchable.basejob import BaseJob from smartsim.settings import LaunchSettings +from smartsim._core.utils.helpers import create_short_id_str + +if t.TYPE_CHECKING: + from smartsim.entity.entity import SmartSimEntity if t.TYPE_CHECKING: from smartsim.entity.entity import SmartSimEntity @@ -50,12 +54,24 @@ def __init__( self, entity: SmartSimEntity, launch_settings: LaunchSettings, + name: str = "job", + **kwargs: t.Any, ): super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) + self._name = deepcopy(name) + self._ensemble_name = kwargs.get('ensemble_name', None) + if self._ensemble_name is not None: + self._ensemble_name += f"-{create_short_id_str()}" # TODO: self.warehouse_runner = JobWarehouseRunner + # TODO do we want the user to be allowed to reset the Job name? Therefore, add setter + @property + def name(self) -> str: + """Retrieves the name of the Job.""" + return deepcopy(self._name) + @property def entity(self) -> SmartSimEntity: return deepcopy(self._entity) diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index de7ed691b..a8ef4440a 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -19,9 +19,16 @@ class JobGroup(BaseJobGroup): def __init__( self, jobs: t.List[BaseJob], + name: str = "jobGroup", ) -> None: super().__init__() self._jobs = deepcopy(jobs) + self._name = deepcopy(name) + + @property + def name(self) -> str: + """Retrieves the name of the JobGroup.""" + return deepcopy(self._name) @property def jobs(self) -> t.List[BaseJob]: diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index de2d7b91d..e9fcc9f73 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -220,11 +220,14 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, + ) -> t.Tuple[t.Sequence[str], str]: return ( "aprun", *(self.format_launch_args() or ()), "--", *exe.as_program_arguments(), - ) + ), job_execution_path diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 8fb2ebc48..3b3dc1e6a 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -69,7 +69,10 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, ) -> DragonRunRequestView: exe_, *args = exe.as_program_arguments() return DragonRunRequestView( @@ -81,7 +84,7 @@ def finalize( # the command execute next to any generated files. A similar # problem exists for the other settings. # TODO: Find a way to inject this path - path=os.getcwd(), + path=job_execution_path, env=env, # TODO: Not sure how this info is injected name=None, diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index bec63a802..6e2729e0a 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -122,11 +122,14 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, + ) -> t.Tuple[t.Sequence[str], str]: return ( "jsrun", *(self.format_launch_args() or ()), "--", *exe.as_program_arguments(), - ) + ), job_execution_path diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 7ce79fbc3..d410958b4 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -225,9 +225,17 @@ def launcher_str(self) -> str: return LauncherType.Mpirun.value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ("mpirun", *self.format_launch_args(), "--", *exe.as_program_arguments()) + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, + ) -> t.Tuple[t.Sequence[str], str]: + return ( + "mpirun", + *self.format_launch_args(), + "--", + *exe.as_program_arguments(), + ), job_execution_path @dispatch(to_launcher=ShellLauncher) @@ -237,14 +245,17 @@ def launcher_str(self) -> str: return LauncherType.Mpiexec.value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, + ) -> t.Tuple[t.Sequence[str], str]: return ( "mpiexec", *self.format_launch_args(), "--", *exe.as_program_arguments(), - ) + ), job_execution_path @dispatch(to_launcher=ShellLauncher) @@ -254,11 +265,14 @@ def launcher_str(self) -> str: return LauncherType.Orterun.value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, + ) -> t.Tuple[t.Sequence[str], str]: return ( "orterun", *self.format_launch_args(), "--", *exe.as_program_arguments(), - ) + ), job_execution_path diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index 1b2ed17bf..81518ddd0 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -156,11 +156,14 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, + ) -> t.Tuple[t.Sequence[str], str]: return ( "mpiexec", *(self.format_launch_args() or ()), "--", *exe.as_program_arguments(), - ) + ), job_execution_path diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index db2d673cb..7cbd5a752 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -322,11 +322,14 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: + self, + exe: ExecutableLike, + env: t.Mapping[str, str | None], + job_execution_path: str, + ) -> t.Tuple[t.Sequence[str], str]: return ( "srun", *(self.format_launch_args() or ()), "--", *exe.as_program_arguments(), - ) + ), job_execution_path diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py index b125046cd..0012992b2 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/builders/launchArgBuilder.py @@ -59,7 +59,7 @@ def set(self, arg: str, val: str | None) -> None: """Set the launch arguments""" @abstractmethod - def finalize(self, exe: ExecutableLike, env: t.Mapping[str, str | None]) -> _T: + def finalize(self, exe: ExecutableLike, env: t.Mapping[str, str | None], job_execution_path: str) -> t.Tuple[t.Sequence[str], str]: """Prepare an entity for launch using the built options""" def format_launch_args(self) -> t.Union[t.List[str], None]: diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index f2c945f3a..0e466f89c 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -156,10 +156,14 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - def start(self, launchable: t.Sequence[str]) -> LaunchedJobID: + def start( + self, launchable: t.Sequence[str], job_execution_path: str + ) -> LaunchedJobID: id_ = create_job_id() exe, *rest = launchable - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest)) + self._launched[id_] = sp.Popen( + (helpers.expand_exe_path(exe), *rest), cwd=job_execution_path + ) # can specify a different dir for Popen return id_ @classmethod diff --git a/tests/temp_tests/test_launchable.py b/tests/temp_tests/test_launchable.py index 2a77817e5..a8b946438 100644 --- a/tests/temp_tests/test_launchable.py +++ b/tests/temp_tests/test_launchable.py @@ -79,7 +79,7 @@ def test_job_init_deepcopy(): def test_add_mpmd_pair(): - entity = SmartSimEntity("test_name", "python", LaunchSettings("slurm")) + entity = SmartSimEntity("test_name", LaunchSettings("slurm")) mpmd_job = MPMDJob() mpmd_job.add_mpmd_pair(entity, LaunchSettings("slurm")) @@ -154,10 +154,10 @@ def test_add_mpmd_pair_check_launcher_error(): """Test that an error is raised when a pairs is added to an mpmd job using add_mpmd_pair that does not have the same launcher type""" mpmd_pairs = [] - entity1 = SmartSimEntity("entity1", "python", LaunchSettings("slurm")) + entity1 = SmartSimEntity("entity1", LaunchSettings("slurm")) launch_settings1 = LaunchSettings("slurm") - entity2 = SmartSimEntity("entity2", "python", LaunchSettings("pals")) + entity2 = SmartSimEntity("entity2", LaunchSettings("pals")) launch_settings2 = LaunchSettings("pals") pair1 = MPMDPair(entity1, launch_settings1) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 7fa95cb6d..8e23cf26b 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -182,6 +182,7 @@ def test_invalid_exclude_hostlist_format(): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = AprunArgBuilder(args).finalize(echo_executable_like, {}) +def test_formatting_launch_args(echo_executable_like, args, expected, test_dir): + cmd, path = AprunArgBuilder(args).finalize(echo_executable_like, {}, test_dir) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 004090eef..1aefa2cfc 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -36,14 +36,14 @@ def test_dragon_class_methods(function, value, flag, result): @pytest.mark.parametrize("nodes", (NOT_SET, 20, 40)) @pytest.mark.parametrize("tasks_per_node", (NOT_SET, 1, 20)) def test_formatting_launch_args_into_request( - echo_executable_like, nodes, tasks_per_node + echo_executable_like, nodes, tasks_per_node, test_dir ): builder = DragonArgBuilder({}) if nodes is not NOT_SET: builder.set_nodes(nodes) if tasks_per_node is not NOT_SET: builder.set_tasks_per_node(tasks_per_node) - req = builder.finalize(echo_executable_like, {}) + req = builder.finalize(echo_executable_like, {}, test_dir) args = dict( (k, v) @@ -54,10 +54,11 @@ def test_formatting_launch_args_into_request( if v is not NOT_SET ) expected = DragonRunRequest( - exe="echo", exe_args=["hello", "world"], path="/tmp", env={}, **args + exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, **args ) assert req.nodes == expected.nodes assert req.tasks_per_node == expected.tasks_per_node assert req.hostlist == expected.hostlist assert req.pmi_enabled == expected.pmi_enabled + assert req.path == expected.path diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 592c80ce7..15b001f3f 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -91,6 +91,7 @@ def test_launch_args(): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = JsrunArgBuilder(args).finalize(echo_executable_like, {}) +def test_formatting_launch_args(echo_executable_like, args, expected, test_dir): + cmd, path = JsrunArgBuilder(args).finalize(echo_executable_like, {}, test_dir) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 9b651c220..b1f669c2d 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -248,6 +248,9 @@ def test_invalid_hostlist_format(launcher): ), ), ) -def test_formatting_launch_args(echo_executable_like, cls, cmd, args, expected): - fmt = cls(args).finalize(echo_executable_like, {}) +def test_formatting_launch_args( + echo_executable_like, cls, cmd, args, expected, test_dir +): + fmt, path = cls(args).finalize(echo_executable_like, {}, test_dir) assert tuple(fmt) == (cmd,) + expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index a0bc7821c..3a67611da 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -102,6 +102,7 @@ def test_invalid_hostlist_format(): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = PalsMpiexecArgBuilder(args).finalize(echo_executable_like, {}) +def test_formatting_launch_args(echo_executable_like, args, expected, test_dir): + cmd, path = PalsMpiexecArgBuilder(args).finalize(echo_executable_like, {}, test_dir) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index bfa7dd9e1..af3aae0d6 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -288,6 +288,7 @@ def test_set_het_groups(monkeypatch): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = SlurmArgBuilder(args).finalize(echo_executable_like, {}) +def test_formatting_launch_args(echo_executable_like, args, expected, test_dir): + cmd, path = SlurmArgBuilder(args).finalize(echo_executable_like, {}, test_dir) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index 3f170dfcb..4eb578a71 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -58,7 +58,6 @@ def test_ensemble_user_created_strategy(mock_launcher_settings, test_dir): "test_ensemble", "echo", ("hello", "world"), - path=test_dir, permutation_strategy=user_created_function, ).as_jobs(mock_launcher_settings) assert len(jobs) == 1 @@ -72,7 +71,6 @@ def test_ensemble_without_any_members_raises_when_cast_to_jobs( "test_ensemble", "echo", ("hello", "world"), - path=test_dir, file_parameters=_2x2_PARAMS, permutation_strategy="random", max_permutations=30, @@ -86,7 +84,6 @@ def test_strategy_error_raised_if_a_strategy_that_dne_is_requested(test_dir): "test_ensemble", "echo", ("hello",), - path=test_dir, permutation_strategy="THIS-STRATEGY-DNE", )._create_applications() @@ -105,7 +102,6 @@ def test_replicated_applications_have_eq_deep_copies_of_parameters(params, test_ "test_ensemble", "echo", ("hello",), - path=test_dir, replicas=4, file_parameters=params, )._create_applications() @@ -151,7 +147,6 @@ def test_all_perm_strategy( "test_ensemble", "echo", ("hello", "world"), - path=test_dir, file_parameters=params, exe_arg_parameters=exe_arg_params, permutation_strategy="all_perm", @@ -206,7 +201,6 @@ def test_step_strategy( "test_ensemble", "echo", ("hello", "world"), - path=test_dir, file_parameters=params, exe_arg_parameters=exe_arg_params, permutation_strategy="step", diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py new file mode 100644 index 000000000..72be67b3c --- /dev/null +++ b/tests/test_generator/test_generator.py @@ -0,0 +1,107 @@ +import datetime +from logging import DEBUG, INFO +from os import environ +from os import path as osp + +import pytest + +from smartsim import Experiment +from smartsim._core.generation.generator import Generator +from smartsim.entity import Application, Ensemble +from smartsim.launchable import Job, JobGroup +from smartsim.settings.builders.launch import SlurmArgBuilder +from smartsim.settings.dispatch import Dispatcher +from smartsim.settings.launchSettings import LaunchSettings + + +class NoOpLauncher: + @classmethod + def create(cls, _): + return cls() + + def start(self, _): + return "anything" + +class EchoApp: + def as_program_arguments(self): + return ["echo", "Hello", "World!"] + + +@pytest.fixture +def gen_instance_for_job(test_dir, wlmutils) -> Generator: + """Fixture to create an instance of Generator.""" + experiment_path = osp.join(test_dir, "experiment_name") + run_ID = ( + "run-" + + datetime.datetime.now().strftime("%H:%M:%S") + + "-" + + datetime.datetime.now().strftime("%Y-%m-%d") + ) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + application_1 = Application("app_name", exe="python", run_settings="RunSettings") + job = Job(application_1, launch_settings) + return Generator(gen_path=experiment_path, run_ID=run_ID, job=job) + + +@pytest.fixture +def job_group_instance(test_dir, wlmutils) -> Generator: + """Fixture to create an instance of Generator.""" + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + application_1 = Application("app_name_1", exe="python", run_settings="RunSettings") + application_2 = Application("app_name_2", exe="python", run_settings="RunSettings") + job_group = JobGroup(application_1, application_2, launch_settings) + return job_group + + +@pytest.fixture +def job_instance(test_dir, wlmutils) -> Generator: + """Fixture to create an instance of Generator.""" + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job = Job(EchoApp(), launch_settings) + return job + + +def test_default_log_level(gen_instance_for_job): + """Test if the default log level is INFO.""" + assert gen_instance_for_job.log_level == INFO + + +def test_debug_log_level(gen_instance_for_job): + """Test if the log level is DEBUG when environment variable is set to "debug".""" + environ["SMARTSIM_LOG_LEVEL"] = "debug" + assert gen_instance_for_job.log_level == DEBUG + # Clean up: unset the environment variable + # TODO might need to set it to info here? + environ.pop("SMARTSIM_LOG_LEVEL", None) + + +def test_log_file_path(gen_instance_for_job): + """Test if the log_file property returns the correct path.""" + expected_path = osp.join(gen_instance_for_job.path, "smartsim_params.txt") + assert gen_instance_for_job.log_file == expected_path + + +def test_generate_job_directory(gen_instance_for_job): + gen_instance_for_job.generate_experiment() + assert osp.isdir(gen_instance_for_job.path) + assert osp.isdir(gen_instance_for_job.log_path) + + +def test_full_exp_generate_job_directory(test_dir, job_instance): + no_op_dispatch = Dispatcher() + no_op_dispatch.dispatch(SlurmArgBuilder, to_launcher=NoOpLauncher) + no_op_exp = Experiment( + name="No-Op-Exp", exp_path=test_dir, settings_dispatcher=no_op_dispatch + ) + job_execution_path = no_op_exp._generate(job_instance) + assert osp.isdir(job_execution_path) + +def test_generate_ensemble_directory(test_dir, wlmutils): + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + for job in job_list: + run_ID = "temp_run" + gen = Generator(gen_path=test_dir, run_ID=run_ID, job=job) + gen.generate_experiment() + assert osp.isdir(gen.path) \ No newline at end of file From 4e9f5b28b662494920c60eec66a52bc860791426 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Thu, 18 Jul 2024 02:25:55 -0500 Subject: [PATCH 22/82] Dispatcher takes a format function with launcher like --- setup.py | 4 +- smartsim/error/errors.py | 4 + smartsim/experiment.py | 49 ++++++------- smartsim/settings/dispatch.py | 134 ++++++++++++++++++++++++++++------ 4 files changed, 140 insertions(+), 51 deletions(-) diff --git a/setup.py b/setup.py index 05b6ef70b..d820563d9 100644 --- a/setup.py +++ b/setup.py @@ -179,7 +179,8 @@ def has_ext_modules(_placeholder): "pydantic==1.10.14", "pyzmq>=25.1.2", "pygithub>=2.3.0", - "numpy<2" + "numpy<2", + "typing_extensions>=4.1.0", ] # Add SmartRedis at specific version @@ -203,7 +204,6 @@ def has_ext_modules(_placeholder): "types-tqdm", "types-tensorflow==2.12.0.9", "types-setuptools", - "typing_extensions>=4.1.0", ], # see smartsim/_core/_install/buildenv.py for more details **versions.ml_extras_required(), diff --git a/smartsim/error/errors.py b/smartsim/error/errors.py index 8500e4947..3f32bd3f0 100644 --- a/smartsim/error/errors.py +++ b/smartsim/error/errors.py @@ -112,6 +112,10 @@ class LauncherUnsupportedFeature(LauncherError): """Raised when the launcher does not support a given method""" +class LauncherNotFoundError(LauncherError): + """A requested launcher could not be found""" + + class AllocationError(LauncherError): """Raised when there is a problem with the user WLM allocation""" diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 8a4ed42f6..c2e56d19a 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -38,8 +38,7 @@ from tabulate import tabulate from smartsim._core.config import CONFIG -from smartsim._core.utils.helpers import first -from smartsim.error.errors import SSUnsupportedError +from smartsim.error.errors import LauncherNotFoundError, SSUnsupportedError from smartsim.settings.dispatch import default_dispatcher from smartsim.status import SmartSimStatus @@ -58,11 +57,8 @@ if t.TYPE_CHECKING: from smartsim.launchable.job import Job - from smartsim.settings.builders.launchArgBuilder import ( - ExecutableLike, - LaunchArgBuilder, - ) - from smartsim.settings.dispatch import Dispatcher, LauncherLike + from smartsim.settings.builders.launchArgBuilder import LaunchArgBuilder + from smartsim.settings.dispatch import Dispatcher, ExecutableLike, LauncherLike from smartsim.types import LaunchedJobID logger = get_logger(__name__) @@ -194,27 +190,30 @@ def start_jobs(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: ) def _start(job: Job) -> LaunchedJobID: - builder: LaunchArgBuilder[t.Any] = job.launch_settings.launch_args - launcher_type = self._dispatcher.get_launcher_for(builder) - launcher = first( - lambda launcher: type(launcher) is launcher_type, - self._active_launchers, - ) - if launcher is None: - launcher = launcher_type.create(self) - self._active_launchers.add(launcher) + args = job.launch_settings.launch_args + env = job.launch_settings.env_vars # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - # FIXME: Opting out of type check here. Fix this later!! - # TODO: Very much dislike that we have to pass in attrs off of `job` - # into `builder`, which is itself an attr of an attr of `job`. - # Why is `Job` not generic based on launch arg builder? - # FIXME: Remove this dangerous cast after `SmartSimEntity` conforms - # to protocol + # FIXME: Remove this cast after `SmartSimEntity` conforms to + # protocol. For now, live with the "dangerous" type cast # --------------------------------------------------------------------- - exe_like = t.cast("ExecutableLike", job.entity) - finalized = builder.finalize(exe_like, job.launch_settings.env_vars) + exe = t.cast("ExecutableLike", job.entity) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - return launcher.start(finalized) + dispatch = self._dispatcher.get_dispatch(args) + try: + launch_config = dispatch.configure_first_compatible_launcher( + from_available_launchers=self._active_launchers, + with_settings=args, + ) + except LauncherNotFoundError: + launch_config = dispatch.create_new_launcher_configuration( + for_experiment=self, with_settings=args + ) + # Save the underlying launcher instance. That way we do not need to + # spin up a launcher instance for each individual job, and it makes + # it easier to monitor job statuses + # pylint: disable-next=protected-access + self._active_launchers.add(launch_config._adapted_launcher) + return launch_config.start(exe, env) return tuple(map(_start, jobs)) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index f2c945f3a..71343438e 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -26,21 +26,33 @@ from __future__ import annotations +import dataclasses import subprocess as sp import typing as t import uuid +from typing_extensions import Self, TypeVarTuple, Unpack + from smartsim._core.utils import helpers +from smartsim.error import errors from smartsim.types import LaunchedJobID if t.TYPE_CHECKING: - from typing_extensions import Self from smartsim.experiment import Experiment from smartsim.settings.builders import LaunchArgBuilder + _T = t.TypeVar("_T") +_Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) +_TDispatchable = t.TypeVar("_TDispatchable", bound="LaunchArgBuilder") +_EnvironMappingType: t.TypeAlias = t.Mapping[str, str | None] +_FormatterType: t.TypeAlias = t.Callable[ + [_TDispatchable, "ExecutableLike", _EnvironMappingType], _T +] +_LaunchConfigType: t.TypeAlias = "_LauncherAdapter[ExecutableLike, _EnvironMappingType]" +_UnkownType: t.TypeAlias = t.NoReturn @t.final @@ -53,7 +65,8 @@ def __init__( self, *, dispatch_registry: ( - t.Mapping[type[LaunchArgBuilder[t.Any]], type[LauncherLike[t.Any]]] | None + t.Mapping[type[LaunchArgBuilder], _DispatchRegistration[t.Any, t.Any]] + | None ) = None, ) -> None: self._dispatch_registry = ( @@ -69,38 +82,48 @@ def dispatch( self, args: None = ..., *, + with_format: _FormatterType[_TDispatchable, _T], to_launcher: type[LauncherLike[_T]], allow_overwrite: bool = ..., - ) -> t.Callable[[type[LaunchArgBuilder[_T]]], type[LaunchArgBuilder[_T]]]: ... + ) -> t.Callable[[type[_TDispatchable]], type[_TDispatchable]]: ... @t.overload def dispatch( self, - args: type[LaunchArgBuilder[_T]], + args: type[_TDispatchable], *, + with_format: _FormatterType[_TDispatchable, _T], to_launcher: type[LauncherLike[_T]], allow_overwrite: bool = ..., ) -> None: ... def dispatch( self, - args: type[LaunchArgBuilder[_T]] | None = None, + args: type[_TDispatchable] | None = None, *, + with_format: _FormatterType[_TDispatchable, _T], to_launcher: type[LauncherLike[_T]], allow_overwrite: bool = False, - ) -> t.Callable[[type[LaunchArgBuilder[_T]]], type[LaunchArgBuilder[_T]]] | None: + ) -> t.Callable[[type[_TDispatchable]], type[_TDispatchable]] | None: """A type safe way to add a mapping of settings builder to launcher to handle the settings at launch time. """ + err_msg: str | None = None + if getattr(to_launcher, "_is_protocol", False): + err_msg = f"Cannot dispatch to protocol class `{to_launcher.__name__}`" + elif getattr(to_launcher, "__abstractmethods__", frozenset()): + err_msg = f"Cannot dispatch to abstract class `{to_launcher.__name__}`" + if err_msg is not None: + raise TypeError(err_msg) - def register( - args_: type[LaunchArgBuilder[_T]], / - ) -> type[LaunchArgBuilder[_T]]: + def register(args_: type[_TDispatchable], /) -> type[_TDispatchable]: if args_ in self._dispatch_registry and not allow_overwrite: - launcher_type = self._dispatch_registry[args_] + launcher_type = self._dispatch_registry[args_].launcher_type raise TypeError( f"{args_.__name__} has already been registered to be " f"launched with {launcher_type}" ) - self._dispatch_registry[args_] = to_launcher + self._dispatch_registry[args_] = _DispatchRegistration( + with_format, to_launcher + ) return args_ if args is not None: @@ -108,27 +131,90 @@ def register( return None return register - def get_launcher_for( - self, args: LaunchArgBuilder[_T] | type[LaunchArgBuilder[_T]], / - ) -> type[LauncherLike[_T]]: + def get_dispatch( + self, args: _TDispatchable | type[_TDispatchable] + ) -> _DispatchRegistration[_TDispatchable, _UnkownType]: """Find a type of launcher that is registered as being able to launch the output of the provided builder """ if not isinstance(args, type): args = type(args) - launcher_type = self._dispatch_registry.get(args, None) - if launcher_type is None: + dispatch = self._dispatch_registry.get(args, None) + if dispatch is None: raise TypeError( - f"{type(self).__name__} {self} has no launcher type to " - f"dispatch to for argument builder of type {args}" + f"No dispatch for `{type(args).__name__}` has been registered " + f"has been registered with {type(self).__name__} `{self}`" ) # Note the sleight-of-hand here: we are secretly casting a type of - # `LauncherLike[Any]` to `LauncherLike[_T]`. This is safe to do if all - # entries in the mapping were added using a type safe method (e.g. - # `Dispatcher.dispatch`), but if a user were to supply a custom - # dispatch registry or otherwise modify the registry THIS IS NOT - # NECESSARILY 100% TYPE SAFE!! - return launcher_type + # `_DispatchRegistration[Any, Any]` -> + # `_DispatchRegistration[_TDispatchable, _T]`. + # where `_T` is unbound! + # + # This is safe to do if all entries in the mapping were added using a + # type safe method (e.g. `Dispatcher.dispatch`), but if a user were to + # supply a custom dispatch registry or otherwise modify the registry + # this is not necessarily 100% type safe!! + return dispatch + + +@t.final +@dataclasses.dataclass(frozen=True) +class _DispatchRegistration(t.Generic[_TDispatchable, _T]): + formatter: _FormatterType[_TDispatchable, _T] + launcher_type: type[LauncherLike[_T]] + + def _is_compatible_launcher(self, launcher) -> bool: + return type(launcher) is self.launcher_type + + def create_new_launcher_configuration( + self, for_experiment: Experiment, with_settings: _TDispatchable + ) -> _LaunchConfigType: + launcher = self.launcher_type.create(for_experiment) + return self.create_adapter_from_launcher(launcher, with_settings) + + def create_adapter_from_launcher( + self, launcher: LauncherLike[_T], settings: _TDispatchable + ) -> _LaunchConfigType: + if not self._is_compatible_launcher(launcher): + raise TypeError( + f"Cannot create launcher adapter from launcher `{launcher}` " + f"of type `{type(launcher)}`; expected launcher of type " + f"exactly `{self.launcher_type}`" + ) + + def format_(exe: ExecutableLike, env: _EnvironMappingType) -> _T: + return self.formatter(settings, exe, env) + + return _LauncherAdapter(launcher, format_) + + def configure_first_compatible_launcher( + self, + with_settings: _TDispatchable, + from_available_launchers: t.Iterable[LauncherLike[t.Any]], + ) -> _LaunchConfigType: + launcher = helpers.first(self._is_compatible_launcher, from_available_launchers) + if launcher is None: + raise errors.LauncherNotFoundError( + f"No launcher of exactly type `{self.launcher_type.__name__}` " + "could be found from provided launchers" + ) + return self.create_adapter_from_launcher(launcher, with_settings) + + +@t.final +class _LauncherAdapter(t.Generic[Unpack[_Ts]]): + def __init__( + self, launcher: LauncherLike[_T], map_: t.Callable[[Unpack[_Ts]], _T] + ) -> None: + # NOTE: We need to cast off the `_T` -> `Any` in the `__init__` + # signature to hide the transform from users of this class. If + # possible, try not to expose outside of protected methods! + self._adapt: t.Callable[[Unpack[_Ts]], t.Any] = map_ + self._adapted_launcher: LauncherLike[t.Any] = launcher + + def start(self, *args: Unpack[_Ts]) -> LaunchedJobID: + payload = self._adapt(*args) + return self._adapted_launcher.start(payload) default_dispatcher: t.Final = Dispatcher() From 0b9ec1a46c07a5d752652974d9250e2cf1f7f3dd Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Thu, 18 Jul 2024 02:27:32 -0500 Subject: [PATCH 23/82] Re-wrire up default dispatcher --- .../_core/launcher/dragon/dragonLauncher.py | 30 +++++++++++++-- smartsim/settings/builders/launch/alps.py | 19 ++-------- smartsim/settings/builders/launch/dragon.py | 27 +------------ smartsim/settings/builders/launch/local.py | 14 ++----- smartsim/settings/builders/launch/lsf.py | 19 ++-------- smartsim/settings/builders/launch/mpi.py | 38 +++---------------- smartsim/settings/builders/launch/pals.py | 19 ++-------- smartsim/settings/builders/launch/slurm.py | 19 ++-------- .../settings/builders/launchArgBuilder.py | 11 +----- smartsim/settings/dispatch.py | 28 ++++++++++++++ smartsim/settings/launchSettings.py | 6 +-- 11 files changed, 80 insertions(+), 150 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index e137037c1..74e22e160 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -341,7 +341,31 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: # circular import # ----------------------------------------------------------------------------- from smartsim.settings.builders.launch.dragon import DragonArgBuilder -from smartsim.settings.dispatch import dispatch - -dispatch(DragonArgBuilder, to_launcher=DragonLauncher) +from smartsim.settings.dispatch import ExecutableLike, dispatch + + +def _as_run_request_view( + run_req_args: DragonArgBuilder, exe: ExecutableLike, env: t.Mapping[str, str | None] +) -> DragonRunRequestView: + exe_, *args = exe.as_program_arguments() + return DragonRunRequestView( + exe=exe_, + exe_args=args, + # FIXME: Currently this is hard coded because the schema requires + # it, but in future, it is almost certainly necessary that + # this will need to be injected by the user or by us to have + # the command execute next to any generated files. A similar + # problem exists for the other settings. + # TODO: Find a way to inject this path + path=os.getcwd(), + env=env, + # TODO: Not sure how this info is injected + name=None, + output_file=None, + error_file=None, + **run_req_args._launch_args, + ) + + +dispatch(DragonArgBuilder, with_format=_as_run_request_view, to_launcher=DragonLauncher) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index de2d7b91d..f325777e4 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -29,20 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch +from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder -if t.TYPE_CHECKING: - from smartsim.settings.builders.launchArgBuilder import ExecutableLike - logger = get_logger(__name__) -@dispatch(to_launcher=ShellLauncher) -class AprunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): +@dispatch(with_format=shell_format(run_command="aprun"), to_launcher=ShellLauncher) +class AprunArgBuilder(LaunchArgBuilder): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wdir"} @@ -218,13 +215,3 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value - - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ( - "aprun", - *(self.format_launch_args() or ()), - "--", - *exe.as_program_arguments(), - ) diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 8fb2ebc48..3cbb1c6d0 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -29,7 +29,6 @@ import os import typing as t -from smartsim._core.schemas.dragonRequests import DragonRunRequestView from smartsim.log import get_logger from ...common import StringArgument, set_check_input @@ -37,12 +36,12 @@ from ..launchArgBuilder import LaunchArgBuilder if t.TYPE_CHECKING: - from smartsim.settings.builders.launchArgBuilder import ExecutableLike + from smartsim.settings.dispatch import ExecutableLike logger = get_logger(__name__) -class DragonArgBuilder(LaunchArgBuilder[DragonRunRequestView]): +class DragonArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Dragon.value @@ -67,25 +66,3 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value - - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> DragonRunRequestView: - exe_, *args = exe.as_program_arguments() - return DragonRunRequestView( - exe=exe_, - exe_args=args, - # FIXME: Currently this is hard coded because the schema requires - # it, but in future, it is almost certainly necessary that - # this will need to be injected by the user or by us to have - # the command execute next to any generated files. A similar - # problem exists for the other settings. - # TODO: Find a way to inject this path - path=os.getcwd(), - env=env, - # TODO: Not sure how this info is injected - name=None, - output_file=None, - error_file=None, - **self._launch_args, - ) diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index aeb018bc4..21fb71c8a 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -29,20 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch +from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder -if t.TYPE_CHECKING: - from smartsim.settings.builders.launchArgBuilder import ExecutableLike - logger = get_logger(__name__) -@dispatch(to_launcher=ShellLauncher) -class LocalArgBuilder(LaunchArgBuilder[t.Sequence[str]]): +@dispatch(with_format=shell_format(run_command=None), to_launcher=ShellLauncher) +class LocalArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Local.value @@ -77,8 +74,3 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value - - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return exe.as_program_arguments() diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index bec63a802..13a32fd73 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -29,20 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch +from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder -if t.TYPE_CHECKING: - from smartsim.settings.builders.launchArgBuilder import ExecutableLike - logger = get_logger(__name__) -@dispatch(to_launcher=ShellLauncher) -class JsrunArgBuilder(LaunchArgBuilder[t.Sequence[str]]): +@dispatch(with_format=shell_format(run_command="jsrun"), to_launcher=ShellLauncher) +class JsrunArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Lsf.value @@ -120,13 +117,3 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value - - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ( - "jsrun", - *(self.format_launch_args() or ()), - "--", - *exe.as_program_arguments(), - ) diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 7ce79fbc3..ea24564da 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -29,19 +29,16 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch +from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder -if t.TYPE_CHECKING: - from smartsim.settings.builders.launchArgBuilder import ExecutableLike - logger = get_logger(__name__) -class _BaseMPIArgBuilder(LaunchArgBuilder[t.Sequence[str]]): +class _BaseMPIArgBuilder(LaunchArgBuilder): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wd", "wdir"} @@ -218,47 +215,22 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value -@dispatch(to_launcher=ShellLauncher) +@dispatch(with_format=shell_format(run_command="mpirun"), to_launcher=ShellLauncher) class MpiArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpirun.value - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ("mpirun", *self.format_launch_args(), "--", *exe.as_program_arguments()) - -@dispatch(to_launcher=ShellLauncher) +@dispatch(with_format=shell_format(run_command="mpiexec"), to_launcher=ShellLauncher) class MpiexecArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpiexec.value - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ( - "mpiexec", - *self.format_launch_args(), - "--", - *exe.as_program_arguments(), - ) - -@dispatch(to_launcher=ShellLauncher) +@dispatch(with_format=shell_format(run_command="orterun"), to_launcher=ShellLauncher) class OrteArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Orterun.value - - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ( - "orterun", - *self.format_launch_args(), - "--", - *exe.as_program_arguments(), - ) diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index 1b2ed17bf..4f2155c1f 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -29,20 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch +from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder -if t.TYPE_CHECKING: - from smartsim.settings.builders.launchArgBuilder import ExecutableLike - logger = get_logger(__name__) -@dispatch(to_launcher=ShellLauncher) -class PalsMpiexecArgBuilder(LaunchArgBuilder[t.Sequence[str]]): +@dispatch(with_format=shell_format(run_command="mpiexec"), to_launcher=ShellLauncher) +class PalsMpiexecArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Pals.value @@ -154,13 +151,3 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value - - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ( - "mpiexec", - *(self.format_launch_args() or ()), - "--", - *exe.as_program_arguments(), - ) diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index db2d673cb..907a6da6c 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -31,20 +31,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch +from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder -if t.TYPE_CHECKING: - from smartsim.settings.builders.launchArgBuilder import ExecutableLike - logger = get_logger(__name__) -@dispatch(to_launcher=ShellLauncher) -class SlurmArgBuilder(LaunchArgBuilder[t.Sequence[str]]): +@dispatch(with_format=shell_format(run_command="srun"), to_launcher=ShellLauncher) +class SlurmArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Slurm.value @@ -320,13 +317,3 @@ def set(self, key: str, value: str | None) -> None: if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value - - def finalize( - self, exe: ExecutableLike, env: t.Mapping[str, str | None] - ) -> t.Sequence[str]: - return ( - "srun", - *(self.format_launch_args() or ()), - "--", - *exe.as_program_arguments(), - ) diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py index b125046cd..2c09dd2e8 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/builders/launchArgBuilder.py @@ -40,7 +40,7 @@ _T = t.TypeVar("_T") -class LaunchArgBuilder(ABC, t.Generic[_T]): +class LaunchArgBuilder(ABC): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the responsibility of child classes for each launcher to translate @@ -58,10 +58,6 @@ def launcher_str(self) -> str: def set(self, arg: str, val: str | None) -> None: """Set the launch arguments""" - @abstractmethod - def finalize(self, exe: ExecutableLike, env: t.Mapping[str, str | None]) -> _T: - """Prepare an entity for launch using the built options""" - def format_launch_args(self) -> t.Union[t.List[str], None]: """Build formatted launch arguments""" logger.warning( @@ -95,8 +91,3 @@ def format_env_vars( def __str__(self) -> str: # pragma: no-cover string = f"\nLaunch Arguments:\n{fmt_dict(self._launch_args)}" return string - - -class ExecutableLike(t.Protocol): - @abstractmethod - def as_program_arguments(self) -> t.Sequence[str]: ... diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 71343438e..a69a6587a 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -230,12 +230,40 @@ def create_job_id() -> LaunchedJobID: return LaunchedJobID(str(uuid.uuid4())) +class ExecutableLike(t.Protocol): + def as_program_arguments(self) -> t.Sequence[str]: ... + + class LauncherLike(t.Protocol[_T_contra]): def start(self, launchable: _T_contra) -> LaunchedJobID: ... @classmethod def create(cls, exp: Experiment) -> Self: ... +# TODO: This is just a nice helper function that I am using for the time being +# to wire everything up! In reality it might be a bit too confusing and +# meta-program-y for production code. Check with the core team to see +# what they think!! +def shell_format( + run_command: str | None, +) -> _FormatterType[LaunchArgBuilder, t.Sequence[str]]: + def impl( + args: LaunchArgBuilder, exe: ExecutableLike, env: _EnvironMappingType + ) -> t.Sequence[str]: + return ( + ( + run_command, + *(args.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + ) + if run_command is not None + else exe.as_program_arguments() + ) + + return impl + + class ShellLauncher: """Mock launcher for launching/tracking simple shell commands""" diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 9078a04d9..dec6034d8 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -67,7 +67,7 @@ def launcher(self) -> str: return self._launcher.value @property - def launch_args(self) -> LaunchArgBuilder[t.Any]: + def launch_args(self) -> LaunchArgBuilder: """Return the launch argument translator.""" return self._arg_builder @@ -88,9 +88,7 @@ def env_vars(self, value: dict[str, str | None]) -> None: """Set the environment variables.""" self._env_vars = copy.deepcopy(value) - def _get_arg_builder( - self, launch_args: StringArgument | None - ) -> LaunchArgBuilder[t.Any]: + def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder: """Map the Launcher to the LaunchArgBuilder""" if self._launcher == LauncherType.Slurm: return SlurmArgBuilder(launch_args) From 6d5a3c46707e4441670b71369147cae104167df4 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Thu, 18 Jul 2024 22:43:22 -0500 Subject: [PATCH 24/82] Add tests for new dispatch API, make old tests pass --- smartsim/settings/builders/launch/alps.py | 3 +- smartsim/settings/builders/launch/local.py | 3 +- smartsim/settings/builders/launch/lsf.py | 3 +- smartsim/settings/builders/launch/mpi.py | 9 +- smartsim/settings/builders/launch/pals.py | 3 +- smartsim/settings/builders/launch/slurm.py | 3 +- tests/temp_tests/test_settings/conftest.py | 9 +- .../test_settings/test_alpsLauncher.py | 7 +- .../temp_tests/test_settings/test_dispatch.py | 333 ++++++++++++++++-- .../test_settings/test_dragonLauncher.py | 9 +- .../test_settings/test_localLauncher.py | 7 +- .../test_settings/test_lsfLauncher.py | 4 +- .../test_settings/test_mpiLauncher.py | 21 +- .../test_settings/test_palsLauncher.py | 7 +- .../test_settings/test_slurmLauncher.py | 7 +- .../test_settings/test_slurmScheduler.py | 1 - 16 files changed, 357 insertions(+), 72 deletions(-) diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index f325777e4..09d5931ac 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -36,9 +36,10 @@ from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) +_format_aprun_command = shell_format(run_command="aprun") -@dispatch(with_format=shell_format(run_command="aprun"), to_launcher=ShellLauncher) +@dispatch(with_format=_format_aprun_command, to_launcher=ShellLauncher) class AprunArgBuilder(LaunchArgBuilder): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index 21fb71c8a..7002a6831 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -36,9 +36,10 @@ from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) +_format_local_command = shell_format(run_command=None) -@dispatch(with_format=shell_format(run_command=None), to_launcher=ShellLauncher) +@dispatch(with_format=_format_local_command, to_launcher=ShellLauncher) class LocalArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index 13a32fd73..ec99d51b9 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -36,9 +36,10 @@ from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) +_format_jsrun_command = shell_format(run_command="jsrun") -@dispatch(with_format=shell_format(run_command="jsrun"), to_launcher=ShellLauncher) +@dispatch(with_format=_format_jsrun_command, to_launcher=ShellLauncher) class JsrunArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index ea24564da..139096010 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -36,6 +36,9 @@ from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) +_format_mpirun_command = shell_format("mpirun") +_format_mpiexec_command = shell_format("mpiexec") +_format_orterun_command = shell_format("orterun") class _BaseMPIArgBuilder(LaunchArgBuilder): @@ -215,21 +218,21 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value -@dispatch(with_format=shell_format(run_command="mpirun"), to_launcher=ShellLauncher) +@dispatch(with_format=_format_mpirun_command, to_launcher=ShellLauncher) class MpiArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpirun.value -@dispatch(with_format=shell_format(run_command="mpiexec"), to_launcher=ShellLauncher) +@dispatch(with_format=_format_mpiexec_command, to_launcher=ShellLauncher) class MpiexecArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpiexec.value -@dispatch(with_format=shell_format(run_command="orterun"), to_launcher=ShellLauncher) +@dispatch(with_format=_format_orterun_command, to_launcher=ShellLauncher) class OrteArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index 4f2155c1f..1e7ed814e 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -36,9 +36,10 @@ from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) +_format_mpiexec_command = shell_format(run_command="mpiexec") -@dispatch(with_format=shell_format(run_command="mpiexec"), to_launcher=ShellLauncher) +@dispatch(with_format=_format_mpiexec_command, to_launcher=ShellLauncher) class PalsMpiexecArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 907a6da6c..72058f983 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -38,9 +38,10 @@ from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) +_format_srun_command = shell_format(run_command="srun") -@dispatch(with_format=shell_format(run_command="srun"), to_launcher=ShellLauncher) +@dispatch(with_format=_format_srun_command, to_launcher=ShellLauncher) class SlurmArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/tests/temp_tests/test_settings/conftest.py b/tests/temp_tests/test_settings/conftest.py index ebf361e97..72061264f 100644 --- a/tests/temp_tests/test_settings/conftest.py +++ b/tests/temp_tests/test_settings/conftest.py @@ -24,8 +24,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from unittest.mock import Mock - import pytest from smartsim.settings import dispatch @@ -34,7 +32,7 @@ @pytest.fixture def echo_executable_like(): - class _ExeLike(launch.ExecutableLike): + class _ExeLike(dispatch.ExecutableLike): def as_program_arguments(self): return ("echo", "hello", "world") @@ -44,13 +42,10 @@ def as_program_arguments(self): @pytest.fixture def settings_builder(): class _SettingsBuilder(launch.LaunchArgBuilder): + def set(self, arg, val): ... def launcher_str(self): return "Mock Settings Builder" - def set(self, arg, val): ... - def finalize(self, exe, env): - return Mock() - yield _SettingsBuilder({}) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 7fa95cb6d..5ac2f8e11 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.alps import AprunArgBuilder +from smartsim.settings.builders.launch.alps import ( + AprunArgBuilder, + _format_aprun_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -183,5 +186,5 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = AprunArgBuilder(args).finalize(echo_executable_like, {}) + cmd = _format_aprun_command(AprunArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_dispatch.py b/tests/temp_tests/test_settings/test_dispatch.py index ccd1e81cd..78c44ad54 100644 --- a/tests/temp_tests/test_settings/test_dispatch.py +++ b/tests/temp_tests/test_settings/test_dispatch.py @@ -24,31 +24,56 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import abc import contextlib +import dataclasses +import io import pytest +from smartsim.error import errors from smartsim.settings import dispatch pytestmark = pytest.mark.group_a +FORMATTED = object() -def test_declaritive_form_dispatch_declaration(launcher_like, settings_builder): + +def format_fn(args, exe, env): + return FORMATTED + + +@pytest.fixture +def expected_dispatch_registry(launcher_like, settings_builder): + yield { + type(settings_builder): dispatch._DispatchRegistration( + format_fn, type(launcher_like) + ) + } + + +def test_declaritive_form_dispatch_declaration( + launcher_like, settings_builder, expected_dispatch_registry +): d = dispatch.Dispatcher() - assert type(settings_builder) == d.dispatch(to_launcher=type(launcher_like))( - type(settings_builder) - ) - assert d._dispatch_registry == {type(settings_builder): type(launcher_like)} + assert type(settings_builder) == d.dispatch( + with_format=format_fn, to_launcher=type(launcher_like) + )(type(settings_builder)) + assert d._dispatch_registry == expected_dispatch_registry -def test_imperative_form_dispatch_declaration(launcher_like, settings_builder): +def test_imperative_form_dispatch_declaration( + launcher_like, settings_builder, expected_dispatch_registry +): d = dispatch.Dispatcher() - assert None == d.dispatch(type(settings_builder), to_launcher=type(launcher_like)) - assert d._dispatch_registry == {type(settings_builder): type(launcher_like)} + assert None == d.dispatch( + type(settings_builder), to_launcher=type(launcher_like), with_format=format_fn + ) + assert d._dispatch_registry == expected_dispatch_registry def test_dispatchers_from_same_registry_do_not_cross_polute( - launcher_like, settings_builder + launcher_like, settings_builder, expected_dispatch_registry ): some_starting_registry = {} d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) @@ -60,12 +85,16 @@ def test_dispatchers_from_same_registry_do_not_cross_polute( d1._dispatch_registry is not d2._dispatch_registry is not some_starting_registry ) - d2.dispatch(type(settings_builder), to_launcher=type(launcher_like)) + d2.dispatch( + type(settings_builder), with_format=format_fn, to_launcher=type(launcher_like) + ) assert d1._dispatch_registry == {} - assert d2._dispatch_registry == {type(settings_builder): type(launcher_like)} + assert d2._dispatch_registry == expected_dispatch_registry -def test_copied_dispatchers_do_not_cross_pollute(launcher_like, settings_builder): +def test_copied_dispatchers_do_not_cross_pollute( + launcher_like, settings_builder, expected_dispatch_registry +): some_starting_registry = {} d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) d2 = d1.copy() @@ -76,70 +105,304 @@ def test_copied_dispatchers_do_not_cross_pollute(launcher_like, settings_builder d1._dispatch_registry is not d2._dispatch_registry is not some_starting_registry ) - d2.dispatch(type(settings_builder), to_launcher=type(launcher_like)) + d2.dispatch( + type(settings_builder), to_launcher=type(launcher_like), with_format=format_fn + ) assert d1._dispatch_registry == {} - assert d2._dispatch_registry == {type(settings_builder): type(launcher_like)} + assert d2._dispatch_registry == expected_dispatch_registry @pytest.mark.parametrize( "add_dispatch, expected_ctx", ( pytest.param( - lambda d, s, l: d.dispatch(s, to_launcher=l), + lambda d, s, l: d.dispatch(s, to_launcher=l, with_format=format_fn), pytest.raises(TypeError, match="has already been registered"), id="Imperative -- Disallowed implicitly", ), pytest.param( - lambda d, s, l: d.dispatch(s, to_launcher=l, allow_overwrite=True), + lambda d, s, l: d.dispatch( + s, to_launcher=l, with_format=format_fn, allow_overwrite=True + ), contextlib.nullcontext(), id="Imperative -- Allowed with flag", ), pytest.param( - lambda d, s, l: d.dispatch(to_launcher=l)(s), + lambda d, s, l: d.dispatch(to_launcher=l, with_format=format_fn)(s), pytest.raises(TypeError, match="has already been registered"), id="Declarative -- Disallowed implicitly", ), pytest.param( - lambda d, s, l: d.dispatch(to_launcher=l, allow_overwrite=True)(s), + lambda d, s, l: d.dispatch( + to_launcher=l, with_format=format_fn, allow_overwrite=True + )(s), contextlib.nullcontext(), id="Declarative -- Allowed with flag", ), ), ) def test_dispatch_overwriting( - add_dispatch, expected_ctx, launcher_like, settings_builder + add_dispatch, + expected_ctx, + launcher_like, + settings_builder, + expected_dispatch_registry, ): - registry = {type(settings_builder): type(launcher_like)} - d = dispatch.Dispatcher(dispatch_registry=registry) + d = dispatch.Dispatcher(dispatch_registry=expected_dispatch_registry) with expected_ctx: add_dispatch(d, type(settings_builder), type(launcher_like)) @pytest.mark.parametrize( - "map_settings", + "type_or_instance", ( - pytest.param(type, id="From settings type"), - pytest.param(lambda s: s, id="From settings instance"), + pytest.param(type, id="type"), + pytest.param(lambda x: x, id="instance"), ), ) -def test_dispatch_can_retrieve_launcher_to_dispatch_to( - map_settings, launcher_like, settings_builder +def test_dispatch_can_retrieve_dispatch_info_from_dispatch_registry( + expected_dispatch_registry, launcher_like, settings_builder, type_or_instance ): - registry = {type(settings_builder): type(launcher_like)} - d = dispatch.Dispatcher(dispatch_registry=registry) - assert type(launcher_like) == d.get_launcher_for(map_settings(settings_builder)) + d = dispatch.Dispatcher(dispatch_registry=expected_dispatch_registry) + assert dispatch._DispatchRegistration( + format_fn, type(launcher_like) + ) == d.get_dispatch(type_or_instance(settings_builder)) @pytest.mark.parametrize( - "map_settings", + "type_or_instance", ( - pytest.param(type, id="From settings type"), - pytest.param(lambda s: s, id="From settings instance"), + pytest.param(type, id="type"), + pytest.param(lambda x: x, id="instance"), ), ) def test_dispatch_raises_if_settings_type_not_registered( - map_settings, launcher_like, settings_builder + settings_builder, type_or_instance ): d = dispatch.Dispatcher(dispatch_registry={}) - with pytest.raises(TypeError, match="no launcher type to dispatch to"): - d.get_launcher_for(map_settings(settings_builder)) + with pytest.raises( + TypeError, match="No dispatch for `.+?(?=`)` has been registered" + ): + d.get_dispatch(type_or_instance(settings_builder)) + + +class LauncherABC(abc.ABC): + @abc.abstractmethod + def start(self, launchable): ... + @classmethod + @abc.abstractmethod + def create(cls, exp): ... + + +class PartImplLauncherABC(LauncherABC): + def start(self, launchable): + return dispatch.create_job_id() + + +class FullImplLauncherABC(PartImplLauncherABC): + @classmethod + def create(cls, exp): + return cls() + + +@pytest.mark.parametrize( + "cls, ctx", + ( + pytest.param( + dispatch.LauncherLike, + pytest.raises(TypeError, match="Cannot dispatch to protocol"), + id="Cannot dispatch to protocol class", + ), + pytest.param( + "launcher_like", + contextlib.nullcontext(None), + id="Can dispatch to protocol implementation", + ), + pytest.param( + LauncherABC, + pytest.raises(TypeError, match="Cannot dispatch to abstract class"), + id="Cannot dispatch to abstract class", + ), + pytest.param( + PartImplLauncherABC, + pytest.raises(TypeError, match="Cannot dispatch to abstract class"), + id="Cannot dispatch to partially implemented abstract class", + ), + pytest.param( + FullImplLauncherABC, + contextlib.nullcontext(None), + id="Can dispatch to fully implemented abstract class", + ), + ), +) +def test_register_dispatch_to_launcher_types(request, cls, ctx): + if isinstance(cls, str): + cls = request.getfixturevalue(cls) + d = dispatch.Dispatcher() + with ctx: + d.dispatch(to_launcher=cls, with_format=format_fn) + + +@dataclasses.dataclass +class BufferWriterLauncher(dispatch.LauncherLike[list[str]]): + buf: io.StringIO + + @classmethod + def create(cls, exp): + return cls(io.StringIO()) + + def start(self, strs): + self.buf.writelines(f"{s}\n" for s in strs) + return dispatch.create_job_id() + + +class BufferWriterLauncherSubclass(BufferWriterLauncher): ... + + +@pytest.fixture +def buffer_writer_dispatch(): + stub_format_fn = lambda *a, **kw: ["some", "strings"] + return dispatch._DispatchRegistration(stub_format_fn, BufferWriterLauncher) + + +@pytest.mark.parametrize( + "input_, map_, expected", + ( + pytest.param( + ["list", "of", "strings"], + lambda xs: xs, + ["list\n", "of\n", "strings\n"], + id="[str] -> [str]", + ), + pytest.param( + "words on new lines", + lambda x: x.split(), + ["words\n", "on\n", "new\n", "lines\n"], + id="str -> [str]", + ), + pytest.param( + range(1, 4), + lambda xs: [str(x) for x in xs], + ["1\n", "2\n", "3\n"], + id="[int] -> [str]", + ), + ), +) +def test_launcher_adapter_correctly_adapts_input_to_launcher(input_, map_, expected): + buf = io.StringIO() + adapter = dispatch._LauncherAdapter(BufferWriterLauncher(buf), map_) + adapter.start(input_) + buf.seek(0) + assert buf.readlines() == expected + + +@pytest.mark.parametrize( + "launcher_instance, ctx", + ( + pytest.param( + BufferWriterLauncher(io.StringIO()), + contextlib.nullcontext(None), + id="Correctly configures expected launcher", + ), + pytest.param( + BufferWriterLauncherSubclass(io.StringIO()), + pytest.raises( + TypeError, + match="^Cannot create launcher adapter.*expected launcher of type .+$", + ), + id="Errors if launcher types are disparate", + ), + pytest.param( + "launcher_like", + pytest.raises( + TypeError, + match="^Cannot create launcher adapter.*expected launcher of type .+$", + ), + id="Errors if types are not an exact match", + ), + ), +) +def test_dispatch_registration_can_configure_adapter_for_existing_launcher_instance( + request, settings_builder, buffer_writer_dispatch, launcher_instance, ctx +): + if isinstance(launcher_instance, str): + launcher_instance = request.getfixturevalue(launcher_instance) + with ctx: + adapter = buffer_writer_dispatch.create_adapter_from_launcher( + launcher_instance, settings_builder + ) + assert adapter._adapted_launcher is launcher_instance + + +@pytest.mark.parametrize( + "launcher_instances, ctx", + ( + pytest.param( + (BufferWriterLauncher(io.StringIO()),), + contextlib.nullcontext(None), + id="Correctly configures expected launcher", + ), + pytest.param( + ( + "launcher_like", + "launcher_like", + BufferWriterLauncher(io.StringIO()), + "launcher_like", + ), + contextlib.nullcontext(None), + id="Correctly ignores incompatible launchers instances", + ), + pytest.param( + (), + pytest.raises( + errors.LauncherNotFoundError, + match="^No launcher of exactly type.+could be found from provided launchers$", + ), + id="Errors if no launcher could be found", + ), + pytest.param( + ( + "launcher_like", + BufferWriterLauncherSubclass(io.StringIO), + "launcher_like", + ), + pytest.raises( + errors.LauncherNotFoundError, + match="^No launcher of exactly type.+could be found from provided launchers$", + ), + id="Errors if no launcher matches expected type exactly", + ), + ), +) +def test_dispatch_registration_configures_first_compatible_launcher_from_sequence_of_launchers( + request, settings_builder, buffer_writer_dispatch, launcher_instances, ctx +): + def resolve_instance(inst): + return request.getfixturevalue(inst) if isinstance(inst, str) else inst + + launcher_instances = tuple(map(resolve_instance, launcher_instances)) + + with ctx: + adapter = buffer_writer_dispatch.configure_first_compatible_launcher( + with_settings=settings_builder, from_available_launchers=launcher_instances + ) + + +def test_dispatch_registration_can_create_a_laucher_for_an_experiment_and_can_reconfigure_it_later( + settings_builder, buffer_writer_dispatch +): + class MockExperiment: ... + + exp = MockExperiment() + adapter_1 = buffer_writer_dispatch.create_new_launcher_configuration( + for_experiment=exp, with_settings=settings_builder + ) + assert type(adapter_1._adapted_launcher) == buffer_writer_dispatch.launcher_type + existing_launcher = adapter_1._adapted_launcher + + adapter_2 = buffer_writer_dispatch.create_adapter_from_launcher( + existing_launcher, settings_builder + ) + assert type(adapter_2._adapted_launcher) == buffer_writer_dispatch.launcher_type + assert adapter_1._adapted_launcher is adapter_2._adapted_launcher + assert adapter_1 is not adapter_2 diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 004090eef..57ae67d68 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -1,5 +1,6 @@ import pytest +from smartsim._core.launcher.dragon.dragonLauncher import _as_run_request_view from smartsim._core.schemas.dragonRequests import DragonRunRequest from smartsim.settings import LaunchSettings from smartsim.settings.builders.launch.dragon import DragonArgBuilder @@ -38,12 +39,12 @@ def test_dragon_class_methods(function, value, flag, result): def test_formatting_launch_args_into_request( echo_executable_like, nodes, tasks_per_node ): - builder = DragonArgBuilder({}) + args = DragonArgBuilder({}) if nodes is not NOT_SET: - builder.set_nodes(nodes) + args.set_nodes(nodes) if tasks_per_node is not NOT_SET: - builder.set_tasks_per_node(tasks_per_node) - req = builder.finalize(echo_executable_like, {}) + args.set_tasks_per_node(tasks_per_node) + req = _as_run_request_view(args, echo_executable_like, {}) args = dict( (k, v) diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 4eb314a8b..d69657f23 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.local import LocalArgBuilder +from smartsim.settings.builders.launch.local import ( + LocalArgBuilder, + _format_local_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -115,5 +118,5 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(echo_executable_like): - cmd = LocalArgBuilder({}).finalize(echo_executable_like, {}) + cmd = _format_local_command(LocalArgBuilder({}), echo_executable_like, {}) assert tuple(cmd) == ("echo", "hello", "world") diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 592c80ce7..91f65821b 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -1,7 +1,7 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.lsf import JsrunArgBuilder +from smartsim.settings.builders.launch.lsf import JsrunArgBuilder, _format_jsrun_command from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -92,5 +92,5 @@ def test_launch_args(): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = JsrunArgBuilder(args).finalize(echo_executable_like, {}) + cmd = _format_jsrun_command(JsrunArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 9b651c220..54fed657e 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -7,6 +7,9 @@ MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder, + _format_mpiexec_command, + _format_mpirun_command, + _format_orterun_command, ) from smartsim.settings.launchCommand import LauncherType @@ -210,11 +213,15 @@ def test_invalid_hostlist_format(launcher): @pytest.mark.parametrize( - "cls, cmd", + "cls, fmt, cmd", ( - pytest.param(MpiArgBuilder, "mpirun", id="w/ mpirun"), - pytest.param(MpiexecArgBuilder, "mpiexec", id="w/ mpiexec"), - pytest.param(OrteArgBuilder, "orterun", id="w/ orterun"), + pytest.param(MpiArgBuilder, _format_mpirun_command, "mpirun", id="w/ mpirun"), + pytest.param( + MpiexecArgBuilder, _format_mpiexec_command, "mpiexec", id="w/ mpiexec" + ), + pytest.param( + OrteArgBuilder, _format_orterun_command, "orterun", id="w/ orterun" + ), ), ) @pytest.mark.parametrize( @@ -248,6 +255,6 @@ def test_invalid_hostlist_format(launcher): ), ), ) -def test_formatting_launch_args(echo_executable_like, cls, cmd, args, expected): - fmt = cls(args).finalize(echo_executable_like, {}) - assert tuple(fmt) == (cmd,) + expected +def test_formatting_launch_args(echo_executable_like, cls, fmt, cmd, args, expected): + fmt_cmd = fmt(cls(args), echo_executable_like, {}) + assert tuple(fmt_cmd) == (cmd,) + expected diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index a0bc7821c..5b74e2d0c 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.pals import PalsMpiexecArgBuilder +from smartsim.settings.builders.launch.pals import ( + PalsMpiexecArgBuilder, + _format_mpiexec_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -103,5 +106,5 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = PalsMpiexecArgBuilder(args).finalize(echo_executable_like, {}) + cmd = _format_mpiexec_command(PalsMpiexecArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index bfa7dd9e1..2a84c831e 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.slurm import SlurmArgBuilder +from smartsim.settings.builders.launch.slurm import ( + SlurmArgBuilder, + _format_srun_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -289,5 +292,5 @@ def test_set_het_groups(monkeypatch): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = SlurmArgBuilder(args).finalize(echo_executable_like, {}) + cmd = _format_srun_command(SlurmArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index 0a34b6473..5c65d367a 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -105,6 +105,5 @@ def test_sbatch_manual(): slurmScheduler.scheduler_args.set_account("A3531") slurmScheduler.scheduler_args.set_walltime("10:00:00") formatted = slurmScheduler.format_batch_args() - print(f"here: {formatted}") result = ["--nodes=5", "--account=A3531", "--time=10:00:00"] assert formatted == result From 6b7943a6f7b47dcc7bb0449e1587c6b0d7156a30 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Fri, 19 Jul 2024 22:37:01 -0500 Subject: [PATCH 25/82] Upper pin typing extensions (thanks TF) --- setup.py | 2 +- smartsim/settings/dispatch.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index d820563d9..05e1c6436 100644 --- a/setup.py +++ b/setup.py @@ -180,7 +180,7 @@ def has_ext_modules(_placeholder): "pyzmq>=25.1.2", "pygithub>=2.3.0", "numpy<2", - "typing_extensions>=4.1.0", + "typing_extensions>=4.1.0,<4.6", ] # Add SmartRedis at specific version diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index a69a6587a..814797824 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -38,7 +38,6 @@ from smartsim.types import LaunchedJobID if t.TYPE_CHECKING: - from smartsim.experiment import Experiment from smartsim.settings.builders import LaunchArgBuilder @@ -47,7 +46,7 @@ _Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) _TDispatchable = t.TypeVar("_TDispatchable", bound="LaunchArgBuilder") -_EnvironMappingType: t.TypeAlias = t.Mapping[str, str | None] +_EnvironMappingType: t.TypeAlias = t.Mapping[str, "str | None"] _FormatterType: t.TypeAlias = t.Callable[ [_TDispatchable, "ExecutableLike", _EnvironMappingType], _T ] @@ -163,7 +162,7 @@ class _DispatchRegistration(t.Generic[_TDispatchable, _T]): formatter: _FormatterType[_TDispatchable, _T] launcher_type: type[LauncherLike[_T]] - def _is_compatible_launcher(self, launcher) -> bool: + def _is_compatible_launcher(self, launcher: LauncherLike[t.Any]) -> bool: return type(launcher) is self.launcher_type def create_new_launcher_configuration( From 82ee19fd59ca12be5c4c04e18c4fae967d19c5a6 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Sat, 20 Jul 2024 18:44:01 -0500 Subject: [PATCH 26/82] Make 3.9 compatable, lint, better names --- pyproject.toml | 12 ++- .../_core/launcher/dragon/dragonLauncher.py | 2 +- smartsim/experiment.py | 10 +- smartsim/settings/builders/launch/alps.py | 8 +- smartsim/settings/builders/launch/dragon.py | 8 +- smartsim/settings/builders/launch/local.py | 6 +- smartsim/settings/builders/launch/lsf.py | 8 +- smartsim/settings/builders/launch/mpi.py | 14 +-- smartsim/settings/builders/launch/pals.py | 8 +- smartsim/settings/builders/launch/slurm.py | 6 +- smartsim/settings/dispatch.py | 102 +++++++++--------- .../test_settings/test_alpsLauncher.py | 7 +- .../test_settings/test_dragonLauncher.py | 19 ++-- .../test_settings/test_localLauncher.py | 7 +- .../test_settings/test_lsfLauncher.py | 4 +- .../test_settings/test_mpiLauncher.py | 14 ++- .../test_settings/test_palsLauncher.py | 4 +- .../test_settings/test_slurmLauncher.py | 7 +- 18 files changed, 121 insertions(+), 125 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bda99459d..5df64aa97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,17 @@ module = [ "smartsim._core.control.controller", "smartsim._core.control.manifest", "smartsim._core.entrypoints.dragon_client", - "smartsim._core.launcher.*", + "smartsim._core.launcher.colocated", + "smartsim._core.launcher.launcher", + "smartsim._core.launcher.local.*", + "smartsim._core.launcher.lsf.*", + "smartsim._core.launcher.pbs.*", + "smartsim._core.launcher.sge.*", + "smartsim._core.launcher.slurm.*", + "smartsim._core.launcher.step.*", + "smartsim._core.launcher.stepInfo", + "smartsim._core.launcher.stepMapping", + "smartsim._core.launcher.taskManager", "smartsim._core.utils.serialize", "smartsim._core.utils.telemetry.*", "smartsim.database.*", diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 74e22e160..949fcb044 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -338,7 +338,7 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # TODO: Remove this registry and move back to builder file after fixing -# circular import +# circular import caused by `DragonLauncher.supported_rs` # ----------------------------------------------------------------------------- from smartsim.settings.builders.launch.dragon import DragonArgBuilder from smartsim.settings.dispatch import ExecutableLike, dispatch diff --git a/smartsim/experiment.py b/smartsim/experiment.py index c2e56d19a..ca29191df 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -28,7 +28,6 @@ from __future__ import annotations -import itertools import os import os.path as osp import textwrap @@ -38,8 +37,8 @@ from tabulate import tabulate from smartsim._core.config import CONFIG -from smartsim.error.errors import LauncherNotFoundError, SSUnsupportedError -from smartsim.settings.dispatch import default_dispatcher +from smartsim.error import errors +from smartsim.settings.dispatch import DEFAULT_DISPATCHER from smartsim.status import SmartSimStatus from ._core import Controller, Generator, Manifest, previewrenderer @@ -53,7 +52,6 @@ ) from .error import SmartSimError from .log import ctx_exp_path, get_logger, method_contextualizer -from .settings import BatchSettings, Container, RunSettings if t.TYPE_CHECKING: from smartsim.launchable.job import Job @@ -113,7 +111,7 @@ def __init__( name: str, exp_path: str | None = None, *, # Keyword arguments only - settings_dispatcher: Dispatcher = default_dispatcher, + settings_dispatcher: Dispatcher = DEFAULT_DISPATCHER, ): """Initialize an Experiment instance. @@ -204,7 +202,7 @@ def _start(job: Job) -> LaunchedJobID: from_available_launchers=self._active_launchers, with_settings=args, ) - except LauncherNotFoundError: + except errors.LauncherNotFoundError: launch_config = dispatch.create_new_launcher_configuration( for_experiment=self, with_settings=args ) diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index 09d5931ac..5826a2de4 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -29,17 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format +from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn -from ...common import StringArgument, set_check_input +from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) -_format_aprun_command = shell_format(run_command="aprun") +_as_aprun_command = make_shell_format_fn(run_command="aprun") -@dispatch(with_format=_format_aprun_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_aprun_command, to_launcher=ShellLauncher) class AprunArgBuilder(LaunchArgBuilder): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 3cbb1c6d0..4ba793bf7 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -26,18 +26,12 @@ from __future__ import annotations -import os -import typing as t - from smartsim.log import get_logger -from ...common import StringArgument, set_check_input +from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder -if t.TYPE_CHECKING: - from smartsim.settings.dispatch import ExecutableLike - logger = get_logger(__name__) diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index 7002a6831..49ef3ad92 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -29,17 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format +from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) -_format_local_command = shell_format(run_command=None) +_as_local_command = make_shell_format_fn(run_command=None) -@dispatch(with_format=_format_local_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_local_command, to_launcher=ShellLauncher) class LocalArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index ec99d51b9..1fe8ea30b 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -29,17 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format +from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn -from ...common import StringArgument, set_check_input +from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) -_format_jsrun_command = shell_format(run_command="jsrun") +_as_jsrun_command = make_shell_format_fn(run_command="jsrun") -@dispatch(with_format=_format_jsrun_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) class JsrunArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 139096010..a0d947418 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -29,16 +29,16 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format +from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) -_format_mpirun_command = shell_format("mpirun") -_format_mpiexec_command = shell_format("mpiexec") -_format_orterun_command = shell_format("orterun") +_as_mpirun_command = make_shell_format_fn("mpirun") +_as_mpiexec_command = make_shell_format_fn("mpiexec") +_as_orterun_command = make_shell_format_fn("orterun") class _BaseMPIArgBuilder(LaunchArgBuilder): @@ -218,21 +218,21 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value -@dispatch(with_format=_format_mpirun_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_mpirun_command, to_launcher=ShellLauncher) class MpiArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpirun.value -@dispatch(with_format=_format_mpiexec_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_mpiexec_command, to_launcher=ShellLauncher) class MpiexecArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpiexec.value -@dispatch(with_format=_format_orterun_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_orterun_command, to_launcher=ShellLauncher) class OrteArgBuilder(_BaseMPIArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index 1e7ed814e..eeb438455 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -29,17 +29,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format +from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn -from ...common import StringArgument, set_check_input +from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) -_format_mpiexec_command = shell_format(run_command="mpiexec") +_as_pals_command = make_shell_format_fn(run_command="mpiexec") -@dispatch(with_format=_format_mpiexec_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_pals_command, to_launcher=ShellLauncher) class PalsMpiexecArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 72058f983..f80a5b8b9 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -31,17 +31,17 @@ import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, shell_format +from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) -_format_srun_command = shell_format(run_command="srun") +_as_srun_command = make_shell_format_fn(run_command="srun") -@dispatch(with_format=_format_srun_command, to_launcher=ShellLauncher) +@dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) class SlurmArgBuilder(LaunchArgBuilder): def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 814797824..cda8dd9ba 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -31,7 +31,7 @@ import typing as t import uuid -from typing_extensions import Self, TypeVarTuple, Unpack +from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack from smartsim._core.utils import helpers from smartsim.error import errors @@ -41,17 +41,17 @@ from smartsim.experiment import Experiment from smartsim.settings.builders import LaunchArgBuilder - -_T = t.TypeVar("_T") _Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) -_TDispatchable = t.TypeVar("_TDispatchable", bound="LaunchArgBuilder") -_EnvironMappingType: t.TypeAlias = t.Mapping[str, "str | None"] -_FormatterType: t.TypeAlias = t.Callable[ - [_TDispatchable, "ExecutableLike", _EnvironMappingType], _T +_DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArgBuilder") +_LaunchableT = t.TypeVar("_LaunchableT") + +_EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] +_FormatterType: TypeAlias = t.Callable[ + [_DispatchableT, "ExecutableLike", _EnvironMappingType], _LaunchableT ] -_LaunchConfigType: t.TypeAlias = "_LauncherAdapter[ExecutableLike, _EnvironMappingType]" -_UnkownType: t.TypeAlias = t.NoReturn +_LaunchConfigType: TypeAlias = "_LauncherAdapter[ExecutableLike, _EnvironMappingType]" +_UnkownType: TypeAlias = t.NoReturn @t.final @@ -81,27 +81,27 @@ def dispatch( self, args: None = ..., *, - with_format: _FormatterType[_TDispatchable, _T], - to_launcher: type[LauncherLike[_T]], + with_format: _FormatterType[_DispatchableT, _LaunchableT], + to_launcher: type[LauncherLike[_LaunchableT]], allow_overwrite: bool = ..., - ) -> t.Callable[[type[_TDispatchable]], type[_TDispatchable]]: ... + ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]]: ... @t.overload def dispatch( self, - args: type[_TDispatchable], + args: type[_DispatchableT], *, - with_format: _FormatterType[_TDispatchable, _T], - to_launcher: type[LauncherLike[_T]], + with_format: _FormatterType[_DispatchableT, _LaunchableT], + to_launcher: type[LauncherLike[_LaunchableT]], allow_overwrite: bool = ..., ) -> None: ... def dispatch( self, - args: type[_TDispatchable] | None = None, + args: type[_DispatchableT] | None = None, *, - with_format: _FormatterType[_TDispatchable, _T], - to_launcher: type[LauncherLike[_T]], + with_format: _FormatterType[_DispatchableT, _LaunchableT], + to_launcher: type[LauncherLike[_LaunchableT]], allow_overwrite: bool = False, - ) -> t.Callable[[type[_TDispatchable]], type[_TDispatchable]] | None: + ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]] | None: """A type safe way to add a mapping of settings builder to launcher to handle the settings at launch time. """ @@ -113,7 +113,7 @@ def dispatch( if err_msg is not None: raise TypeError(err_msg) - def register(args_: type[_TDispatchable], /) -> type[_TDispatchable]: + def register(args_: type[_DispatchableT], /) -> type[_DispatchableT]: if args_ in self._dispatch_registry and not allow_overwrite: launcher_type = self._dispatch_registry[args_].launcher_type raise TypeError( @@ -131,48 +131,51 @@ def register(args_: type[_TDispatchable], /) -> type[_TDispatchable]: return register def get_dispatch( - self, args: _TDispatchable | type[_TDispatchable] - ) -> _DispatchRegistration[_TDispatchable, _UnkownType]: + self, args: _DispatchableT | type[_DispatchableT] + ) -> _DispatchRegistration[_DispatchableT, _UnkownType]: """Find a type of launcher that is registered as being able to launch the output of the provided builder """ if not isinstance(args, type): args = type(args) - dispatch = self._dispatch_registry.get(args, None) - if dispatch is None: + dispatch_ = self._dispatch_registry.get(args, None) + if dispatch_ is None: raise TypeError( f"No dispatch for `{type(args).__name__}` has been registered " f"has been registered with {type(self).__name__} `{self}`" ) # Note the sleight-of-hand here: we are secretly casting a type of # `_DispatchRegistration[Any, Any]` -> - # `_DispatchRegistration[_TDispatchable, _T]`. - # where `_T` is unbound! + # `_DispatchRegistration[_DispatchableT, _LaunchableT]`. + # where `_LaunchableT` is unbound! # # This is safe to do if all entries in the mapping were added using a # type safe method (e.g. `Dispatcher.dispatch`), but if a user were to # supply a custom dispatch registry or otherwise modify the registry # this is not necessarily 100% type safe!! - return dispatch + return dispatch_ @t.final @dataclasses.dataclass(frozen=True) -class _DispatchRegistration(t.Generic[_TDispatchable, _T]): - formatter: _FormatterType[_TDispatchable, _T] - launcher_type: type[LauncherLike[_T]] +class _DispatchRegistration(t.Generic[_DispatchableT, _LaunchableT]): + formatter: _FormatterType[_DispatchableT, _LaunchableT] + launcher_type: type[LauncherLike[_LaunchableT]] def _is_compatible_launcher(self, launcher: LauncherLike[t.Any]) -> bool: + # Disabling because we want to match the the type of the dispatch + # *exactly* as specified by the user + # pylint: disable-next=unidiomatic-typecheck return type(launcher) is self.launcher_type def create_new_launcher_configuration( - self, for_experiment: Experiment, with_settings: _TDispatchable + self, for_experiment: Experiment, with_settings: _DispatchableT ) -> _LaunchConfigType: launcher = self.launcher_type.create(for_experiment) return self.create_adapter_from_launcher(launcher, with_settings) def create_adapter_from_launcher( - self, launcher: LauncherLike[_T], settings: _TDispatchable + self, launcher: LauncherLike[_LaunchableT], settings: _DispatchableT ) -> _LaunchConfigType: if not self._is_compatible_launcher(launcher): raise TypeError( @@ -181,14 +184,14 @@ def create_adapter_from_launcher( f"exactly `{self.launcher_type}`" ) - def format_(exe: ExecutableLike, env: _EnvironMappingType) -> _T: + def format_(exe: ExecutableLike, env: _EnvironMappingType) -> _LaunchableT: return self.formatter(settings, exe, env) return _LauncherAdapter(launcher, format_) def configure_first_compatible_launcher( self, - with_settings: _TDispatchable, + with_settings: _DispatchableT, from_available_launchers: t.Iterable[LauncherLike[t.Any]], ) -> _LaunchConfigType: launcher = helpers.first(self._is_compatible_launcher, from_available_launchers) @@ -203,11 +206,14 @@ def configure_first_compatible_launcher( @t.final class _LauncherAdapter(t.Generic[Unpack[_Ts]]): def __init__( - self, launcher: LauncherLike[_T], map_: t.Callable[[Unpack[_Ts]], _T] + self, + launcher: LauncherLike[_LaunchableT], + map_: t.Callable[[Unpack[_Ts]], _LaunchableT], ) -> None: - # NOTE: We need to cast off the `_T` -> `Any` in the `__init__` - # signature to hide the transform from users of this class. If - # possible, try not to expose outside of protected methods! + # NOTE: We need to cast off the `_LaunchableT` -> `Any` in the + # `__init__` method signature to hide the transform from users of + # this class. If possible, this type should not be exposed to + # users of this class! self._adapt: t.Callable[[Unpack[_Ts]], t.Any] = map_ self._adapted_launcher: LauncherLike[t.Any] = launcher @@ -216,8 +222,11 @@ def start(self, *args: Unpack[_Ts]) -> LaunchedJobID: return self._adapted_launcher.start(payload) -default_dispatcher: t.Final = Dispatcher() -dispatch: t.Final = default_dispatcher.dispatch +DEFAULT_DISPATCHER: t.Final = Dispatcher() +# Disabling because we want this to look and feel like a top level function, +# but don't want to have a second copy of the nasty overloads +# pylint: disable-next=invalid-name +dispatch: t.Final = DEFAULT_DISPATCHER.dispatch # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @@ -236,18 +245,14 @@ def as_program_arguments(self) -> t.Sequence[str]: ... class LauncherLike(t.Protocol[_T_contra]): def start(self, launchable: _T_contra) -> LaunchedJobID: ... @classmethod - def create(cls, exp: Experiment) -> Self: ... + def create(cls, exp: Experiment, /) -> Self: ... -# TODO: This is just a nice helper function that I am using for the time being -# to wire everything up! In reality it might be a bit too confusing and -# meta-program-y for production code. Check with the core team to see -# what they think!! -def shell_format( +def make_shell_format_fn( run_command: str | None, ) -> _FormatterType[LaunchArgBuilder, t.Sequence[str]]: def impl( - args: LaunchArgBuilder, exe: ExecutableLike, env: _EnvironMappingType + args: LaunchArgBuilder, exe: ExecutableLike, _env: _EnvironMappingType ) -> t.Sequence[str]: return ( ( @@ -272,11 +277,12 @@ def __init__(self) -> None: def start(self, launchable: t.Sequence[str]) -> LaunchedJobID: id_ = create_job_id() exe, *rest = launchable + # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest)) return id_ @classmethod - def create(cls, exp: Experiment) -> Self: + def create(cls, _: Experiment) -> Self: return cls() diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 5ac2f8e11..cb6e829eb 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -1,10 +1,7 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.alps import ( - AprunArgBuilder, - _format_aprun_command, -) +from smartsim.settings.builders.launch.alps import AprunArgBuilder, _as_aprun_command from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -186,5 +183,5 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _format_aprun_command(AprunArgBuilder(args), echo_executable_like, {}) + cmd = _as_aprun_command(AprunArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 57ae67d68..cc57e329f 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -1,7 +1,7 @@ import pytest from smartsim._core.launcher.dragon.dragonLauncher import _as_run_request_view -from smartsim._core.schemas.dragonRequests import DragonRunRequest +from smartsim._core.schemas.dragonRequests import DragonRunRequestView from smartsim.settings import LaunchSettings from smartsim.settings.builders.launch.dragon import DragonArgBuilder from smartsim.settings.launchCommand import LauncherType @@ -46,18 +46,17 @@ def test_formatting_launch_args_into_request( args.set_tasks_per_node(tasks_per_node) req = _as_run_request_view(args, echo_executable_like, {}) - args = dict( - (k, v) - for k, v in ( - ("nodes", nodes), - ("tasks_per_node", tasks_per_node), - ) + args = { + k: v + for k, v in { + "nodes": nodes, + "tasks_per_node": tasks_per_node, + }.items() if v is not NOT_SET - ) - expected = DragonRunRequest( + } + expected = DragonRunRequestView( exe="echo", exe_args=["hello", "world"], path="/tmp", env={}, **args ) - assert req.nodes == expected.nodes assert req.tasks_per_node == expected.tasks_per_node assert req.hostlist == expected.hostlist diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index d69657f23..07adb231b 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -1,10 +1,7 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.local import ( - LocalArgBuilder, - _format_local_command, -) +from smartsim.settings.builders.launch.local import LocalArgBuilder, _as_local_command from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -118,5 +115,5 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(echo_executable_like): - cmd = _format_local_command(LocalArgBuilder({}), echo_executable_like, {}) + cmd = _as_local_command(LocalArgBuilder({}), echo_executable_like, {}) assert tuple(cmd) == ("echo", "hello", "world") diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 91f65821b..9d276c0e7 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -1,7 +1,7 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.lsf import JsrunArgBuilder, _format_jsrun_command +from smartsim.settings.builders.launch.lsf import JsrunArgBuilder, _as_jsrun_command from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -92,5 +92,5 @@ def test_launch_args(): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _format_jsrun_command(JsrunArgBuilder(args), echo_executable_like, {}) + cmd = _as_jsrun_command(JsrunArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 54fed657e..16947590c 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -7,9 +7,9 @@ MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder, - _format_mpiexec_command, - _format_mpirun_command, - _format_orterun_command, + _as_mpiexec_command, + _as_mpirun_command, + _as_orterun_command, ) from smartsim.settings.launchCommand import LauncherType @@ -215,13 +215,11 @@ def test_invalid_hostlist_format(launcher): @pytest.mark.parametrize( "cls, fmt, cmd", ( - pytest.param(MpiArgBuilder, _format_mpirun_command, "mpirun", id="w/ mpirun"), + pytest.param(MpiArgBuilder, _as_mpirun_command, "mpirun", id="w/ mpirun"), pytest.param( - MpiexecArgBuilder, _format_mpiexec_command, "mpiexec", id="w/ mpiexec" - ), - pytest.param( - OrteArgBuilder, _format_orterun_command, "orterun", id="w/ orterun" + MpiexecArgBuilder, _as_mpiexec_command, "mpiexec", id="w/ mpiexec" ), + pytest.param(OrteArgBuilder, _as_orterun_command, "orterun", id="w/ orterun"), ), ) @pytest.mark.parametrize( diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 5b74e2d0c..47fa64713 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -3,7 +3,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.builders.launch.pals import ( PalsMpiexecArgBuilder, - _format_mpiexec_command, + _as_pals_command, ) from smartsim.settings.launchCommand import LauncherType @@ -106,5 +106,5 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _format_mpiexec_command(PalsMpiexecArgBuilder(args), echo_executable_like, {}) + cmd = _as_pals_command(PalsMpiexecArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 2a84c831e..5935ec690 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -1,10 +1,7 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.slurm import ( - SlurmArgBuilder, - _format_srun_command, -) +from smartsim.settings.builders.launch.slurm import SlurmArgBuilder, _as_srun_command from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -292,5 +289,5 @@ def test_set_het_groups(monkeypatch): ), ) def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _format_srun_command(SlurmArgBuilder(args), echo_executable_like, {}) + cmd = _as_srun_command(SlurmArgBuilder(args), echo_executable_like, {}) assert tuple(cmd) == expected From 36bcfbb39eb35999b2e5de9439c1366b358ff7b1 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 17 Jul 2024 18:19:43 -0500 Subject: [PATCH 27/82] Address reviewer comments --- .../_core/launcher/dragon/dragonConnector.py | 16 +++++ .../_core/launcher/dragon/dragonLauncher.py | 6 +- smartsim/experiment.py | 59 +++++++++++-------- .../settings/builders/launchArgBuilder.py | 15 +++-- smartsim/settings/dispatch.py | 34 ++++++----- smartsim/settings/launchSettings.py | 10 +++- tests/temp_tests/test_settings/conftest.py | 12 ++-- .../test_settings/test_alpsLauncher.py | 4 +- .../temp_tests/test_settings/test_dispatch.py | 46 +++++++-------- .../test_settings/test_dragonLauncher.py | 4 +- .../test_settings/test_localLauncher.py | 4 +- .../test_settings/test_lsfLauncher.py | 4 +- .../test_settings/test_mpiLauncher.py | 4 +- .../test_settings/test_palsLauncher.py | 4 +- .../test_settings/test_slurmLauncher.py | 4 +- 15 files changed, 134 insertions(+), 92 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonConnector.py b/smartsim/_core/launcher/dragon/dragonConnector.py index ca721eeaa..60fbf3ce7 100644 --- a/smartsim/_core/launcher/dragon/dragonConnector.py +++ b/smartsim/_core/launcher/dragon/dragonConnector.py @@ -522,6 +522,22 @@ def _dragon_cleanup( def _resolve_dragon_path(fallback: t.Union[str, "os.PathLike[str]"]) -> Path: + """Return the path at which a user should set up a dragon server. + + The order of path resolution is: + 1) If the the user has set a global dragon path via + `Config.dragon_server_path` use that without alteration. + 2) Use the `fallback` path which should be the path to an existing + directory. Append the default dragon server subdirectory defined by + `Config.dragon_default_subdir` + + Currently this function will raise if a user attempts to specify multiple + dragon server paths via `:` seperation. + + :param fallback: The path to an existing directory on the file system to + use if the global dragon directory is not set. + :returns: The path to directory in which the dragon server should run. + """ config = get_config() dragon_server_path = config.dragon_server_path or os.path.join( fallback, config.dragon_default_subdir diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 949fcb044..078a89278 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -341,11 +341,13 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: # circular import caused by `DragonLauncher.supported_rs` # ----------------------------------------------------------------------------- from smartsim.settings.builders.launch.dragon import DragonArgBuilder -from smartsim.settings.dispatch import ExecutableLike, dispatch +from smartsim.settings.dispatch import ExecutableProtocol, dispatch def _as_run_request_view( - run_req_args: DragonArgBuilder, exe: ExecutableLike, env: t.Mapping[str, str | None] + run_req_args: DragonArgBuilder, + exe: ExecutableProtocol, + env: t.Mapping[str, str | None], ) -> DragonRunRequestView: exe_, *args = exe.as_program_arguments() return DragonRunRequestView( diff --git a/smartsim/experiment.py b/smartsim/experiment.py index ca29191df..6f1a5744a 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -55,8 +55,11 @@ if t.TYPE_CHECKING: from smartsim.launchable.job import Job - from smartsim.settings.builders.launchArgBuilder import LaunchArgBuilder - from smartsim.settings.dispatch import Dispatcher, ExecutableLike, LauncherLike + from smartsim.settings.dispatch import ( + Dispatcher, + ExecutableProtocol, + LauncherProtocol, + ) from smartsim.types import LaunchedJobID logger = get_logger(__name__) @@ -106,13 +109,7 @@ class Experiment: and utilized throughout runtime. """ - def __init__( - self, - name: str, - exp_path: str | None = None, - *, # Keyword arguments only - settings_dispatcher: Dispatcher = DEFAULT_DISPATCHER, - ): + def __init__(self, name: str, exp_path: str | None = None): """Initialize an Experiment instance. With the default settings, the Experiment will use the @@ -152,9 +149,6 @@ def __init__( :param name: name for the ``Experiment`` :param exp_path: path to location of ``Experiment`` directory - :param settings_dispatcher: The dispatcher the experiment will use to - figure determine how to launch a job. If none is provided, the - experiment will use the default dispatcher. """ self.name = name if exp_path: @@ -167,24 +161,39 @@ def __init__( exp_path = osp.join(getcwd(), name) self.exp_path = exp_path + """The path under which the experiment operate""" - # TODO: Remove this! The contoller is becoming obsolete + # TODO: Remove this! The controller is becoming obsolete self._control = Controller(launcher="local") - self._dispatcher = settings_dispatcher - self._active_launchers: set[LauncherLike[t.Any]] = set() + self._active_launchers: set[LauncherProtocol[t.Any]] = set() """The active launchers created, used, and reused by the experiment""" - self.fs_identifiers: t.Set[str] = set() + self._fs_identifiers: t.Set[str] = set() + """Set of feature store identifiers currently in use by this + experiment""" self._telemetry_cfg = ExperimentTelemetryConfiguration() - - def start_jobs(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: - """WIP: replacemnt method to launch jobs using the new API""" + """Switch to specify if telemetry data should be produced for this + experiment""" + + def start_jobs( + self, *jobs: Job, dispatcher: Dispatcher = DEFAULT_DISPATCHER + ) -> tuple[LaunchedJobID, ...]: + """Execute a collection of `Job` instances. + + :param jobs: The collection of jobs instances to start + :param dispatcher: The dispatcher that should be used to determine how + to start a job based on its settings. If not specified it will + default to a dispatcher pre-configured by SmartSim. + :returns: A sequence of ids with order corresponding to the sequence of + jobs that can be used to query or alter the status of that + particular execution of the job. + """ if not jobs: raise TypeError( f"{type(self).__name__}.start_jobs() missing at least 1 required " - "positional argument" + "positional argument of type `Job`" ) def _start(job: Job) -> LaunchedJobID: @@ -194,9 +203,9 @@ def _start(job: Job) -> LaunchedJobID: # FIXME: Remove this cast after `SmartSimEntity` conforms to # protocol. For now, live with the "dangerous" type cast # --------------------------------------------------------------------- - exe = t.cast("ExecutableLike", job.entity) + exe = t.cast("ExecutableProtocol", job.entity) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - dispatch = self._dispatcher.get_dispatch(args) + dispatch = dispatcher.get_dispatch(args) try: launch_config = dispatch.configure_first_compatible_launcher( from_available_launchers=self._active_launchers, @@ -603,7 +612,7 @@ def _create_entity_dir(self, start_manifest: Manifest) -> None: def create_entity_dir( entity: t.Union[FeatureStore, Application, Ensemble] ) -> None: - if not os.path.isdir(entity.path): + if not osp.isdir(entity.path): os.makedirs(entity.path) for application in start_manifest.applications: @@ -620,11 +629,11 @@ def __str__(self) -> str: def _append_to_fs_identifier_list(self, fs_identifier: str) -> None: """Check if fs_identifier already exists when calling create_feature_store""" - if fs_identifier in self.fs_identifiers: + if fs_identifier in self._fs_identifiers: logger.warning( f"A feature store with the identifier {fs_identifier} has already been made " "An error will be raised if multiple Feature Stores are started " "with the same identifier" ) # Otherwise, add - self.fs_identifiers.add(fs_identifier) + self._fs_identifiers.add(fs_identifier) diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py index 2c09dd2e8..8ebfdb0f0 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/builders/launchArgBuilder.py @@ -37,9 +37,6 @@ logger = get_logger(__name__) -_T = t.TypeVar("_T") - - class LaunchArgBuilder(ABC): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the @@ -48,6 +45,11 @@ class LaunchArgBuilder(ABC): """ def __init__(self, launch_args: t.Dict[str, str | None] | None) -> None: + """Initialize a new `LaunchArgBuilder` instance. + + :param launch_args: A mapping of argument to be used to initialize the + argument builder. + """ self._launch_args = copy.deepcopy(launch_args) or {} @abstractmethod @@ -56,7 +58,12 @@ def launcher_str(self) -> str: @abstractmethod def set(self, arg: str, val: str | None) -> None: - """Set the launch arguments""" + """Set a launch argument + + :param arg: The argument name to set + :param val: The value to set the argument to as a `str` (if + applicable). Otherwise `None` + """ def format_launch_args(self) -> t.Union[t.List[str], None]: """Build formatted launch arguments""" diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index cda8dd9ba..aceead700 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -48,9 +48,11 @@ _EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableLike", _EnvironMappingType], _LaunchableT + [_DispatchableT, "ExecutableProtocol", _EnvironMappingType], _LaunchableT ] -_LaunchConfigType: TypeAlias = "_LauncherAdapter[ExecutableLike, _EnvironMappingType]" +_LaunchConfigType: TypeAlias = ( + "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType]" +) _UnkownType: TypeAlias = t.NoReturn @@ -82,7 +84,7 @@ def dispatch( args: None = ..., *, with_format: _FormatterType[_DispatchableT, _LaunchableT], - to_launcher: type[LauncherLike[_LaunchableT]], + to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]]: ... @t.overload @@ -91,7 +93,7 @@ def dispatch( args: type[_DispatchableT], *, with_format: _FormatterType[_DispatchableT, _LaunchableT], - to_launcher: type[LauncherLike[_LaunchableT]], + to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> None: ... def dispatch( @@ -99,7 +101,7 @@ def dispatch( args: type[_DispatchableT] | None = None, *, with_format: _FormatterType[_DispatchableT, _LaunchableT], - to_launcher: type[LauncherLike[_LaunchableT]], + to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = False, ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]] | None: """A type safe way to add a mapping of settings builder to launcher to @@ -150,7 +152,7 @@ def get_dispatch( # where `_LaunchableT` is unbound! # # This is safe to do if all entries in the mapping were added using a - # type safe method (e.g. `Dispatcher.dispatch`), but if a user were to + # type safe method (e.g. `Dispatcher.dispatch`), but if a user were to # supply a custom dispatch registry or otherwise modify the registry # this is not necessarily 100% type safe!! return dispatch_ @@ -160,9 +162,9 @@ def get_dispatch( @dataclasses.dataclass(frozen=True) class _DispatchRegistration(t.Generic[_DispatchableT, _LaunchableT]): formatter: _FormatterType[_DispatchableT, _LaunchableT] - launcher_type: type[LauncherLike[_LaunchableT]] + launcher_type: type[LauncherProtocol[_LaunchableT]] - def _is_compatible_launcher(self, launcher: LauncherLike[t.Any]) -> bool: + def _is_compatible_launcher(self, launcher: LauncherProtocol[t.Any]) -> bool: # Disabling because we want to match the the type of the dispatch # *exactly* as specified by the user # pylint: disable-next=unidiomatic-typecheck @@ -175,7 +177,7 @@ def create_new_launcher_configuration( return self.create_adapter_from_launcher(launcher, with_settings) def create_adapter_from_launcher( - self, launcher: LauncherLike[_LaunchableT], settings: _DispatchableT + self, launcher: LauncherProtocol[_LaunchableT], settings: _DispatchableT ) -> _LaunchConfigType: if not self._is_compatible_launcher(launcher): raise TypeError( @@ -184,7 +186,7 @@ def create_adapter_from_launcher( f"exactly `{self.launcher_type}`" ) - def format_(exe: ExecutableLike, env: _EnvironMappingType) -> _LaunchableT: + def format_(exe: ExecutableProtocol, env: _EnvironMappingType) -> _LaunchableT: return self.formatter(settings, exe, env) return _LauncherAdapter(launcher, format_) @@ -192,7 +194,7 @@ def format_(exe: ExecutableLike, env: _EnvironMappingType) -> _LaunchableT: def configure_first_compatible_launcher( self, with_settings: _DispatchableT, - from_available_launchers: t.Iterable[LauncherLike[t.Any]], + from_available_launchers: t.Iterable[LauncherProtocol[t.Any]], ) -> _LaunchConfigType: launcher = helpers.first(self._is_compatible_launcher, from_available_launchers) if launcher is None: @@ -207,7 +209,7 @@ def configure_first_compatible_launcher( class _LauncherAdapter(t.Generic[Unpack[_Ts]]): def __init__( self, - launcher: LauncherLike[_LaunchableT], + launcher: LauncherProtocol[_LaunchableT], map_: t.Callable[[Unpack[_Ts]], _LaunchableT], ) -> None: # NOTE: We need to cast off the `_LaunchableT` -> `Any` in the @@ -215,7 +217,7 @@ def __init__( # this class. If possible, this type should not be exposed to # users of this class! self._adapt: t.Callable[[Unpack[_Ts]], t.Any] = map_ - self._adapted_launcher: LauncherLike[t.Any] = launcher + self._adapted_launcher: LauncherProtocol[t.Any] = launcher def start(self, *args: Unpack[_Ts]) -> LaunchedJobID: payload = self._adapt(*args) @@ -238,11 +240,11 @@ def create_job_id() -> LaunchedJobID: return LaunchedJobID(str(uuid.uuid4())) -class ExecutableLike(t.Protocol): +class ExecutableProtocol(t.Protocol): def as_program_arguments(self) -> t.Sequence[str]: ... -class LauncherLike(t.Protocol[_T_contra]): +class LauncherProtocol(t.Protocol[_T_contra]): def start(self, launchable: _T_contra) -> LaunchedJobID: ... @classmethod def create(cls, exp: Experiment, /) -> Self: ... @@ -252,7 +254,7 @@ def make_shell_format_fn( run_command: str | None, ) -> _FormatterType[LaunchArgBuilder, t.Sequence[str]]: def impl( - args: LaunchArgBuilder, exe: ExecutableLike, _env: _EnvironMappingType + args: LaunchArgBuilder, exe: ExecutableProtocol, _env: _EnvironMappingType ) -> t.Sequence[str]: return ( ( diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index dec6034d8..0990e2e82 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -63,7 +63,7 @@ def __init__( @property def launcher(self) -> str: - """Return the launcher name.""" + """The launcher type""" return self._launcher.value @property @@ -89,7 +89,13 @@ def env_vars(self, value: dict[str, str | None]) -> None: self._env_vars = copy.deepcopy(value) def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder: - """Map the Launcher to the LaunchArgBuilder""" + """Map the Launcher to the LaunchArgBuilder. This method should only be + called once during construction. + + :param launch_args: A mapping of argument to be used to initialize the + argument builder. + :returns: The appropriate argument builder for the settings instance. + """ if self._launcher == LauncherType.Slurm: return SlurmArgBuilder(launch_args) elif self._launcher == LauncherType.Mpiexec: diff --git a/tests/temp_tests/test_settings/conftest.py b/tests/temp_tests/test_settings/conftest.py index 72061264f..1368006a9 100644 --- a/tests/temp_tests/test_settings/conftest.py +++ b/tests/temp_tests/test_settings/conftest.py @@ -31,12 +31,12 @@ @pytest.fixture -def echo_executable_like(): - class _ExeLike(dispatch.ExecutableLike): +def mock_echo_executable(): + class _MockExe(dispatch.ExecutableProtocol): def as_program_arguments(self): return ("echo", "hello", "world") - yield _ExeLike() + yield _MockExe() @pytest.fixture @@ -50,8 +50,8 @@ def launcher_str(self): @pytest.fixture -def launcher_like(): - class _LuancherLike(dispatch.LauncherLike): +def mock_launcher(): + class _MockLauncher(dispatch.LauncherProtocol): def start(self, launchable): return dispatch.create_job_id() @@ -59,4 +59,4 @@ def start(self, launchable): def create(cls, exp): return cls() - yield _LuancherLike() + yield _MockLauncher() diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index cb6e829eb..367b30c7f 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -182,6 +182,6 @@ def test_invalid_exclude_hostlist_format(): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _as_aprun_command(AprunArgBuilder(args), echo_executable_like, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_aprun_command(AprunArgBuilder(args), mock_echo_executable, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_dispatch.py b/tests/temp_tests/test_settings/test_dispatch.py index 78c44ad54..673c8998a 100644 --- a/tests/temp_tests/test_settings/test_dispatch.py +++ b/tests/temp_tests/test_settings/test_dispatch.py @@ -44,36 +44,36 @@ def format_fn(args, exe, env): @pytest.fixture -def expected_dispatch_registry(launcher_like, settings_builder): +def expected_dispatch_registry(mock_launcher, settings_builder): yield { type(settings_builder): dispatch._DispatchRegistration( - format_fn, type(launcher_like) + format_fn, type(mock_launcher) ) } def test_declaritive_form_dispatch_declaration( - launcher_like, settings_builder, expected_dispatch_registry + mock_launcher, settings_builder, expected_dispatch_registry ): d = dispatch.Dispatcher() assert type(settings_builder) == d.dispatch( - with_format=format_fn, to_launcher=type(launcher_like) + with_format=format_fn, to_launcher=type(mock_launcher) )(type(settings_builder)) assert d._dispatch_registry == expected_dispatch_registry def test_imperative_form_dispatch_declaration( - launcher_like, settings_builder, expected_dispatch_registry + mock_launcher, settings_builder, expected_dispatch_registry ): d = dispatch.Dispatcher() assert None == d.dispatch( - type(settings_builder), to_launcher=type(launcher_like), with_format=format_fn + type(settings_builder), to_launcher=type(mock_launcher), with_format=format_fn ) assert d._dispatch_registry == expected_dispatch_registry def test_dispatchers_from_same_registry_do_not_cross_polute( - launcher_like, settings_builder, expected_dispatch_registry + mock_launcher, settings_builder, expected_dispatch_registry ): some_starting_registry = {} d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) @@ -86,14 +86,14 @@ def test_dispatchers_from_same_registry_do_not_cross_polute( ) d2.dispatch( - type(settings_builder), with_format=format_fn, to_launcher=type(launcher_like) + type(settings_builder), with_format=format_fn, to_launcher=type(mock_launcher) ) assert d1._dispatch_registry == {} assert d2._dispatch_registry == expected_dispatch_registry def test_copied_dispatchers_do_not_cross_pollute( - launcher_like, settings_builder, expected_dispatch_registry + mock_launcher, settings_builder, expected_dispatch_registry ): some_starting_registry = {} d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) @@ -106,7 +106,7 @@ def test_copied_dispatchers_do_not_cross_pollute( ) d2.dispatch( - type(settings_builder), to_launcher=type(launcher_like), with_format=format_fn + type(settings_builder), to_launcher=type(mock_launcher), with_format=format_fn ) assert d1._dispatch_registry == {} assert d2._dispatch_registry == expected_dispatch_registry @@ -144,13 +144,13 @@ def test_copied_dispatchers_do_not_cross_pollute( def test_dispatch_overwriting( add_dispatch, expected_ctx, - launcher_like, + mock_launcher, settings_builder, expected_dispatch_registry, ): d = dispatch.Dispatcher(dispatch_registry=expected_dispatch_registry) with expected_ctx: - add_dispatch(d, type(settings_builder), type(launcher_like)) + add_dispatch(d, type(settings_builder), type(mock_launcher)) @pytest.mark.parametrize( @@ -161,11 +161,11 @@ def test_dispatch_overwriting( ), ) def test_dispatch_can_retrieve_dispatch_info_from_dispatch_registry( - expected_dispatch_registry, launcher_like, settings_builder, type_or_instance + expected_dispatch_registry, mock_launcher, settings_builder, type_or_instance ): d = dispatch.Dispatcher(dispatch_registry=expected_dispatch_registry) assert dispatch._DispatchRegistration( - format_fn, type(launcher_like) + format_fn, type(mock_launcher) ) == d.get_dispatch(type_or_instance(settings_builder)) @@ -209,12 +209,12 @@ def create(cls, exp): "cls, ctx", ( pytest.param( - dispatch.LauncherLike, + dispatch.LauncherProtocol, pytest.raises(TypeError, match="Cannot dispatch to protocol"), id="Cannot dispatch to protocol class", ), pytest.param( - "launcher_like", + "mock_launcher", contextlib.nullcontext(None), id="Can dispatch to protocol implementation", ), @@ -244,7 +244,7 @@ def test_register_dispatch_to_launcher_types(request, cls, ctx): @dataclasses.dataclass -class BufferWriterLauncher(dispatch.LauncherLike[list[str]]): +class BufferWriterLauncher(dispatch.LauncherProtocol[list[str]]): buf: io.StringIO @classmethod @@ -313,7 +313,7 @@ def test_launcher_adapter_correctly_adapts_input_to_launcher(input_, map_, expec id="Errors if launcher types are disparate", ), pytest.param( - "launcher_like", + "mock_launcher", pytest.raises( TypeError, match="^Cannot create launcher adapter.*expected launcher of type .+$", @@ -344,10 +344,10 @@ def test_dispatch_registration_can_configure_adapter_for_existing_launcher_insta ), pytest.param( ( - "launcher_like", - "launcher_like", + "mock_launcher", + "mock_launcher", BufferWriterLauncher(io.StringIO()), - "launcher_like", + "mock_launcher", ), contextlib.nullcontext(None), id="Correctly ignores incompatible launchers instances", @@ -362,9 +362,9 @@ def test_dispatch_registration_can_configure_adapter_for_existing_launcher_insta ), pytest.param( ( - "launcher_like", + "mock_launcher", BufferWriterLauncherSubclass(io.StringIO), - "launcher_like", + "mock_launcher", ), pytest.raises( errors.LauncherNotFoundError, diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index cc57e329f..c71b1d548 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -37,14 +37,14 @@ def test_dragon_class_methods(function, value, flag, result): @pytest.mark.parametrize("nodes", (NOT_SET, 20, 40)) @pytest.mark.parametrize("tasks_per_node", (NOT_SET, 1, 20)) def test_formatting_launch_args_into_request( - echo_executable_like, nodes, tasks_per_node + mock_echo_executable, nodes, tasks_per_node ): args = DragonArgBuilder({}) if nodes is not NOT_SET: args.set_nodes(nodes) if tasks_per_node is not NOT_SET: args.set_tasks_per_node(tasks_per_node) - req = _as_run_request_view(args, echo_executable_like, {}) + req = _as_run_request_view(args, mock_echo_executable, {}) args = { k: v diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 07adb231b..b3cb4108f 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -114,6 +114,6 @@ def test_format_env_vars(): assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] -def test_formatting_returns_original_exe(echo_executable_like): - cmd = _as_local_command(LocalArgBuilder({}), echo_executable_like, {}) +def test_formatting_returns_original_exe(mock_echo_executable): + cmd = _as_local_command(LocalArgBuilder({}), mock_echo_executable, {}) assert tuple(cmd) == ("echo", "hello", "world") diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 9d276c0e7..636d2896f 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -91,6 +91,6 @@ def test_launch_args(): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _as_jsrun_command(JsrunArgBuilder(args), echo_executable_like, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_jsrun_command(JsrunArgBuilder(args), mock_echo_executable, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 16947590c..23df78c92 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -253,6 +253,6 @@ def test_invalid_hostlist_format(launcher): ), ), ) -def test_formatting_launch_args(echo_executable_like, cls, fmt, cmd, args, expected): - fmt_cmd = fmt(cls(args), echo_executable_like, {}) +def test_formatting_launch_args(mock_echo_executable, cls, fmt, cmd, args, expected): + fmt_cmd = fmt(cls(args), mock_echo_executable, {}) assert tuple(fmt_cmd) == (cmd,) + expected diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 47fa64713..18d85a778 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -105,6 +105,6 @@ def test_invalid_hostlist_format(): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _as_pals_command(PalsMpiexecArgBuilder(args), echo_executable_like, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_pals_command(PalsMpiexecArgBuilder(args), mock_echo_executable, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 5935ec690..e3b73aee7 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -288,6 +288,6 @@ def test_set_het_groups(monkeypatch): ), ), ) -def test_formatting_launch_args(echo_executable_like, args, expected): - cmd = _as_srun_command(SlurmArgBuilder(args), echo_executable_like, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_srun_command(SlurmArgBuilder(args), mock_echo_executable, {}) assert tuple(cmd) == expected From 0a1ebba97ac56709c089aa862e7e2afdc8eb2f28 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Sun, 21 Jul 2024 05:50:35 -0500 Subject: [PATCH 28/82] Add tests for `Experiment.start_jobs` --- smartsim/experiment.py | 17 ++- smartsim/settings/dispatch.py | 6 +- tests/test_experiment.py | 188 ++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 tests/test_experiment.py diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 6f1a5744a..8915a620d 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -177,11 +177,12 @@ def __init__(self, name: str, exp_path: str | None = None): experiment""" def start_jobs( - self, *jobs: Job, dispatcher: Dispatcher = DEFAULT_DISPATCHER + self, job: Job, *jobs: Job, dispatcher: Dispatcher = DEFAULT_DISPATCHER ) -> tuple[LaunchedJobID, ...]: """Execute a collection of `Job` instances. - :param jobs: The collection of jobs instances to start + :param job: The job instance to start + :param jobs: A collection of other job instances to start :param dispatcher: The dispatcher that should be used to determine how to start a job based on its settings. If not specified it will default to a dispatcher pre-configured by SmartSim. @@ -190,12 +191,6 @@ def start_jobs( particular execution of the job. """ - if not jobs: - raise TypeError( - f"{type(self).__name__}.start_jobs() missing at least 1 required " - "positional argument of type `Job`" - ) - def _start(job: Job) -> LaunchedJobID: args = job.launch_settings.launch_args env = job.launch_settings.env_vars @@ -222,7 +217,7 @@ def _start(job: Job) -> LaunchedJobID: self._active_launchers.add(launch_config._adapted_launcher) return launch_config.start(exe, env) - return tuple(map(_start, jobs)) + return _start(job), *map(_start, jobs) @_contextualize def start( @@ -592,8 +587,8 @@ def _launch_summary(self, manifest: Manifest) -> None: === Launch Summary === Experiment: {self.name} Experiment Path: {self.exp_path} - Launchers: - {textwrap.indent(" - ", launcher_list)} + Launcher(s): + {textwrap.indent(" - ", launcher_list) if launcher_list else " "} """) if manifest.applications: diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index aceead700..9cb6b7f37 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -245,7 +245,7 @@ def as_program_arguments(self) -> t.Sequence[str]: ... class LauncherProtocol(t.Protocol[_T_contra]): - def start(self, launchable: _T_contra) -> LaunchedJobID: ... + def start(self, launchable: _T_contra, /) -> LaunchedJobID: ... @classmethod def create(cls, exp: Experiment, /) -> Self: ... @@ -276,9 +276,9 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - def start(self, launchable: t.Sequence[str]) -> LaunchedJobID: + def start(self, command: t.Sequence[str]) -> LaunchedJobID: id_ = create_job_id() - exe, *rest = launchable + exe, *rest = command # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest)) return id_ diff --git a/tests/test_experiment.py b/tests/test_experiment.py new file mode 100644 index 000000000..1cd1ee5c3 --- /dev/null +++ b/tests/test_experiment.py @@ -0,0 +1,188 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import annotations + +import dataclasses +import itertools +import tempfile +import typing as t +import uuid +import weakref + +import pytest + +from smartsim.entity import _mock, entity +from smartsim.experiment import Experiment +from smartsim.launchable import job +from smartsim.settings import dispatch, launchSettings +from smartsim.settings.builders import launchArgBuilder + +pytestmark = pytest.mark.group_a + + +@pytest.fixture +def experiment(test_dir): + yield Experiment(f"test-exp-{uuid.uuid4()}", test_dir) + + +@pytest.fixture +def dispatcher(): + d = dispatch.Dispatcher() + to_record = lambda *a: LaunchRecord(*a) + d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) + yield d + + +@pytest.fixture +def job_maker(monkeypatch): + def iter_jobs(): + for i in itertools.count(): + settings = launchSettings.LaunchSettings("local") + monkeypatch.setattr(settings, "_arg_builder", MockLaunchArgs(i)) + yield job.Job(EchoHelloWorldEntity(), settings) + + jobs = iter_jobs() + return lambda: next(jobs) + + +@dataclasses.dataclass(frozen=True) +class NoOpRecordLauncher(dispatch.LauncherProtocol): + created_by_experiment: Experiment + launched_order: list[LaunchRecord] = dataclasses.field(default_factory=list) + ids_to_launched: dict[dispatch.LaunchedJobID, LaunchRecord] = dataclasses.field( + default_factory=dict + ) + + __hash__ = object.__hash__ + + @classmethod + def create(cls, exp): + return cls(exp) + + def start(self, record: LaunchRecord): + id_ = dispatch.create_job_id() + self.launched_order.append(record) + self.ids_to_launched[id_] = record + return id_ + + +@dataclasses.dataclass(frozen=True) +class LaunchRecord: + launch_args: launchArgBuilder.LaunchArgBuilder + entity: entity.SmartSimEntity + env: t.Mapping[str, str | None] + + @classmethod + def from_job(cls, job): + args = job._launch_settings.launch_args + entity = job._entity + env = job._launch_settings.env_vars + return cls(args, entity, env) + + +class MockLaunchArgs(launchArgBuilder.LaunchArgBuilder): + def __init__(self, count): + super().__init__({}) + self.count = count + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return other.count == self.count + + def launcher_str(self): + return "test-launch-args" + + def set(self, arg, val): ... + + +class EchoHelloWorldEntity(entity.SmartSimEntity): + def __init__(self): + path = tempfile.TemporaryDirectory() + self._finalizer = weakref.finalize(self, path.cleanup) + super().__init__("test-entity", path, _mock.Mock()) + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return self.as_program_arguments() == other.as_program_arguments() + + def as_program_arguments(self): + return ("echo", "Hello", "World!") + + +def test_start_raises_if_no_args_supplied(experiment): + with pytest.raises(TypeError, match="missing 1 required positional argument"): + experiment.start_jobs() + + +# fmt: off +@pytest.mark.parametrize( + "num_jobs", [pytest.param(i, id=f"{i} job(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)] +) +@pytest.mark.parametrize( + "make_jobs", ( + pytest.param(lambda maker, n: tuple(maker() for _ in range(n)), id="many job instances"), + pytest.param(lambda maker, n: (maker(),) * n , id="same job instance many times"), + ), +) +# fmt: on +def test_start_can_launch_jobs(experiment, job_maker, dispatcher, make_jobs, num_jobs): + jobs = make_jobs(job_maker, num_jobs) + assert len(experiment._active_launchers) == 0, "Initialized w/ launchers" + launched_ids = experiment.start_jobs(*jobs, dispatcher=dispatcher) + assert len(experiment._active_launchers) == 1, "Unexpected number of launchers" + (launcher,) = experiment._active_launchers + assert isinstance(launcher, NoOpRecordLauncher), "Unexpected launcher type" + assert launcher.created_by_experiment is experiment, "Not created by experiment" + assert ( + len(jobs) == len(launcher.launched_order) == len(launched_ids) == num_jobs + ), "Inconsistent number of jobs/launched jobs/launched ids/expected number of jobs" + expected_launched = [LaunchRecord.from_job(job) for job in jobs] + assert expected_launched == list(launcher.launched_order), "Unexpected launch order" + expected_id_map = dict(zip(launched_ids, expected_launched)) + assert expected_id_map == launcher.ids_to_launched, "IDs returned in wrong order" + + +@pytest.mark.parametrize( + "num_starts", + [pytest.param(i, id=f"{i} start(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)], +) +def test_start_can_start_a_job_multiple_times_accross_multiple_calls( + experiment, job_maker, dispatcher, num_starts +): + assert len(experiment._active_launchers) == 0, "Initialized w/ launchers" + job = job_maker() + ids_to_launches = { + experiment.start_jobs(job, dispatcher=dispatcher)[0]: LaunchRecord.from_job(job) + for _ in range(num_starts) + } + assert len(experiment._active_launchers) == 1, "Did not reuse the launcher" + (launcher,) = experiment._active_launchers + assert isinstance(launcher, NoOpRecordLauncher), "Unexpected launcher type" + assert len(launcher.launched_order) == num_starts, "Unexpected number launches" + assert ids_to_launches == launcher.ids_to_launched, "Job was not re-launched" From b729e9152cfc0c7435c606f151337e2525d86e51 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Mon, 22 Jul 2024 12:59:43 -0500 Subject: [PATCH 29/82] Rename `Builder` -> `Arguments` --- .../_core/launcher/dragon/dragonLauncher.py | 8 ++- .../{builders => arguments}/__init__.py | 6 +- .../{builders => arguments}/batch/__init__.py | 12 ++-- .../{builders => arguments}/batch/lsf.py | 4 +- .../{builders => arguments}/batch/pbs.py | 4 +- .../{builders => arguments}/batch/slurm.py | 4 +- .../batchArguments.py} | 2 +- .../settings/arguments/launch/__init__.py | 19 ++++++ .../{builders => arguments}/launch/alps.py | 4 +- .../{builders => arguments}/launch/dragon.py | 4 +- .../{builders => arguments}/launch/local.py | 4 +- .../{builders => arguments}/launch/lsf.py | 4 +- .../{builders => arguments}/launch/mpi.py | 10 ++-- .../{builders => arguments}/launch/pals.py | 4 +- .../{builders => arguments}/launch/slurm.py | 4 +- .../launchArguments.py} | 7 +-- smartsim/settings/batchSettings.py | 34 ++++++----- smartsim/settings/builders/launch/__init__.py | 19 ------ smartsim/settings/dispatch.py | 21 ++++--- smartsim/settings/launchSettings.py | 60 ++++++++++--------- tests/temp_tests/test_settings/conftest.py | 10 ++-- .../test_settings/test_alpsLauncher.py | 15 +++-- .../temp_tests/test_settings/test_dispatch.py | 48 +++++++-------- .../test_settings/test_dragonLauncher.py | 6 +- .../test_settings/test_localLauncher.py | 9 ++- .../test_settings/test_lsfLauncher.py | 13 ++-- .../test_settings/test_mpiLauncher.py | 26 ++++---- .../test_settings/test_palsLauncher.py | 8 +-- .../test_settings/test_pbsScheduler.py | 4 +- .../test_settings/test_slurmLauncher.py | 13 ++-- .../test_settings/test_slurmScheduler.py | 4 +- tests/test_experiment.py | 8 +-- 32 files changed, 211 insertions(+), 187 deletions(-) rename smartsim/settings/{builders => arguments}/__init__.py (90%) rename smartsim/settings/{builders => arguments}/batch/__init__.py (87%) rename smartsim/settings/{builders => arguments}/batch/lsf.py (98%) rename smartsim/settings/{builders => arguments}/batch/pbs.py (98%) rename smartsim/settings/{builders => arguments}/batch/slurm.py (98%) rename smartsim/settings/{builders/batchArgBuilder.py => arguments/batchArguments.py} (99%) create mode 100644 smartsim/settings/arguments/launch/__init__.py rename smartsim/settings/{builders => arguments}/launch/alps.py (98%) rename smartsim/settings/{builders => arguments}/launch/dragon.py (96%) rename smartsim/settings/{builders => arguments}/launch/local.py (97%) rename smartsim/settings/{builders => arguments}/launch/lsf.py (97%) rename smartsim/settings/{builders => arguments}/launch/mpi.py (96%) rename smartsim/settings/{builders => arguments}/launch/pals.py (98%) rename smartsim/settings/{builders => arguments}/launch/slurm.py (99%) rename smartsim/settings/{builders/launchArgBuilder.py => arguments/launchArguments.py} (95%) delete mode 100644 smartsim/settings/builders/launch/__init__.py diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 078a89278..2a7182eea 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -340,12 +340,12 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: # TODO: Remove this registry and move back to builder file after fixing # circular import caused by `DragonLauncher.supported_rs` # ----------------------------------------------------------------------------- -from smartsim.settings.builders.launch.dragon import DragonArgBuilder +from smartsim.settings.arguments.launch.dragon import DragonLaunchArguments from smartsim.settings.dispatch import ExecutableProtocol, dispatch def _as_run_request_view( - run_req_args: DragonArgBuilder, + run_req_args: DragonLaunchArguments, exe: ExecutableProtocol, env: t.Mapping[str, str | None], ) -> DragonRunRequestView: @@ -369,5 +369,7 @@ def _as_run_request_view( ) -dispatch(DragonArgBuilder, with_format=_as_run_request_view, to_launcher=DragonLauncher) +dispatch( + DragonLaunchArguments, with_format=_as_run_request_view, to_launcher=DragonLauncher +) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/smartsim/settings/builders/__init__.py b/smartsim/settings/arguments/__init__.py similarity index 90% rename from smartsim/settings/builders/__init__.py rename to smartsim/settings/arguments/__init__.py index 9cfdd5f9c..cd216526c 100644 --- a/smartsim/settings/builders/__init__.py +++ b/smartsim/settings/arguments/__init__.py @@ -24,7 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .batchArgBuilder import BatchArgBuilder -from .launchArgBuilder import LaunchArgBuilder +from .batchArguments import BatchArguments +from .launchArguments import LaunchArguments -__all__ = ["LaunchArgBuilder", "BatchArgBuilder"] +__all__ = ["LaunchArguments", "BatchArguments"] diff --git a/smartsim/settings/builders/batch/__init__.py b/smartsim/settings/arguments/batch/__init__.py similarity index 87% rename from smartsim/settings/builders/batch/__init__.py rename to smartsim/settings/arguments/batch/__init__.py index 41dcbbfc2..e6dc055ea 100644 --- a/smartsim/settings/builders/batch/__init__.py +++ b/smartsim/settings/arguments/batch/__init__.py @@ -24,12 +24,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .lsf import BsubBatchArgBuilder -from .pbs import QsubBatchArgBuilder -from .slurm import SlurmBatchArgBuilder +from .lsf import BsubBatchArguments +from .pbs import QsubBatchArguments +from .slurm import SlurmBatchArguments __all__ = [ - "BsubBatchArgBuilder", - "QsubBatchArgBuilder", - "SlurmBatchArgBuilder", + "BsubBatchArguments", + "QsubBatchArguments", + "SlurmBatchArguments", ] diff --git a/smartsim/settings/builders/batch/lsf.py b/smartsim/settings/arguments/batch/lsf.py similarity index 98% rename from smartsim/settings/builders/batch/lsf.py rename to smartsim/settings/arguments/batch/lsf.py index 4bb7bbd27..4f6e80a70 100644 --- a/smartsim/settings/builders/batch/lsf.py +++ b/smartsim/settings/arguments/batch/lsf.py @@ -32,12 +32,12 @@ from ...batchCommand import SchedulerType from ...common import StringArgument -from ..batchArgBuilder import BatchArgBuilder +from ..batchArguments import BatchArguments logger = get_logger(__name__) -class BsubBatchArgBuilder(BatchArgBuilder): +class BsubBatchArguments(BatchArguments): def scheduler_str(self) -> str: """Get the string representation of the scheduler""" return SchedulerType.Lsf.value diff --git a/smartsim/settings/builders/batch/pbs.py b/smartsim/settings/arguments/batch/pbs.py similarity index 98% rename from smartsim/settings/builders/batch/pbs.py rename to smartsim/settings/arguments/batch/pbs.py index d04b4beba..d67f1be7b 100644 --- a/smartsim/settings/builders/batch/pbs.py +++ b/smartsim/settings/arguments/batch/pbs.py @@ -34,12 +34,12 @@ from ....error import SSConfigError from ...batchCommand import SchedulerType from ...common import StringArgument -from ..batchArgBuilder import BatchArgBuilder +from ..batchArguments import BatchArguments logger = get_logger(__name__) -class QsubBatchArgBuilder(BatchArgBuilder): +class QsubBatchArguments(BatchArguments): def scheduler_str(self) -> str: """Get the string representation of the scheduler""" return SchedulerType.Pbs.value diff --git a/smartsim/settings/builders/batch/slurm.py b/smartsim/settings/arguments/batch/slurm.py similarity index 98% rename from smartsim/settings/builders/batch/slurm.py rename to smartsim/settings/arguments/batch/slurm.py index 5a03f5acd..eca26176d 100644 --- a/smartsim/settings/builders/batch/slurm.py +++ b/smartsim/settings/arguments/batch/slurm.py @@ -33,12 +33,12 @@ from ...batchCommand import SchedulerType from ...common import StringArgument -from ..batchArgBuilder import BatchArgBuilder +from ..batchArguments import BatchArguments logger = get_logger(__name__) -class SlurmBatchArgBuilder(BatchArgBuilder): +class SlurmBatchArguments(BatchArguments): def scheduler_str(self) -> str: """Get the string representation of the scheduler""" return SchedulerType.Slurm.value diff --git a/smartsim/settings/builders/batchArgBuilder.py b/smartsim/settings/arguments/batchArguments.py similarity index 99% rename from smartsim/settings/builders/batchArgBuilder.py rename to smartsim/settings/arguments/batchArguments.py index ad466f254..a85148697 100644 --- a/smartsim/settings/builders/batchArgBuilder.py +++ b/smartsim/settings/arguments/batchArguments.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) -class BatchArgBuilder(ABC): +class BatchArguments(ABC): """Abstract base class that defines all generic scheduler argument methods that are not supported. It is the responsibility of child classes for each launcher to translate diff --git a/smartsim/settings/arguments/launch/__init__.py b/smartsim/settings/arguments/launch/__init__.py new file mode 100644 index 000000000..30502394b --- /dev/null +++ b/smartsim/settings/arguments/launch/__init__.py @@ -0,0 +1,19 @@ +from .alps import AprunLaunchArguments +from .dragon import DragonLaunchArguments +from .local import LocalLaunchArguments +from .lsf import JsrunLaunchArguments +from .mpi import MpiexecLaunchArguments, MpirunLaunchArguments, OrterunLaunchArguments +from .pals import PalsMpiexecLaunchArguments +from .slurm import SlurmLaunchArguments + +__all__ = [ + "AprunLaunchArguments", + "DragonLaunchArguments", + "LocalLaunchArguments", + "JsrunLaunchArguments", + "MpiLaunchArguments", + "MpiexecLaunchArguments", + "OrteLaunchArguments", + "PalsMpiexecLaunchArguments", + "SlurmLaunchArguments", +] diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/arguments/launch/alps.py similarity index 98% rename from smartsim/settings/builders/launch/alps.py rename to smartsim/settings/arguments/launch/alps.py index 5826a2de4..e92bc7b85 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/arguments/launch/alps.py @@ -33,14 +33,14 @@ from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) _as_aprun_command = make_shell_format_fn(run_command="aprun") @dispatch(with_format=_as_aprun_command, to_launcher=ShellLauncher) -class AprunArgBuilder(LaunchArgBuilder): +class AprunLaunchArguments(LaunchArguments): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wdir"} diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/arguments/launch/dragon.py similarity index 96% rename from smartsim/settings/builders/launch/dragon.py rename to smartsim/settings/arguments/launch/dragon.py index 4ba793bf7..7a6a5aab4 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/arguments/launch/dragon.py @@ -30,12 +30,12 @@ from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) -class DragonArgBuilder(LaunchArgBuilder): +class DragonLaunchArguments(LaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Dragon.value diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/arguments/launch/local.py similarity index 97% rename from smartsim/settings/builders/launch/local.py rename to smartsim/settings/arguments/launch/local.py index 49ef3ad92..f89299500 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -33,14 +33,14 @@ from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) _as_local_command = make_shell_format_fn(run_command=None) @dispatch(with_format=_as_local_command, to_launcher=ShellLauncher) -class LocalArgBuilder(LaunchArgBuilder): +class LocalLaunchArguments(LaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Local.value diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py similarity index 97% rename from smartsim/settings/builders/launch/lsf.py rename to smartsim/settings/arguments/launch/lsf.py index 1fe8ea30b..83e5cdc94 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -33,14 +33,14 @@ from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) _as_jsrun_command = make_shell_format_fn(run_command="jsrun") @dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) -class JsrunArgBuilder(LaunchArgBuilder): +class JsrunLaunchArguments(LaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Lsf.value diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/arguments/launch/mpi.py similarity index 96% rename from smartsim/settings/builders/launch/mpi.py rename to smartsim/settings/arguments/launch/mpi.py index a0d947418..edd93b4ac 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/arguments/launch/mpi.py @@ -33,7 +33,7 @@ from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) _as_mpirun_command = make_shell_format_fn("mpirun") @@ -41,7 +41,7 @@ _as_orterun_command = make_shell_format_fn("orterun") -class _BaseMPIArgBuilder(LaunchArgBuilder): +class _BaseMPILaunchArguments(LaunchArguments): def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wd", "wdir"} @@ -219,21 +219,21 @@ def set(self, key: str, value: str | None) -> None: @dispatch(with_format=_as_mpirun_command, to_launcher=ShellLauncher) -class MpiArgBuilder(_BaseMPIArgBuilder): +class MpirunLaunchArguments(_BaseMPILaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpirun.value @dispatch(with_format=_as_mpiexec_command, to_launcher=ShellLauncher) -class MpiexecArgBuilder(_BaseMPIArgBuilder): +class MpiexecLaunchArguments(_BaseMPILaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Mpiexec.value @dispatch(with_format=_as_orterun_command, to_launcher=ShellLauncher) -class OrteArgBuilder(_BaseMPIArgBuilder): +class OrterunLaunchArguments(_BaseMPILaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Orterun.value diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/arguments/launch/pals.py similarity index 98% rename from smartsim/settings/builders/launch/pals.py rename to smartsim/settings/arguments/launch/pals.py index eeb438455..ed4de2131 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/arguments/launch/pals.py @@ -33,14 +33,14 @@ from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) _as_pals_command = make_shell_format_fn(run_command="mpiexec") @dispatch(with_format=_as_pals_command, to_launcher=ShellLauncher) -class PalsMpiexecArgBuilder(LaunchArgBuilder): +class PalsMpiexecLaunchArguments(LaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Pals.value diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py similarity index 99% rename from smartsim/settings/builders/launch/slurm.py rename to smartsim/settings/arguments/launch/slurm.py index f80a5b8b9..bba79b969 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -35,14 +35,14 @@ from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) _as_srun_command = make_shell_format_fn(run_command="srun") @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) -class SlurmArgBuilder(LaunchArgBuilder): +class SlurmLaunchArguments(LaunchArguments): def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Slurm.value diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/arguments/launchArguments.py similarity index 95% rename from smartsim/settings/builders/launchArgBuilder.py rename to smartsim/settings/arguments/launchArguments.py index 8ebfdb0f0..1a407a252 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/arguments/launchArguments.py @@ -37,7 +37,7 @@ logger = get_logger(__name__) -class LaunchArgBuilder(ABC): +class LaunchArguments(ABC): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the responsibility of child classes for each launcher to translate @@ -45,10 +45,9 @@ class LaunchArgBuilder(ABC): """ def __init__(self, launch_args: t.Dict[str, str | None] | None) -> None: - """Initialize a new `LaunchArgBuilder` instance. + """Initialize a new `LaunchArguments` instance. - :param launch_args: A mapping of argument to be used to initialize the - argument builder. + :param launch_args: A mapping of arguments to values to pre-initialize """ self._launch_args = copy.deepcopy(launch_args) or {} diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index 79a559ecb..6649fa5f7 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -32,12 +32,12 @@ from smartsim.log import get_logger from .._core.utils.helpers import fmt_dict +from .arguments import BatchArguments +from .arguments.batch.lsf import BsubBatchArguments +from .arguments.batch.pbs import QsubBatchArguments +from .arguments.batch.slurm import SlurmBatchArguments from .baseSettings import BaseSettings from .batchCommand import SchedulerType -from .builders import BatchArgBuilder -from .builders.batch.lsf import BsubBatchArgBuilder -from .builders.batch.pbs import QsubBatchArgBuilder -from .builders.batch.slurm import SlurmBatchArgBuilder from .common import StringArgument logger = get_logger(__name__) @@ -54,7 +54,7 @@ def __init__( self._batch_scheduler = SchedulerType(batch_scheduler) except ValueError: raise ValueError(f"Invalid scheduler type: {batch_scheduler}") from None - self._arg_builder = self._get_arg_builder(scheduler_args) + self._arguments = self._get_arguments(scheduler_args) self.env_vars = env_vars or {} @property @@ -68,9 +68,9 @@ def batch_scheduler(self) -> str: return self._batch_scheduler.value @property - def scheduler_args(self) -> BatchArgBuilder: + def scheduler_args(self) -> BatchArguments: """Return the batch argument translator.""" - return self._arg_builder + return self._arguments @property def env_vars(self) -> StringArgument: @@ -82,16 +82,20 @@ def env_vars(self, value: t.Dict[str, str | None]) -> None: """Set the environment variables.""" self._env_vars = copy.deepcopy(value) - def _get_arg_builder( - self, scheduler_args: StringArgument | None - ) -> BatchArgBuilder: - """Map the Scheduler to the BatchArgBuilder""" + def _get_arguments(self, scheduler_args: StringArgument | None) -> BatchArguments: + """Map the Scheduler to the BatchArguments. This method should only be + called once during construction. + + :param scheduler_args: A mapping of arguments names to values to be + used to initialize the arguments + :returns: The appropriate type for the settings instance. + """ if self._batch_scheduler == SchedulerType.Slurm: - return SlurmBatchArgBuilder(scheduler_args) + return SlurmBatchArguments(scheduler_args) elif self._batch_scheduler == SchedulerType.Lsf: - return BsubBatchArgBuilder(scheduler_args) + return BsubBatchArguments(scheduler_args) elif self._batch_scheduler == SchedulerType.Pbs: - return QsubBatchArgBuilder(scheduler_args) + return QsubBatchArguments(scheduler_args) else: raise ValueError(f"Invalid scheduler type: {self._batch_scheduler}") @@ -100,7 +104,7 @@ def format_batch_args(self) -> t.List[str]: :return: batch arguments for Sbatch """ - return self._arg_builder.format_batch_args() + return self._arguments.format_batch_args() def __str__(self) -> str: # pragma: no-cover string = f"\nScheduler: {self.scheduler}{self.scheduler_args}" diff --git a/smartsim/settings/builders/launch/__init__.py b/smartsim/settings/builders/launch/__init__.py deleted file mode 100644 index d593c59f7..000000000 --- a/smartsim/settings/builders/launch/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from .alps import AprunArgBuilder -from .dragon import DragonArgBuilder -from .local import LocalArgBuilder -from .lsf import JsrunArgBuilder -from .mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder -from .pals import PalsMpiexecArgBuilder -from .slurm import SlurmArgBuilder - -__all__ = [ - "AprunArgBuilder", - "DragonArgBuilder", - "LocalArgBuilder", - "JsrunArgBuilder", - "MpiArgBuilder", - "MpiexecArgBuilder", - "OrteArgBuilder", - "PalsMpiexecArgBuilder", - "SlurmArgBuilder", -] diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 9cb6b7f37..6fa6fb864 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -39,11 +39,11 @@ if t.TYPE_CHECKING: from smartsim.experiment import Experiment - from smartsim.settings.builders import LaunchArgBuilder + from smartsim.settings.arguments import LaunchArguments _Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) -_DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArgBuilder") +_DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") _LaunchableT = t.TypeVar("_LaunchableT") _EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] @@ -59,15 +59,14 @@ @t.final class Dispatcher: """A class capable of deciding which launcher type should be used to launch - a given settings builder type. + a given settings type. """ def __init__( self, *, dispatch_registry: ( - t.Mapping[type[LaunchArgBuilder], _DispatchRegistration[t.Any, t.Any]] - | None + t.Mapping[type[LaunchArguments], _DispatchRegistration[t.Any, t.Any]] | None ) = None, ) -> None: self._dispatch_registry = ( @@ -104,8 +103,8 @@ def dispatch( to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = False, ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]] | None: - """A type safe way to add a mapping of settings builder to launcher to - handle the settings at launch time. + """A type safe way to add a mapping of settings type to launcher type + to handle a settings instance at launch time. """ err_msg: str | None = None if getattr(to_launcher, "_is_protocol", False): @@ -135,8 +134,8 @@ def register(args_: type[_DispatchableT], /) -> type[_DispatchableT]: def get_dispatch( self, args: _DispatchableT | type[_DispatchableT] ) -> _DispatchRegistration[_DispatchableT, _UnkownType]: - """Find a type of launcher that is registered as being able to launch - the output of the provided builder + """Find a type of launcher that is registered as being able to launch a + settings instance of the provided type """ if not isinstance(args, type): args = type(args) @@ -252,9 +251,9 @@ def create(cls, exp: Experiment, /) -> Self: ... def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[LaunchArgBuilder, t.Sequence[str]]: +) -> _FormatterType[LaunchArguments, t.Sequence[str]]: def impl( - args: LaunchArgBuilder, exe: ExecutableProtocol, _env: _EnvironMappingType + args: LaunchArguments, exe: ExecutableProtocol, _env: _EnvironMappingType ) -> t.Sequence[str]: return ( ( diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 0990e2e82..066309d19 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -32,15 +32,19 @@ from smartsim.log import get_logger from .._core.utils.helpers import fmt_dict +from .arguments import LaunchArguments +from .arguments.launch.alps import AprunLaunchArguments +from .arguments.launch.dragon import DragonLaunchArguments +from .arguments.launch.local import LocalLaunchArguments +from .arguments.launch.lsf import JsrunLaunchArguments +from .arguments.launch.mpi import ( + MpiexecLaunchArguments, + MpirunLaunchArguments, + OrterunLaunchArguments, +) +from .arguments.launch.pals import PalsMpiexecLaunchArguments +from .arguments.launch.slurm import SlurmLaunchArguments from .baseSettings import BaseSettings -from .builders import LaunchArgBuilder -from .builders.launch.alps import AprunArgBuilder -from .builders.launch.dragon import DragonArgBuilder -from .builders.launch.local import LocalArgBuilder -from .builders.launch.lsf import JsrunArgBuilder -from .builders.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder -from .builders.launch.pals import PalsMpiexecArgBuilder -from .builders.launch.slurm import SlurmArgBuilder from .common import StringArgument from .launchCommand import LauncherType @@ -58,7 +62,7 @@ def __init__( self._launcher = LauncherType(launcher) except ValueError: raise ValueError(f"Invalid launcher type: {launcher}") - self._arg_builder = self._get_arg_builder(launch_args) + self._arguments = self._get_arguments(launch_args) self.env_vars = env_vars or {} @property @@ -67,9 +71,9 @@ def launcher(self) -> str: return self._launcher.value @property - def launch_args(self) -> LaunchArgBuilder: + def launch_args(self) -> LaunchArguments: """Return the launch argument translator.""" - return self._arg_builder + return self._arguments @launch_args.setter def launch_args(self, args: t.Mapping[str, str]) -> None: @@ -88,32 +92,32 @@ def env_vars(self, value: dict[str, str | None]) -> None: """Set the environment variables.""" self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder: - """Map the Launcher to the LaunchArgBuilder. This method should only be + def _get_arguments(self, launch_args: StringArgument | None) -> LaunchArguments: + """Map the Launcher to the LaunchArguments. This method should only be called once during construction. - :param launch_args: A mapping of argument to be used to initialize the - argument builder. - :returns: The appropriate argument builder for the settings instance. + :param launch_args: A mapping of arguments names to values to be used + to initialize the arguments + :returns: The appropriate type for the settings instance. """ if self._launcher == LauncherType.Slurm: - return SlurmArgBuilder(launch_args) + return SlurmLaunchArguments(launch_args) elif self._launcher == LauncherType.Mpiexec: - return MpiexecArgBuilder(launch_args) + return MpiexecLaunchArguments(launch_args) elif self._launcher == LauncherType.Mpirun: - return MpiArgBuilder(launch_args) + return MpirunLaunchArguments(launch_args) elif self._launcher == LauncherType.Orterun: - return OrteArgBuilder(launch_args) + return OrterunLaunchArguments(launch_args) elif self._launcher == LauncherType.Alps: - return AprunArgBuilder(launch_args) + return AprunLaunchArguments(launch_args) elif self._launcher == LauncherType.Lsf: - return JsrunArgBuilder(launch_args) + return JsrunLaunchArguments(launch_args) elif self._launcher == LauncherType.Pals: - return PalsMpiexecArgBuilder(launch_args) + return PalsMpiexecLaunchArguments(launch_args) elif self._launcher == LauncherType.Dragon: - return DragonArgBuilder(launch_args) + return DragonLaunchArguments(launch_args) elif self._launcher == LauncherType.Local: - return LocalArgBuilder(launch_args) + return LocalLaunchArguments(launch_args) else: raise ValueError(f"Invalid launcher type: {self._launcher}") @@ -143,7 +147,7 @@ def format_env_vars(self) -> t.Union[t.List[str], None]: """Build bash compatible environment variable string for Slurm :returns: the formatted string of environment variables """ - return self._arg_builder.format_env_vars(self._env_vars) + return self._arguments.format_env_vars(self._env_vars) def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]], None]: """Build environment variable string for Slurm @@ -152,7 +156,7 @@ def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]], None]: for more information on this, see the slurm documentation for srun :returns: the formatted string of environment variables """ - return self._arg_builder.format_comma_sep_env_vars(self._env_vars) + return self._arguments.format_comma_sep_env_vars(self._env_vars) def format_launch_args(self) -> t.Union[t.List[str], None]: """Return formatted launch arguments @@ -160,7 +164,7 @@ def format_launch_args(self) -> t.Union[t.List[str], None]: literally with no formatting. :return: list run arguments for these settings """ - return self._arg_builder.format_launch_args() + return self._arguments.format_launch_args() def __str__(self) -> str: # pragma: no-cover string = f"\nLauncher: {self.launcher}{self.launch_args}" diff --git a/tests/temp_tests/test_settings/conftest.py b/tests/temp_tests/test_settings/conftest.py index 1368006a9..3edf5af6b 100644 --- a/tests/temp_tests/test_settings/conftest.py +++ b/tests/temp_tests/test_settings/conftest.py @@ -27,7 +27,7 @@ import pytest from smartsim.settings import dispatch -from smartsim.settings.builders import launchArgBuilder as launch +from smartsim.settings.arguments import launchArguments as launch @pytest.fixture @@ -40,13 +40,13 @@ def as_program_arguments(self): @pytest.fixture -def settings_builder(): - class _SettingsBuilder(launch.LaunchArgBuilder): +def mock_launch_args(): + class _MockLaunchArgs(launch.LaunchArguments): def set(self, arg, val): ... def launcher_str(self): - return "Mock Settings Builder" + return "mock-laucnh-args" - yield _SettingsBuilder({}) + yield _MockLaunchArgs({}) @pytest.fixture diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 367b30c7f..3b3084c45 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.alps import AprunArgBuilder, _as_aprun_command +from smartsim.settings.arguments.launch.alps import ( + AprunLaunchArguments, + _as_aprun_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -86,14 +89,14 @@ def test_launcher_str(): ) def test_alps_class_methods(function, value, flag, result): alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) - assert isinstance(alpsLauncher._arg_builder, AprunArgBuilder) + assert isinstance(alpsLauncher._arguments, AprunLaunchArguments) getattr(alpsLauncher.launch_args, function)(*value) assert alpsLauncher.launch_args._launch_args[flag] == result def test_set_verbose_launch(): alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) - assert isinstance(alpsLauncher._arg_builder, AprunArgBuilder) + assert isinstance(alpsLauncher._arguments, AprunLaunchArguments) alpsLauncher.launch_args.set_verbose_launch(True) assert alpsLauncher.launch_args._launch_args == {"debug": "7"} alpsLauncher.launch_args.set_verbose_launch(False) @@ -102,7 +105,7 @@ def test_set_verbose_launch(): def test_set_quiet_launch(): aprunLauncher = LaunchSettings(launcher=LauncherType.Alps) - assert isinstance(aprunLauncher._arg_builder, AprunArgBuilder) + assert isinstance(aprunLauncher._arguments, AprunLaunchArguments) aprunLauncher.launch_args.set_quiet_launch(True) assert aprunLauncher.launch_args._launch_args == {"quiet": None} aprunLauncher.launch_args.set_quiet_launch(False) @@ -112,7 +115,7 @@ def test_set_quiet_launch(): def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} aprunLauncher = LaunchSettings(launcher=LauncherType.Alps, env_vars=env_vars) - assert isinstance(aprunLauncher._arg_builder, AprunArgBuilder) + assert isinstance(aprunLauncher._arguments, AprunLaunchArguments) aprunLauncher.update_env({"OMP_NUM_THREADS": "10"}) formatted = aprunLauncher.format_env_vars() result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] @@ -183,5 +186,5 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_aprun_command(AprunArgBuilder(args), mock_echo_executable, {}) + cmd = _as_aprun_command(AprunLaunchArguments(args), mock_echo_executable, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_dispatch.py b/tests/temp_tests/test_settings/test_dispatch.py index 673c8998a..78e4ec349 100644 --- a/tests/temp_tests/test_settings/test_dispatch.py +++ b/tests/temp_tests/test_settings/test_dispatch.py @@ -44,36 +44,36 @@ def format_fn(args, exe, env): @pytest.fixture -def expected_dispatch_registry(mock_launcher, settings_builder): +def expected_dispatch_registry(mock_launcher, mock_launch_args): yield { - type(settings_builder): dispatch._DispatchRegistration( + type(mock_launch_args): dispatch._DispatchRegistration( format_fn, type(mock_launcher) ) } def test_declaritive_form_dispatch_declaration( - mock_launcher, settings_builder, expected_dispatch_registry + mock_launcher, mock_launch_args, expected_dispatch_registry ): d = dispatch.Dispatcher() - assert type(settings_builder) == d.dispatch( + assert type(mock_launch_args) == d.dispatch( with_format=format_fn, to_launcher=type(mock_launcher) - )(type(settings_builder)) + )(type(mock_launch_args)) assert d._dispatch_registry == expected_dispatch_registry def test_imperative_form_dispatch_declaration( - mock_launcher, settings_builder, expected_dispatch_registry + mock_launcher, mock_launch_args, expected_dispatch_registry ): d = dispatch.Dispatcher() assert None == d.dispatch( - type(settings_builder), to_launcher=type(mock_launcher), with_format=format_fn + type(mock_launch_args), to_launcher=type(mock_launcher), with_format=format_fn ) assert d._dispatch_registry == expected_dispatch_registry def test_dispatchers_from_same_registry_do_not_cross_polute( - mock_launcher, settings_builder, expected_dispatch_registry + mock_launcher, mock_launch_args, expected_dispatch_registry ): some_starting_registry = {} d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) @@ -86,14 +86,14 @@ def test_dispatchers_from_same_registry_do_not_cross_polute( ) d2.dispatch( - type(settings_builder), with_format=format_fn, to_launcher=type(mock_launcher) + type(mock_launch_args), with_format=format_fn, to_launcher=type(mock_launcher) ) assert d1._dispatch_registry == {} assert d2._dispatch_registry == expected_dispatch_registry def test_copied_dispatchers_do_not_cross_pollute( - mock_launcher, settings_builder, expected_dispatch_registry + mock_launcher, mock_launch_args, expected_dispatch_registry ): some_starting_registry = {} d1 = dispatch.Dispatcher(dispatch_registry=some_starting_registry) @@ -106,7 +106,7 @@ def test_copied_dispatchers_do_not_cross_pollute( ) d2.dispatch( - type(settings_builder), to_launcher=type(mock_launcher), with_format=format_fn + type(mock_launch_args), to_launcher=type(mock_launcher), with_format=format_fn ) assert d1._dispatch_registry == {} assert d2._dispatch_registry == expected_dispatch_registry @@ -145,12 +145,12 @@ def test_dispatch_overwriting( add_dispatch, expected_ctx, mock_launcher, - settings_builder, + mock_launch_args, expected_dispatch_registry, ): d = dispatch.Dispatcher(dispatch_registry=expected_dispatch_registry) with expected_ctx: - add_dispatch(d, type(settings_builder), type(mock_launcher)) + add_dispatch(d, type(mock_launch_args), type(mock_launcher)) @pytest.mark.parametrize( @@ -161,12 +161,12 @@ def test_dispatch_overwriting( ), ) def test_dispatch_can_retrieve_dispatch_info_from_dispatch_registry( - expected_dispatch_registry, mock_launcher, settings_builder, type_or_instance + expected_dispatch_registry, mock_launcher, mock_launch_args, type_or_instance ): d = dispatch.Dispatcher(dispatch_registry=expected_dispatch_registry) assert dispatch._DispatchRegistration( format_fn, type(mock_launcher) - ) == d.get_dispatch(type_or_instance(settings_builder)) + ) == d.get_dispatch(type_or_instance(mock_launch_args)) @pytest.mark.parametrize( @@ -177,13 +177,13 @@ def test_dispatch_can_retrieve_dispatch_info_from_dispatch_registry( ), ) def test_dispatch_raises_if_settings_type_not_registered( - settings_builder, type_or_instance + mock_launch_args, type_or_instance ): d = dispatch.Dispatcher(dispatch_registry={}) with pytest.raises( TypeError, match="No dispatch for `.+?(?=`)` has been registered" ): - d.get_dispatch(type_or_instance(settings_builder)) + d.get_dispatch(type_or_instance(mock_launch_args)) class LauncherABC(abc.ABC): @@ -323,13 +323,13 @@ def test_launcher_adapter_correctly_adapts_input_to_launcher(input_, map_, expec ), ) def test_dispatch_registration_can_configure_adapter_for_existing_launcher_instance( - request, settings_builder, buffer_writer_dispatch, launcher_instance, ctx + request, mock_launch_args, buffer_writer_dispatch, launcher_instance, ctx ): if isinstance(launcher_instance, str): launcher_instance = request.getfixturevalue(launcher_instance) with ctx: adapter = buffer_writer_dispatch.create_adapter_from_launcher( - launcher_instance, settings_builder + launcher_instance, mock_launch_args ) assert adapter._adapted_launcher is launcher_instance @@ -375,7 +375,7 @@ def test_dispatch_registration_can_configure_adapter_for_existing_launcher_insta ), ) def test_dispatch_registration_configures_first_compatible_launcher_from_sequence_of_launchers( - request, settings_builder, buffer_writer_dispatch, launcher_instances, ctx + request, mock_launch_args, buffer_writer_dispatch, launcher_instances, ctx ): def resolve_instance(inst): return request.getfixturevalue(inst) if isinstance(inst, str) else inst @@ -384,24 +384,24 @@ def resolve_instance(inst): with ctx: adapter = buffer_writer_dispatch.configure_first_compatible_launcher( - with_settings=settings_builder, from_available_launchers=launcher_instances + with_settings=mock_launch_args, from_available_launchers=launcher_instances ) def test_dispatch_registration_can_create_a_laucher_for_an_experiment_and_can_reconfigure_it_later( - settings_builder, buffer_writer_dispatch + mock_launch_args, buffer_writer_dispatch ): class MockExperiment: ... exp = MockExperiment() adapter_1 = buffer_writer_dispatch.create_new_launcher_configuration( - for_experiment=exp, with_settings=settings_builder + for_experiment=exp, with_settings=mock_launch_args ) assert type(adapter_1._adapted_launcher) == buffer_writer_dispatch.launcher_type existing_launcher = adapter_1._adapted_launcher adapter_2 = buffer_writer_dispatch.create_adapter_from_launcher( - existing_launcher, settings_builder + existing_launcher, mock_launch_args ) assert type(adapter_2._adapted_launcher) == buffer_writer_dispatch.launcher_type assert adapter_1._adapted_launcher is adapter_2._adapted_launcher diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index c71b1d548..0c6fb80ac 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -3,7 +3,7 @@ from smartsim._core.launcher.dragon.dragonLauncher import _as_run_request_view from smartsim._core.schemas.dragonRequests import DragonRunRequestView from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.dragon import DragonArgBuilder +from smartsim.settings.arguments.launch.dragon import DragonLaunchArguments from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -26,7 +26,7 @@ def test_launcher_str(): ) def test_dragon_class_methods(function, value, flag, result): dragonLauncher = LaunchSettings(launcher=LauncherType.Dragon) - assert isinstance(dragonLauncher._arg_builder, DragonArgBuilder) + assert isinstance(dragonLauncher._arguments, DragonLaunchArguments) getattr(dragonLauncher.launch_args, function)(*value) assert dragonLauncher.launch_args._launch_args[flag] == result @@ -39,7 +39,7 @@ def test_dragon_class_methods(function, value, flag, result): def test_formatting_launch_args_into_request( mock_echo_executable, nodes, tasks_per_node ): - args = DragonArgBuilder({}) + args = DragonLaunchArguments({}) if nodes is not NOT_SET: args.set_nodes(nodes) if tasks_per_node is not NOT_SET: diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index b3cb4108f..580e53d36 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.local import LocalArgBuilder, _as_local_command +from smartsim.settings.arguments.launch.local import ( + LocalLaunchArguments, + _as_local_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -110,10 +113,10 @@ def test_format_env_vars(): "D": "12", } localLauncher = LaunchSettings(launcher=LauncherType.Local, env_vars=env_vars) - assert isinstance(localLauncher._arg_builder, LocalArgBuilder) + assert isinstance(localLauncher._arguments, LocalLaunchArguments) assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] def test_formatting_returns_original_exe(mock_echo_executable): - cmd = _as_local_command(LocalArgBuilder({}), mock_echo_executable, {}) + cmd = _as_local_command(LocalLaunchArguments({}), mock_echo_executable, {}) assert tuple(cmd) == ("echo", "hello", "world") diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 636d2896f..c73edb6a9 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.lsf import JsrunArgBuilder, _as_jsrun_command +from smartsim.settings.arguments.launch.lsf import ( + JsrunLaunchArguments, + _as_jsrun_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -24,7 +27,7 @@ def test_launcher_str(): ) def test_lsf_class_methods(function, value, flag, result): lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf) - assert isinstance(lsfLauncher._arg_builder, JsrunArgBuilder) + assert isinstance(lsfLauncher._arguments, JsrunLaunchArguments) getattr(lsfLauncher.launch_args, function)(*value) assert lsfLauncher.launch_args._launch_args[flag] == result @@ -32,7 +35,7 @@ def test_lsf_class_methods(function, value, flag, result): def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": None, "LOGGING": "verbose"} lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf, env_vars=env_vars) - assert isinstance(lsfLauncher._arg_builder, JsrunArgBuilder) + assert isinstance(lsfLauncher._arguments, JsrunLaunchArguments) formatted = lsfLauncher.format_env_vars() assert formatted == ["-E", "OMP_NUM_THREADS", "-E", "LOGGING=verbose"] @@ -47,7 +50,7 @@ def test_launch_args(): "np": 100, } lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf, launch_args=launch_args) - assert isinstance(lsfLauncher._arg_builder, JsrunArgBuilder) + assert isinstance(lsfLauncher._arguments, JsrunLaunchArguments) formatted = lsfLauncher.format_launch_args() result = [ "--latency_priority=gpu-gpu", @@ -92,5 +95,5 @@ def test_launch_args(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_jsrun_command(JsrunArgBuilder(args), mock_echo_executable, {}) + cmd = _as_jsrun_command(JsrunLaunchArguments(args), mock_echo_executable, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 23df78c92..350555ae7 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -3,10 +3,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.mpi import ( - MpiArgBuilder, - MpiexecArgBuilder, - OrteArgBuilder, +from smartsim.settings.arguments.launch.mpi import ( + MpiexecLaunchArguments, + MpirunLaunchArguments, + OrterunLaunchArguments, _as_mpiexec_command, _as_mpirun_command, _as_orterun_command, @@ -107,9 +107,9 @@ def test_launcher_str(launcher): ), ) for l in ( - [LauncherType.Mpirun, MpiArgBuilder], - [LauncherType.Mpiexec, MpiexecArgBuilder], - [LauncherType.Orterun, OrteArgBuilder], + [LauncherType.Mpirun, MpirunLaunchArguments], + [LauncherType.Mpiexec, MpiexecLaunchArguments], + [LauncherType.Orterun, OrterunLaunchArguments], ) ) ) @@ -117,7 +117,7 @@ def test_launcher_str(launcher): ) def test_mpi_class_methods(l, function, value, flag, result): mpiSettings = LaunchSettings(launcher=l[0]) - assert isinstance(mpiSettings._arg_builder, l[1]) + assert isinstance(mpiSettings._arguments, l[1]) getattr(mpiSettings.launch_args, function)(*value) assert mpiSettings.launch_args._launch_args[flag] == result @@ -215,11 +215,15 @@ def test_invalid_hostlist_format(launcher): @pytest.mark.parametrize( "cls, fmt, cmd", ( - pytest.param(MpiArgBuilder, _as_mpirun_command, "mpirun", id="w/ mpirun"), pytest.param( - MpiexecArgBuilder, _as_mpiexec_command, "mpiexec", id="w/ mpiexec" + MpirunLaunchArguments, _as_mpirun_command, "mpirun", id="w/ mpirun" + ), + pytest.param( + MpiexecLaunchArguments, _as_mpiexec_command, "mpiexec", id="w/ mpiexec" + ), + pytest.param( + OrterunLaunchArguments, _as_orterun_command, "orterun", id="w/ orterun" ), - pytest.param(OrteArgBuilder, _as_orterun_command, "orterun", id="w/ orterun"), ), ) @pytest.mark.parametrize( diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 18d85a778..c348fe96f 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.pals import ( - PalsMpiexecArgBuilder, +from smartsim.settings.arguments.launch.pals import ( + PalsMpiexecLaunchArguments, _as_pals_command, ) from smartsim.settings.launchCommand import LauncherType @@ -49,7 +49,7 @@ def test_launcher_str(): ) def test_pals_class_methods(function, value, flag, result): palsLauncher = LaunchSettings(launcher=LauncherType.Pals) - assert isinstance(palsLauncher.launch_args, PalsMpiexecArgBuilder) + assert isinstance(palsLauncher.launch_args, PalsMpiexecLaunchArguments) getattr(palsLauncher.launch_args, function)(*value) assert palsLauncher.launch_args._launch_args[flag] == result assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] @@ -106,5 +106,5 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_pals_command(PalsMpiexecArgBuilder(args), mock_echo_executable, {}) + cmd = _as_pals_command(PalsMpiexecLaunchArguments(args), mock_echo_executable, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index ab3435df5..94da0411a 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import BatchSettings +from smartsim.settings.arguments.batch.pbs import QsubBatchArguments from smartsim.settings.batchCommand import SchedulerType -from smartsim.settings.builders.batch.pbs import QsubBatchArgBuilder def test_scheduler_str(): @@ -35,7 +35,7 @@ def test_scheduler_str(): ) def test_create_pbs_batch(function, value, flag, result): pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.Pbs) - assert isinstance(pbsScheduler.scheduler_args, QsubBatchArgBuilder) + assert isinstance(pbsScheduler.scheduler_args, QsubBatchArguments) getattr(pbsScheduler.scheduler_args, function)(*value) assert pbsScheduler.scheduler_args._scheduler_args[flag] == result diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index e3b73aee7..61c0d55c4 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -1,7 +1,10 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.builders.launch.slurm import SlurmArgBuilder, _as_srun_command +from smartsim.settings.arguments.launch.slurm import ( + SlurmLaunchArguments, + _as_srun_command, +) from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -83,7 +86,7 @@ def test_launcher_str(): ) def test_slurm_class_methods(function, value, flag, result): slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) - assert isinstance(slurmLauncher.launch_args, SlurmArgBuilder) + assert isinstance(slurmLauncher.launch_args, SlurmLaunchArguments) getattr(slurmLauncher.launch_args, function)(*value) assert slurmLauncher.launch_args._launch_args[flag] == result @@ -250,9 +253,9 @@ def test_set_het_groups(monkeypatch): monkeypatch.setenv("SLURM_HET_SIZE", "4") slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) slurmLauncher.launch_args.set_het_group([1]) - assert slurmLauncher._arg_builder._launch_args["het-group"] == "1" + assert slurmLauncher._arguments._launch_args["het-group"] == "1" slurmLauncher.launch_args.set_het_group([3, 2]) - assert slurmLauncher._arg_builder._launch_args["het-group"] == "3,2" + assert slurmLauncher._arguments._launch_args["het-group"] == "3,2" with pytest.raises(ValueError): slurmLauncher.launch_args.set_het_group([4]) @@ -289,5 +292,5 @@ def test_set_het_groups(monkeypatch): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_srun_command(SlurmArgBuilder(args), mock_echo_executable, {}) + cmd = _as_srun_command(SlurmLaunchArguments(args), mock_echo_executable, {}) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index 5c65d367a..38c98b171 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import BatchSettings +from smartsim.settings.arguments.batch.slurm import SlurmBatchArguments from smartsim.settings.batchCommand import SchedulerType -from smartsim.settings.builders.batch.slurm import SlurmBatchArgBuilder def test_scheduler_str(): @@ -57,7 +57,7 @@ def test_create_sbatch(): slurmScheduler = BatchSettings( batch_scheduler=SchedulerType.Slurm, scheduler_args=batch_args ) - assert isinstance(slurmScheduler._arg_builder, SlurmBatchArgBuilder) + assert isinstance(slurmScheduler._arguments, SlurmBatchArguments) args = slurmScheduler.format_batch_args() assert args == ["--exclusive", "--oversubscribe"] diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 1cd1ee5c3..132fb92f3 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -39,7 +39,7 @@ from smartsim.experiment import Experiment from smartsim.launchable import job from smartsim.settings import dispatch, launchSettings -from smartsim.settings.builders import launchArgBuilder +from smartsim.settings.arguments import launchArguments pytestmark = pytest.mark.group_a @@ -62,7 +62,7 @@ def job_maker(monkeypatch): def iter_jobs(): for i in itertools.count(): settings = launchSettings.LaunchSettings("local") - monkeypatch.setattr(settings, "_arg_builder", MockLaunchArgs(i)) + monkeypatch.setattr(settings, "_arguments", MockLaunchArgs(i)) yield job.Job(EchoHelloWorldEntity(), settings) jobs = iter_jobs() @@ -92,7 +92,7 @@ def start(self, record: LaunchRecord): @dataclasses.dataclass(frozen=True) class LaunchRecord: - launch_args: launchArgBuilder.LaunchArgBuilder + launch_args: launchArguments.LaunchArguments entity: entity.SmartSimEntity env: t.Mapping[str, str | None] @@ -104,7 +104,7 @@ def from_job(cls, job): return cls(args, entity, env) -class MockLaunchArgs(launchArgBuilder.LaunchArgBuilder): +class MockLaunchArgs(launchArguments.LaunchArguments): def __init__(self, count): super().__init__({}) self.count = count From 26cb61d4bd3378e4c43346a9ca69f788d3906f82 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 23 Jul 2024 17:24:19 -0500 Subject: [PATCH 30/82] test_generator passing --- smartsim/_core/generation/generator.py | 10 +++++++--- smartsim/_core/launcher/dragon/dragonLauncher.py | 3 ++- smartsim/entity/ensemble.py | 5 ++++- smartsim/experiment.py | 7 ++++--- smartsim/launchable/job.py | 7 ++++--- smartsim/settings/dispatch.py | 7 ++++--- tests/temp_tests/test_settings/test_alpsLauncher.py | 5 ++--- tests/temp_tests/test_settings/test_dragonLauncher.py | 2 +- tests/temp_tests/test_settings/test_lsfLauncher.py | 5 ++--- tests/temp_tests/test_settings/test_mpiLauncher.py | 8 ++++---- tests/temp_tests/test_settings/test_palsLauncher.py | 8 ++++---- tests/temp_tests/test_settings/test_slurmLauncher.py | 6 ++---- tests/test_generator/test_generator.py | 11 +++++++---- 13 files changed, 47 insertions(+), 37 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index c2e287d3e..df3368339 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -77,7 +77,7 @@ def _generate_log_path(self, gen_path: str) -> str: """ log_path = os.path.join(gen_path, "log") return log_path - + def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: """ Generates the path for a job based on its type and ensemble name (if applicable). @@ -89,7 +89,9 @@ def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: """ if job._ensemble_name is None: job_type = f"{job.__class__.__name__.lower()}s" - entity_type = f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" + entity_type = ( + f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" + ) path = os.path.join( gen_path, run_ID, @@ -100,7 +102,9 @@ def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: ) else: job_type = "ensembles" - entity_type = f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" + entity_type = ( + f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" + ) path = os.path.join( gen_path, run_ID, diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 2a7182eea..be6be0212 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -348,6 +348,7 @@ def _as_run_request_view( run_req_args: DragonLaunchArguments, exe: ExecutableProtocol, env: t.Mapping[str, str | None], + path: str, ) -> DragonRunRequestView: exe_, *args = exe.as_program_arguments() return DragonRunRequestView( @@ -359,7 +360,7 @@ def _as_run_request_view( # the command execute next to any generated files. A similar # problem exists for the other settings. # TODO: Find a way to inject this path - path=os.getcwd(), + path=path, env=env, # TODO: Not sure how this info is injected name=None, diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index 883b69f9a..540d9ca84 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -110,4 +110,7 @@ def as_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: apps = self._create_applications() if not apps: raise ValueError("There are no members as part of this ensemble") - return tuple(Job(app, settings, f"job_{i}", ensemble_name=self.name) for i, app in enumerate(apps, 1)) + return tuple( + Job(app, settings, f"job_{i}", ensemble_name=self.name) + for i, app in enumerate(apps, 1) + ) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 2fa8cf34e..8acc741e9 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -163,13 +163,14 @@ def __init__(self, name: str, exp_path: str | None = None): self.exp_path = exp_path """The path under which the experiment operate""" - + self.run_ID = ( "run-" + datetime.datetime.now().strftime("%H:%M:%S") + "-" + datetime.datetime.now().strftime("%Y-%m-%d") ) + """Create the run id for the Experiment""" # TODO: Remove this! The controller is becoming obsolete self._control = Controller(launcher="local") @@ -224,9 +225,9 @@ def _start(job: Job) -> LaunchedJobID: # it easier to monitor job statuses # pylint: disable-next=protected-access self._active_launchers.add(launch_config._adapted_launcher) - #job_execution_path = self._generate(job) + job_execution_path = self._generate(job) - return launch_config.start(exe, env) + return launch_config.start(exe, env, job_execution_path) return _start(job), *map(_start, jobs) diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index 8608beddc..cdb889be1 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -30,9 +30,9 @@ from copy import deepcopy from smartsim._core.commands.launchCommands import LaunchCommands +from smartsim._core.utils.helpers import create_short_id_str from smartsim.launchable.basejob import BaseJob from smartsim.settings import LaunchSettings -from smartsim._core.utils.helpers import create_short_id_str if t.TYPE_CHECKING: from smartsim.entity.entity import SmartSimEntity @@ -55,13 +55,14 @@ def __init__( entity: SmartSimEntity, launch_settings: LaunchSettings, name: str = "job", - **kwargs: t.Any, + *, + ensemble_name: str = None, ): super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) self._name = deepcopy(name) - self._ensemble_name = kwargs.get('ensemble_name', None) + self._ensemble_name = ensemble_name if self._ensemble_name is not None: self._ensemble_name += f"-{create_short_id_str()}" # TODO: self.warehouse_runner = JobWarehouseRunner diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 81fa79442..64b10dbfc 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -275,12 +275,13 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - - def start(self, command: t.Sequence[str], job_execution_path: str) -> LaunchedJobID: + def start(self, command: t.Sequence[str], job_execution_path: str) -> LaunchedJobID: id_ = create_job_id() exe, *rest = command # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=job_execution_path) + self._launched[id_] = sp.Popen( + (helpers.expand_exe_path(exe), *rest), cwd=job_execution_path + ) return id_ @classmethod diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 82c1d65b2..3b3084c45 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -185,7 +185,6 @@ def test_invalid_exclude_hostlist_format(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_aprun_command(AprunLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_aprun_command(AprunLaunchArguments(args), mock_echo_executable, {}) assert tuple(cmd) == expected - assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 4280a7fe8..a73ec64a6 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -44,7 +44,7 @@ def test_formatting_launch_args_into_request( args.set_nodes(nodes) if tasks_per_node is not NOT_SET: args.set_tasks_per_node(tasks_per_node) - req = _as_run_request_view(args, mock_echo_executable, {}) + req = _as_run_request_view(args, mock_echo_executable, {}, test_dir) args = { k: v diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 08c234416..c73edb6a9 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -94,7 +94,6 @@ def test_launch_args(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_jsrun_command(JsrunLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_jsrun_command(JsrunLaunchArguments(args), mock_echo_executable, {}) assert tuple(cmd) == expected - assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 314002d63..70fec42e8 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -257,8 +257,8 @@ def test_invalid_hostlist_format(launcher): ), ), ) - -def test_formatting_launch_args(mock_echo_executable, cls, fmt, cmd, args, expected, test_dir): - fmt_cmd, path = fmt(cls(args), mock_echo_executable, {}) +def test_formatting_launch_args( + mock_echo_executable, cls, fmt, cmd, args, expected +): + fmt_cmd = fmt(cls(args), mock_echo_executable, {}) assert tuple(fmt_cmd) == (cmd,) + expected - assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 8bb9bae14..afd50c219 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -105,8 +105,8 @@ def test_invalid_hostlist_format(): ), ), ) - -def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_pals_command(PalsMpiexecLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_pals_command( + PalsMpiexecLaunchArguments(args), mock_echo_executable, {} + ) assert tuple(cmd) == expected - assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 89ca36945..61c0d55c4 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -291,8 +291,6 @@ def test_set_het_groups(monkeypatch): ), ), ) - -def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_srun_command(SlurmLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected): + cmd = _as_srun_command(SlurmLaunchArguments(args), mock_echo_executable, {}) assert tuple(cmd) == expected - assert path == test_dir diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index 72be67b3c..c5f8b8b86 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -9,7 +9,7 @@ from smartsim._core.generation.generator import Generator from smartsim.entity import Application, Ensemble from smartsim.launchable import Job, JobGroup -from smartsim.settings.builders.launch import SlurmArgBuilder +from smartsim.settings.arguments.launch import SlurmLaunchArguments from smartsim.settings.dispatch import Dispatcher from smartsim.settings.launchSettings import LaunchSettings @@ -22,6 +22,8 @@ def create(cls, _): def start(self, _): return "anything" +def make_shell_format_fn(run_command: str | None): ... + class EchoApp: def as_program_arguments(self): return ["echo", "Hello", "World!"] @@ -89,13 +91,14 @@ def test_generate_job_directory(gen_instance_for_job): def test_full_exp_generate_job_directory(test_dir, job_instance): no_op_dispatch = Dispatcher() - no_op_dispatch.dispatch(SlurmArgBuilder, to_launcher=NoOpLauncher) + no_op_dispatch.dispatch(SlurmLaunchArguments, with_format=make_shell_format_fn("run_command"), to_launcher=NoOpLauncher) no_op_exp = Experiment( - name="No-Op-Exp", exp_path=test_dir, settings_dispatcher=no_op_dispatch + name="No-Op-Exp", exp_path=test_dir ) job_execution_path = no_op_exp._generate(job_instance) assert osp.isdir(job_execution_path) + def test_generate_ensemble_directory(test_dir, wlmutils): ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) @@ -104,4 +107,4 @@ def test_generate_ensemble_directory(test_dir, wlmutils): run_ID = "temp_run" gen = Generator(gen_path=test_dir, run_ID=run_ID, job=job) gen.generate_experiment() - assert osp.isdir(gen.path) \ No newline at end of file + assert osp.isdir(gen.path) From 20d8a8eff21302f4705f83a8334ff2e51e7199d4 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 23 Jul 2024 18:29:14 -0500 Subject: [PATCH 31/82] settings tests passing and mypy errors resolved --- .../_core/launcher/dragon/dragonLauncher.py | 2 +- smartsim/launchable/job.py | 2 +- smartsim/settings/dispatch.py | 22 +++++++++---------- .../test_settings/test_alpsLauncher.py | 5 +++-- .../test_settings/test_dragonLauncher.py | 2 +- .../test_settings/test_localLauncher.py | 5 +++-- .../test_settings/test_lsfLauncher.py | 5 +++-- .../test_settings/test_mpiLauncher.py | 5 +++-- .../test_settings/test_palsLauncher.py | 7 +++--- .../test_settings/test_slurmLauncher.py | 5 +++-- tests/test_experiment.py | 2 +- 11 files changed, 34 insertions(+), 28 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index be6be0212..8fcce1b20 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -347,8 +347,8 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: def _as_run_request_view( run_req_args: DragonLaunchArguments, exe: ExecutableProtocol, - env: t.Mapping[str, str | None], path: str, + env: t.Mapping[str, str | None], ) -> DragonRunRequestView: exe_, *args = exe.as_program_arguments() return DragonRunRequestView( diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index cdb889be1..c3a97acde 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -56,7 +56,7 @@ def __init__( launch_settings: LaunchSettings, name: str = "job", *, - ensemble_name: str = None, + ensemble_name: t.Optional[str] = None, ): super().__init__() self._entity = deepcopy(entity) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 64b10dbfc..e0c456d58 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -48,10 +48,10 @@ _EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _EnvironMappingType], _LaunchableT + [_DispatchableT, "ExecutableProtocol", str, _EnvironMappingType], _LaunchableT ] _LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType]" + "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType, str]" ) _UnkownType: TypeAlias = t.NoReturn @@ -185,8 +185,8 @@ def create_adapter_from_launcher( f"exactly `{self.launcher_type}`" ) - def format_(exe: ExecutableProtocol, env: _EnvironMappingType) -> _LaunchableT: - return self.formatter(settings, exe, env) + def format_(exe: ExecutableProtocol, env: _EnvironMappingType, path: str) -> _LaunchableT: + return self.formatter(settings, exe, path, env) return _LauncherAdapter(launcher, format_) @@ -251,10 +251,10 @@ def create(cls, exp: Experiment, /) -> Self: ... def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[LaunchArguments, t.Sequence[str]]: +) -> _FormatterType[LaunchArguments, tuple[t.Sequence[str],str]]: def impl( - args: LaunchArguments, exe: ExecutableProtocol, _env: _EnvironMappingType - ) -> t.Sequence[str]: + args: LaunchArguments, exe: ExecutableProtocol, path: str, _env: _EnvironMappingType + ) -> t.Tuple[t.Sequence[str], str]: return ( ( run_command, @@ -264,7 +264,7 @@ def impl( ) if run_command is not None else exe.as_program_arguments() - ) + ), path return impl @@ -274,13 +274,13 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - - def start(self, command: t.Sequence[str], job_execution_path: str) -> LaunchedJobID: + def start(self, payload: tuple[t.Sequence[str], str]) -> LaunchedJobID: + command, path = payload id_ = create_job_id() exe, *rest = command # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen( - (helpers.expand_exe_path(exe), *rest), cwd=job_execution_path + (helpers.expand_exe_path(exe), *rest), cwd=path ) return id_ diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 3b3084c45..4746930e6 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -185,6 +185,7 @@ def test_invalid_exclude_hostlist_format(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_aprun_command(AprunLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + cmd, path = _as_aprun_command(AprunLaunchArguments(args), mock_echo_executable, test_dir, {}) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index a73ec64a6..31f72cc99 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -44,7 +44,7 @@ def test_formatting_launch_args_into_request( args.set_nodes(nodes) if tasks_per_node is not NOT_SET: args.set_tasks_per_node(tasks_per_node) - req = _as_run_request_view(args, mock_echo_executable, {}, test_dir) + req = _as_run_request_view(args, mock_echo_executable, test_dir, {}) args = { k: v diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 580e53d36..9d0ec13f0 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -117,6 +117,7 @@ def test_format_env_vars(): assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] -def test_formatting_returns_original_exe(mock_echo_executable): - cmd = _as_local_command(LocalLaunchArguments({}), mock_echo_executable, {}) +def test_formatting_returns_original_exe(mock_echo_executable, test_dir): + cmd, path = _as_local_command(LocalLaunchArguments({}), mock_echo_executable, test_dir, {}) assert tuple(cmd) == ("echo", "hello", "world") + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index c73edb6a9..73335a204 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -94,6 +94,7 @@ def test_launch_args(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_jsrun_command(JsrunLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + cmd, path = _as_jsrun_command(JsrunLaunchArguments(args), mock_echo_executable, test_dir, {}) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 70fec42e8..0a1f9c301 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -258,7 +258,8 @@ def test_invalid_hostlist_format(launcher): ), ) def test_formatting_launch_args( - mock_echo_executable, cls, fmt, cmd, args, expected + mock_echo_executable, cls, fmt, cmd, args, expected, test_dir ): - fmt_cmd = fmt(cls(args), mock_echo_executable, {}) + fmt_cmd, path = fmt(cls(args), mock_echo_executable, test_dir, {}) assert tuple(fmt_cmd) == (cmd,) + expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index afd50c219..12a17df59 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -105,8 +105,9 @@ def test_invalid_hostlist_format(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_pals_command( - PalsMpiexecLaunchArguments(args), mock_echo_executable, {} +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + cmd, path = _as_pals_command( + PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {} ) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 61c0d55c4..6c62dccc9 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -291,6 +291,7 @@ def test_set_het_groups(monkeypatch): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_srun_command(SlurmLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + cmd, path = _as_srun_command(SlurmLaunchArguments(args), mock_echo_executable, test_dir, {}) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 132fb92f3..ddbb6f4fc 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -124,7 +124,7 @@ class EchoHelloWorldEntity(entity.SmartSimEntity): def __init__(self): path = tempfile.TemporaryDirectory() self._finalizer = weakref.finalize(self, path.cleanup) - super().__init__("test-entity", path, _mock.Mock()) + super().__init__("test-entity", _mock.Mock()) def __eq__(self, other): if type(self) is not type(other): From 9ac3762145f9c73f90378b850a745d5d7d35695c Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 24 Jul 2024 10:38:18 -0500 Subject: [PATCH 32/82] doc string updates --- smartsim/_core/generation/generator.py | 25 +++++++++++++++---------- tests/test_experiment.py | 11 ++++++++--- tests/test_generator/test_generator.py | 9 +++++---- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index df3368339..dc932c4ba 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -49,8 +49,8 @@ class Generator: """The primary job of the generator is to create the file structure - for a SmartSim experiment. The Generator is also responsible for reading - and writing into configuration files. + for a SmartSim Experiment. The Generator is also responsible for + writing files into a Job directory. """ def __init__(self, gen_path: str, run_ID: str, job: Job) -> None: @@ -62,31 +62,35 @@ def __init__(self, gen_path: str, run_ID: str, job: Job) -> None: configuration, and file copying within the job directory. :param gen_path: Path in which files need to be generated - :param job: Reference to a SmartSimEntity and LaunchSettings + :param job: Reference to a name, SmartSimEntity and LaunchSettings """ self.job = job + # Generate the job folder path self.path = self._generate_job_path(job, gen_path, run_ID) + # Generate the log folder path self.log_path = self._generate_log_path(gen_path) def _generate_log_path(self, gen_path: str) -> str: """ - Generates the path for logs. + Generate the path for the log folder. - :param gen_path: The base path for job generation - :returns str: The generated path for the log directory. + :param gen_path: The base path job generation + :returns str: The generated path for the log directory """ log_path = os.path.join(gen_path, "log") return log_path def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: """ - Generates the path for a job based on its type and ensemble name (if applicable). + Generates the directory path for a job based on its creation type + (whether created via ensemble or job init). :param job: The Job object :param gen_path: The base path for job generation - :param run_ID: The unique run ID + :param run_ID: The experiments unique run ID :returns str: The generated path for the job. """ + # Attr set in Job to check if Job was created by an Ensemble if job._ensemble_name is None: job_type = f"{job.__class__.__name__.lower()}s" entity_type = ( @@ -100,6 +104,7 @@ def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: entity_type, "run", ) + # Job was created via Ensemble else: job_type = "ensembles" entity_type = ( @@ -148,11 +153,11 @@ def log_file(self) -> str: return join(self.path, "smartsim_params.txt") def generate_experiment(self) -> str: - """Run ensemble and experiment file structure generation + """Generate the directories Generate the file structure for a SmartSim experiment. This includes the writing and configuring of input files for a - application. + job. To have files or directories present in the created entity directories, such as datasets or input files, call diff --git a/tests/test_experiment.py b/tests/test_experiment.py index ddbb6f4fc..f8477f319 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -95,13 +95,15 @@ class LaunchRecord: launch_args: launchArguments.LaunchArguments entity: entity.SmartSimEntity env: t.Mapping[str, str | None] + name: str @classmethod def from_job(cls, job): args = job._launch_settings.launch_args entity = job._entity env = job._launch_settings.env_vars - return cls(args, entity, env) + name = job._name + return cls(args, entity, env, name) class MockLaunchArgs(launchArguments.LaunchArguments): @@ -142,7 +144,7 @@ def test_start_raises_if_no_args_supplied(experiment): # fmt: off @pytest.mark.parametrize( - "num_jobs", [pytest.param(i, id=f"{i} job(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)] + "num_jobs", [pytest.param(i, id=f"{i} job(s)") for i in (1, 2)] ) @pytest.mark.parametrize( "make_jobs", ( @@ -170,7 +172,7 @@ def test_start_can_launch_jobs(experiment, job_maker, dispatcher, make_jobs, num @pytest.mark.parametrize( "num_starts", - [pytest.param(i, id=f"{i} start(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)], + [pytest.param(i, id=f"{i} start(s)") for i in (1, 2)], ) def test_start_can_start_a_job_multiple_times_accross_multiple_calls( experiment, job_maker, dispatcher, num_starts @@ -185,4 +187,7 @@ def test_start_can_start_a_job_multiple_times_accross_multiple_calls( (launcher,) = experiment._active_launchers assert isinstance(launcher, NoOpRecordLauncher), "Unexpected launcher type" assert len(launcher.launched_order) == num_starts, "Unexpected number launches" + print(f"here is the first: {ids_to_launches}") + print(f"here is the second: {launcher.ids_to_launched}") assert ids_to_launches == launcher.ids_to_launched, "Job was not re-launched" + diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index c5f8b8b86..dd64d13ac 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -13,7 +13,6 @@ from smartsim.settings.dispatch import Dispatcher from smartsim.settings.launchSettings import LaunchSettings - class NoOpLauncher: @classmethod def create(cls, _): @@ -47,7 +46,7 @@ def gen_instance_for_job(test_dir, wlmutils) -> Generator: @pytest.fixture def job_group_instance(test_dir, wlmutils) -> Generator: - """Fixture to create an instance of Generator.""" + """Fixture to create an instance of JobGroup.""" launch_settings = LaunchSettings(wlmutils.get_test_launcher()) application_1 = Application("app_name_1", exe="python", run_settings="RunSettings") application_2 = Application("app_name_2", exe="python", run_settings="RunSettings") @@ -56,8 +55,8 @@ def job_group_instance(test_dir, wlmutils) -> Generator: @pytest.fixture -def job_instance(test_dir, wlmutils) -> Generator: - """Fixture to create an instance of Generator.""" +def job_instance(wlmutils) -> Generator: + """Fixture to create an instance of Job.""" launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job = Job(EchoApp(), launch_settings) return job @@ -84,12 +83,14 @@ def test_log_file_path(gen_instance_for_job): def test_generate_job_directory(gen_instance_for_job): + """Test that Job directory was created.""" gen_instance_for_job.generate_experiment() assert osp.isdir(gen_instance_for_job.path) assert osp.isdir(gen_instance_for_job.log_path) def test_full_exp_generate_job_directory(test_dir, job_instance): + """Test that Job directory was created from Experiment.""" no_op_dispatch = Dispatcher() no_op_dispatch.dispatch(SlurmLaunchArguments, with_format=make_shell_format_fn("run_command"), to_launcher=NoOpLauncher) no_op_exp = Experiment( From 24e42ef6b52ed4e3361e89622bce2b9e566791ac Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 24 Jul 2024 12:01:54 -0500 Subject: [PATCH 33/82] experiment tests passing --- smartsim/settings/dispatch.py | 1 - tests/test_experiment.py | 23 +++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index e0c456d58..5f7be061b 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -187,7 +187,6 @@ def create_adapter_from_launcher( def format_(exe: ExecutableProtocol, env: _EnvironMappingType, path: str) -> _LaunchableT: return self.formatter(settings, exe, path, env) - return _LauncherAdapter(launcher, format_) def configure_first_compatible_launcher( diff --git a/tests/test_experiment.py b/tests/test_experiment.py index f8477f319..3c94fe380 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -45,14 +45,18 @@ @pytest.fixture -def experiment(test_dir): - yield Experiment(f"test-exp-{uuid.uuid4()}", test_dir) +def experiment(test_dir, monkeypatch): + exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) + # manually perfer to monkey path out where we want to generate to know and calc + # duplicate both tests + monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") + yield exp @pytest.fixture def dispatcher(): d = dispatch.Dispatcher() - to_record = lambda *a: LaunchRecord(*a) + to_record = lambda settings, exe, path, env: LaunchRecord(settings, exe, env, path) d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) yield d @@ -84,6 +88,8 @@ def create(cls, exp): return cls(exp) def start(self, record: LaunchRecord): + assert isinstance(record.path, str) + assert isinstance(record.env, dict) id_ = dispatch.create_job_id() self.launched_order.append(record) self.ids_to_launched[id_] = record @@ -95,15 +101,15 @@ class LaunchRecord: launch_args: launchArguments.LaunchArguments entity: entity.SmartSimEntity env: t.Mapping[str, str | None] - name: str + path: str @classmethod def from_job(cls, job): args = job._launch_settings.launch_args entity = job._entity env = job._launch_settings.env_vars - name = job._name - return cls(args, entity, env, name) + path = f"/tmp/{job._name}" + return cls(args, entity, env, path) class MockLaunchArgs(launchArguments.LaunchArguments): @@ -187,7 +193,4 @@ def test_start_can_start_a_job_multiple_times_accross_multiple_calls( (launcher,) = experiment._active_launchers assert isinstance(launcher, NoOpRecordLauncher), "Unexpected launcher type" assert len(launcher.launched_order) == num_starts, "Unexpected number launches" - print(f"here is the first: {ids_to_launches}") - print(f"here is the second: {launcher.ids_to_launched}") - assert ids_to_launches == launcher.ids_to_launched, "Job was not re-launched" - + assert ids_to_launches == launcher.ids_to_launched, "Job was not re-launched" \ No newline at end of file From fdf4b6333e02481d0952930cd9263467e1977939 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 24 Jul 2024 12:40:38 -0500 Subject: [PATCH 34/82] pushing one small change to pull --- tests/test_experiment.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 3c94fe380..2968bde70 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -47,8 +47,13 @@ @pytest.fixture def experiment(test_dir, monkeypatch): exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) - # manually perfer to monkey path out where we want to generate to know and calc - # duplicate both tests + monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") + yield exp + + +@pytest.fixture +def experiment_patch_path(test_dir, monkeypatch): + exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") yield exp From 1378114b34721735e4afcf210d1c6a632ba20acb Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 26 Jul 2024 10:46:21 -0500 Subject: [PATCH 35/82] coverage for gen tests, config, copy and symlink --- smartsim/_core/entrypoints/file_operations.py | 3 +- smartsim/_core/generation/generator.py | 165 ++++++++++------- smartsim/entity/model.py | 2 +- tests/test_generator/test_generator.py | 172 ++++++++++++++++-- 4 files changed, 262 insertions(+), 80 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index c57192ea8..189edd5e6 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -35,6 +35,7 @@ import shutil import typing as t from typing import Callable +from os import path as osp from ...log import get_logger @@ -133,7 +134,7 @@ def copy(parsed_args: argparse.Namespace) -> None: dirs_exist_ok=parsed_args.dirs_exist_ok, ) else: - shutil.copyfile(parsed_args.source, parsed_args.dest) + shutil.copy(parsed_args.source, parsed_args.dest) def symlink(parsed_args: argparse.Namespace) -> None: diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index dc932c4ba..7bf7394c8 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -28,7 +28,9 @@ import os import pathlib import shutil +import pickle import typing as t +from glob import glob from datetime import datetime from distutils import dir_util # pylint: disable=deprecated-module from logging import DEBUG, INFO @@ -36,6 +38,7 @@ from os.path import join, relpath from tabulate import tabulate +from pathlib import Path from ...database import FeatureStore from ...entity import Application, TaggedFilesHierarchy @@ -43,6 +46,9 @@ from ...log import get_logger from ..utils.helpers import create_short_id_str +from ..entrypoints import file_operations +from ..entrypoints.file_operations import get_parser + logger = get_logger(__name__) logger.propagate = False @@ -172,9 +178,11 @@ def generate_experiment(self) -> str: e.g. ``THERMO=;90;`` """ + # Create Job directory pathlib.Path(self.path).mkdir(exist_ok=True, parents=True) + # Creat Job log directory pathlib.Path(self.log_path).mkdir(exist_ok=True, parents=True) - + # The log_file only keeps track of the last generation # this is to avoid gigantic files in case the user repeats # generation several times. The information is anyhow @@ -182,19 +190,15 @@ def generate_experiment(self) -> str: with open(self.log_file, mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - - # TODO update this to execute the file operations when entrypoint is merged in - # if isinstance(Application, type(self.job.entity)): - # file_operation_list = self.build_operations() - # self.execute_file_operations(file_operation_list) + # Prevent access to type FeatureStore entities + if isinstance(self.job.entity, Application) and self.job.entity.files: + # Perform file system operations on attached files + self._build_operations() + # Return Job directory path return self.path - # TODO update this to execute the file operations when entrypoint is merged in - def execute_file_operations( - self, file_ops: t.Sequence[t.Sequence[str]] - ) -> None: ... - def build_operations(self) -> t.Sequence[t.Sequence[str]]: + def _build_operations(self) -> t.Sequence[t.Sequence[str]]: """This method generates file system operations based on the provided application. It processes three types of operations: to_copy, to_symlink, and to_configure. For each type, it calls the corresponding private methods and appends the results @@ -203,27 +207,69 @@ def build_operations(self) -> t.Sequence[t.Sequence[str]]: :param app: The application for which operations are generated. :return: A list of lists containing file system operations. """ - application_files = self.job.entity.files - file_operation_list: t.List[t.Sequence[str]] = [] - # Generate copy file system operations - file_operation_list.extend( - self._get_copy_file_system_operation(file_copy) - for file_copy in application_files.copy - ) - # Generate symlink file system operations - file_operation_list.extend( - self._get_symlink_file_system_operation(file_link) - for file_link in application_files.link - ) - # Generate configure file system operations - file_operation_list.extend( - self._write_tagged_entity_files(file_configure) - for file_configure in application_files.tagged - ) - return file_operation_list + if self.job.entity.files.link: + self._get_symlink_file_system_operation(self.job.entity, self.path) + if self.job.entity.files.tagged: + self._write_tagged_entity_files(self.job.entity) + if self.job.entity.files.copy: + self._get_copy_file_system_operation(self.job.entity, self.path) + + + @staticmethod + def _get_copy_file_system_operation(app: Application, dest: str) -> None: + """Get copy file system operation for a file. + + :param linked_file: The file to be copied. + :return: A list of copy file system operations. + """ + for src in app.files.copy: + parser = get_parser() + if Path(src).is_dir: + cmd = f"copy {src} {dest} --dirs_exist_ok" + else: + cmd = f"copy {src} {dest}" + args = cmd.split() + ns = parser.parse_args(args) + file_operations.copy(ns) + + + @staticmethod + def _get_symlink_file_system_operation(app: Application, dest: str) -> None: + """Get symlink file system operation for a file. + + :param linked_file: The file to be symlinked. + :return: A list of symlink file system operations. + """ + for sym in app.files.link: + # Check if path is a directory + if Path(sym).is_dir(): + # Normalize the path to remove trailing slashes + normalized_path = os.path.normpath(sym) + # Get the parent directory (last folder) + parent_dir = os.path.basename(normalized_path) + dest = Path(dest) / parent_dir + parser = get_parser() + cmd = f"symlink {sym} {dest}" + args = cmd.split() + ns = parser.parse_args(args) + file_operations.symlink(ns) + # Path is a file + else: + # Normalize the path to remove trailing slashes + normalized_path = os.path.normpath(sym) + # Get the parent directory (last folder) + parent_file = os.path.basename(normalized_path) + new_dest = os.path.join(dest,parent_file) + parser = get_parser() + cmd = f"symlink {sym} {new_dest}" + args = cmd.split() + ns = parser.parse_args(args) + file_operations.symlink(ns) + + # TODO update this to execute the file operations when entrypoint is merged in - def _write_tagged_entity_files(self, configure_file: str) -> t.Sequence[str]: + def _write_tagged_entity_files(self, app: Application) -> None: """Read, configure and write the tagged input files for a Application instance within an ensemble. This function specifically deals with the tagged files attached to @@ -231,7 +277,7 @@ def _write_tagged_entity_files(self, configure_file: str) -> t.Sequence[str]: :param entity: a Application instance """ - # if entity.files: + # if app.files: # to_write = [] # def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: @@ -242,48 +288,41 @@ def _write_tagged_entity_files(self, configure_file: str) -> t.Sequence[str]: # directory structure # """ # for file in tagged.files: - # dst_path = path.join(entity.path, tagged.base, path.basename(file)) + # dst_path = path.join(self.path, tagged.base, path.basename(file)) # shutil.copyfile(file, dst_path) # to_write.append(dst_path) # for tagged_dir in tagged.dirs: # mkdir( # path.join( - # entity.path, tagged.base, path.basename(tagged_dir.base) + # self.path, tagged.base, path.basename(tagged_dir.base) # ) # ) # _build_tagged_files(tagged_dir) - # if entity.files.tagged_hierarchy: - # _build_tagged_files(entity.files.tagged_hierarchy) - - # # write in changes to configurations - # if isinstance(entity, Application): - # files_to_params = self._writer.configure_tagged_application_files( - # to_write, entity.params - # ) - # self._log_params(entity, files_to_params) - return ["temporary", "config"] - - # TODO replace with entrypoint operation - @staticmethod - def _get_copy_file_system_operation(copy_file: str) -> t.Sequence[str]: - """Get copy file system operation for a file. - - :param linked_file: The file to be copied. - :return: A list of copy file system operations. - """ - return ["temporary", "copy"] - - # TODO replace with entrypoint operation - @staticmethod - def _get_symlink_file_system_operation(linked_file: str) -> t.Sequence[str]: - """Get symlink file system operation for a file. - - :param linked_file: The file to be symlinked. - :return: A list of symlink file system operations. - """ - return ["temporary", "link"] + # if app.files.tagged_hierarchy: + # _build_tagged_files(app.files.tagged_hierarchy) + + # Configure param file + if app.files.tagged: + # copy files to job directory + for file in app.files.tagged: + # Copy the contents of a source to a destination folder + shutil.copy(file, self.path) + # Pickle the dictionary + pickled_dict = pickle.dumps(app.params) + tag = ";" + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + tagged_files = sorted(glob(self.path + "/*")) + for tagged_file in tagged_files: + parser = get_parser() + cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + file_operations.configure(ns) + # TODO address in ticket 723 + # self._log_params(entity, files_to_params) # TODO to be refactored in ticket 723 # def _log_params( diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index 93e07577d..045634b3f 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -227,7 +227,7 @@ def attach_generator_files( "`smartsim_params.txt` is a file automatically " + "generated by SmartSim and cannot be ovewritten." ) - + # files is not a list of entity files self.files = EntityFiles(to_configure, to_copy, to_symlink) @property diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index dd64d13ac..2e1cdd335 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -2,8 +2,13 @@ from logging import DEBUG, INFO from os import environ from os import path as osp +from glob import glob +from os import listdir import pytest +import pathlib +import filecmp +import os from smartsim import Experiment from smartsim._core.generation.generator import Generator @@ -13,6 +18,10 @@ from smartsim.settings.dispatch import Dispatcher from smartsim.settings.launchSettings import LaunchSettings +def get_gen_file(fileutils, filename): + return fileutils.get_test_conf_path(osp.join("generator_files", filename)) + +# Mock Launcher class NoOpLauncher: @classmethod def create(cls, _): @@ -23,6 +32,7 @@ def start(self, _): def make_shell_format_fn(run_command: str | None): ... +# Mock Application class EchoApp: def as_program_arguments(self): return ["echo", "Hello", "World!"] @@ -39,23 +49,13 @@ def gen_instance_for_job(test_dir, wlmutils) -> Generator: + datetime.datetime.now().strftime("%Y-%m-%d") ) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - application_1 = Application("app_name", exe="python", run_settings="RunSettings") - job = Job(application_1, launch_settings) + app = Application("app_name", exe="python", run_settings="RunSettings") + job = Job(app, launch_settings) return Generator(gen_path=experiment_path, run_ID=run_ID, job=job) @pytest.fixture -def job_group_instance(test_dir, wlmutils) -> Generator: - """Fixture to create an instance of JobGroup.""" - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - application_1 = Application("app_name_1", exe="python", run_settings="RunSettings") - application_2 = Application("app_name_2", exe="python", run_settings="RunSettings") - job_group = JobGroup(application_1, application_2, launch_settings) - return job_group - - -@pytest.fixture -def job_instance(wlmutils) -> Generator: +def job_instance(wlmutils) -> Job: """Fixture to create an instance of Job.""" launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job = Job(EchoApp(), launch_settings) @@ -72,7 +72,6 @@ def test_debug_log_level(gen_instance_for_job): environ["SMARTSIM_LOG_LEVEL"] = "debug" assert gen_instance_for_job.log_level == DEBUG # Clean up: unset the environment variable - # TODO might need to set it to info here? environ.pop("SMARTSIM_LOG_LEVEL", None) @@ -89,7 +88,7 @@ def test_generate_job_directory(gen_instance_for_job): assert osp.isdir(gen_instance_for_job.log_path) -def test_full_exp_generate_job_directory(test_dir, job_instance): +def test_exp_private_generate_method(test_dir, job_instance): """Test that Job directory was created from Experiment.""" no_op_dispatch = Dispatcher() no_op_dispatch.dispatch(SlurmLaunchArguments, with_format=make_shell_format_fn("run_command"), to_launcher=NoOpLauncher) @@ -98,6 +97,23 @@ def test_full_exp_generate_job_directory(test_dir, job_instance): ) job_execution_path = no_op_exp._generate(job_instance) assert osp.isdir(job_execution_path) + assert osp.isdir(pathlib.Path(no_op_exp.exp_path) / "log") + + +def test_exp_private_generate_method_ensemble(test_dir,wlmutils): + """Test that Job directory was created from Experiment.""" + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + no_op_dispatch = Dispatcher() + no_op_dispatch.dispatch(launch_settings, with_format=make_shell_format_fn("run_command"), to_launcher=NoOpLauncher) + no_op_exp = Experiment( + name="No-Op-Exp", exp_path=test_dir + ) + for job in job_list: + job_execution_path = no_op_exp._generate(job) + assert osp.isdir(job_execution_path) + assert osp.isdir(pathlib.Path(no_op_exp.exp_path) / "log") def test_generate_ensemble_directory(test_dir, wlmutils): @@ -109,3 +125,129 @@ def test_generate_ensemble_directory(test_dir, wlmutils): gen = Generator(gen_path=test_dir, run_ID=run_ID, job=job) gen.generate_experiment() assert osp.isdir(gen.path) + assert osp.isdir(pathlib.Path(test_dir) / "log" ) + + +def test_generate_copy_file(fileutils,wlmutils,test_dir): + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name","python","RunSettings") + script = fileutils.get_test_conf_path("sleep.py") + app.attach_generator_files(to_copy=script) + job = Job(app,launch_settings) + + # Create the experiment + experiment_path = osp.join(test_dir, "experiment_name") + gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) + path = gen.generate_experiment() + expected_file = pathlib.Path(path) / "sleep.py" + assert osp.isfile(expected_file) + +# TODO FLAGGED +def test_generate_copy_directory(fileutils,wlmutils,test_dir): + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name","python","RunSettings") + copy_dir = get_gen_file(fileutils, "to_copy_dir") + print(copy_dir) + app.attach_generator_files(to_copy=copy_dir) + job = Job(app,launch_settings) + + # Create the experiment + experiment_path = osp.join(test_dir, "experiment_name") + gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) + gen.generate_experiment() + expected_file = pathlib.Path(gen.path) / "to_copy_dir" / "mock.txt" + +def test_generate_symlink_directory(fileutils, wlmutils,test_dir): + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name","python","RunSettings") + # Path of directory to symlink + symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + # Attach directory to Application + app.attach_generator_files(to_symlink=symlink_dir) + # Create Job + job = Job(app,launch_settings) + + # Create the experiment + experiment_path = osp.join(test_dir, "experiment_name") + gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) + # Generate Experiment file structure + gen.generate_experiment() + expected_folder = pathlib.Path(gen.path) / "to_symlink_dir" + assert osp.isdir(expected_folder) + # Combine symlinked file list and original file list for comparison + for written, correct in zip(listdir(symlink_dir), listdir(expected_folder)): + # For each pair, check if the filenames are equal + assert written == correct + +def test_generate_symlink_file(fileutils, wlmutils,test_dir): + assert osp.isfile(pathlib.Path("/lus/sonexion/richaama/Matt/SmartSim/tests/test_configs/generator_files/to_symlink_dir/mock2.txt")) + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name","python","RunSettings") + # Path of directory to symlink + symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + # Get a list of all files in the directory + symlink_files = sorted(glob(symlink_dir + "/*")) + # Attach directory to Application + app.attach_generator_files(to_symlink=symlink_files) + # Create Job + job = Job(app,launch_settings) + # Create the experiment + experiment_path = osp.join(test_dir, "experiment_name") + gen = Generator(gen_path=experiment_path, run_ID="test", job=job) + # Generate Experiment file structure + gen.generate_experiment() + expected_file = pathlib.Path(gen.path) / "mock2.txt" + assert osp.isfile(expected_file) + + +def test_generate_configure(fileutils, wlmutils,test_dir): + # Directory of files to configure + conf_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "marked/") + ) + # Retrieve a list of files for configuration + tagged_files = sorted(glob(conf_path + "/*")) + # Retrieve directory of files to compare after Experiment.generate_experiment completion + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + # Retrieve list of files in correctly tagged directory for comparison + correct_files = sorted(glob(correct_path + "/*")) + # Initialize a Job + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + param_dict = { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + "VALID": "valid", + } + app = Application("name_1","python","RunSettings", params=param_dict) + app.attach_generator_files(to_configure=tagged_files) + job = Job(app,launch_settings) + + # Spin up Experiment + experiment_path = osp.join(test_dir, "experiment_name") + # Spin up Generator + gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) + # Execute file generation + job_path = gen.generate_experiment() + # Retrieve the list of configured files in the test directory + configured_files = sorted(glob(job_path + "/*")) + # Use filecmp.cmp to check that the corresponding files are equal + for written, correct in zip(configured_files, correct_files): + assert filecmp.cmp(written, correct) + # Validate that log file exists + assert osp.isdir(pathlib.Path(experiment_path) / "log") + # Validate that smartsim params files exists + smartsim_params_path = osp.join(job_path, "smartsim_params.txt") + assert osp.isfile(smartsim_params_path) + + + From e4036aa150192f318c12f53423bb8752a394956e Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 26 Jul 2024 13:38:10 -0500 Subject: [PATCH 36/82] Tagged Files Heirarchy added --- smartsim/_core/generation/generator.py | 112 ++++++++++--------------- tests/test_generator/test_generator.py | 5 +- 2 files changed, 47 insertions(+), 70 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 7bf7394c8..917db928c 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -190,10 +190,12 @@ def generate_experiment(self) -> str: with open(self.log_file, mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") + # Prevent access to type FeatureStore entities if isinstance(self.job.entity, Application) and self.job.entity.files: # Perform file system operations on attached files self._build_operations() + # Return Job directory path return self.path @@ -207,12 +209,9 @@ def _build_operations(self) -> t.Sequence[t.Sequence[str]]: :param app: The application for which operations are generated. :return: A list of lists containing file system operations. """ - if self.job.entity.files.link: - self._get_symlink_file_system_operation(self.job.entity, self.path) - if self.job.entity.files.tagged: - self._write_tagged_entity_files(self.job.entity) - if self.job.entity.files.copy: - self._get_copy_file_system_operation(self.job.entity, self.path) + self._get_symlink_file_system_operation(self.job.entity, self.path) + self._write_tagged_entity_files(self.job.entity) + self._get_copy_file_system_operation(self.job.entity, self.path) @staticmethod @@ -222,8 +221,8 @@ def _get_copy_file_system_operation(app: Application, dest: str) -> None: :param linked_file: The file to be copied. :return: A list of copy file system operations. """ + parser = get_parser() for src in app.files.copy: - parser = get_parser() if Path(src).is_dir: cmd = f"copy {src} {dest} --dirs_exist_ok" else: @@ -240,31 +239,17 @@ def _get_symlink_file_system_operation(app: Application, dest: str) -> None: :param linked_file: The file to be symlinked. :return: A list of symlink file system operations. """ + parser = get_parser() for sym in app.files.link: - # Check if path is a directory - if Path(sym).is_dir(): - # Normalize the path to remove trailing slashes - normalized_path = os.path.normpath(sym) - # Get the parent directory (last folder) - parent_dir = os.path.basename(normalized_path) - dest = Path(dest) / parent_dir - parser = get_parser() - cmd = f"symlink {sym} {dest}" - args = cmd.split() - ns = parser.parse_args(args) - file_operations.symlink(ns) - # Path is a file - else: - # Normalize the path to remove trailing slashes - normalized_path = os.path.normpath(sym) - # Get the parent directory (last folder) - parent_file = os.path.basename(normalized_path) - new_dest = os.path.join(dest,parent_file) - parser = get_parser() - cmd = f"symlink {sym} {new_dest}" - args = cmd.split() - ns = parser.parse_args(args) - file_operations.symlink(ns) + # Normalize the path to remove trailing slashes + normalized_path = os.path.normpath(sym) + # Get the parent directory (last folder) + parent_dir = os.path.basename(normalized_path) + dest = Path(dest) / parent_dir + cmd = f"symlink {sym} {dest}" + args = cmd.split() + ns = parser.parse_args(args) + file_operations.symlink(ns) @@ -277,50 +262,45 @@ def _write_tagged_entity_files(self, app: Application) -> None: :param entity: a Application instance """ - # if app.files: - # to_write = [] - - # def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: - # """Using a TaggedFileHierarchy, reproduce the tagged file - # directory structure - - # :param tagged: a TaggedFileHierarchy to be built as a - # directory structure - # """ - # for file in tagged.files: - # dst_path = path.join(self.path, tagged.base, path.basename(file)) - # shutil.copyfile(file, dst_path) - # to_write.append(dst_path) - - # for tagged_dir in tagged.dirs: - # mkdir( - # path.join( - # self.path, tagged.base, path.basename(tagged_dir.base) - # ) - # ) - # _build_tagged_files(tagged_dir) - - # if app.files.tagged_hierarchy: - # _build_tagged_files(app.files.tagged_hierarchy) - - # Configure param file if app.files.tagged: - # copy files to job directory - for file in app.files.tagged: - # Copy the contents of a source to a destination folder - shutil.copy(file, self.path) + to_write = [] + + def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: + """Using a TaggedFileHierarchy, reproduce the tagged file + directory structure + + :param tagged: a TaggedFileHierarchy to be built as a + directory structure + """ + for file in tagged.files: + dst_path = path.join(self.path, tagged.base, path.basename(file)) + print(dst_path) + shutil.copyfile(file, dst_path) + to_write.append(dst_path) + + for tagged_dir in tagged.dirs: + mkdir( + path.join( + self.path, tagged.base, path.basename(tagged_dir.base) + ) + ) + _build_tagged_files(tagged_dir) + if app.files.tagged_hierarchy: + _build_tagged_files(app.files.tagged_hierarchy) + # Pickle the dictionary pickled_dict = pickle.dumps(app.params) + # Default tag delimiter tag = ";" # Encode the pickled dictionary with Base64 encoded_dict = base64.b64encode(pickled_dict).decode("ascii") - tagged_files = sorted(glob(self.path + "/*")) - for tagged_file in tagged_files: - parser = get_parser() - cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" + parser = get_parser() + for dest_path in to_write: + cmd = f"configure {dest_path} {dest_path} {tag} {encoded_dict}" args = cmd.split() ns = parser.parse_args(args) file_operations.configure(ns) + # TODO address in ticket 723 # self._log_params(entity, files_to_params) diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index 2e1cdd335..6255d9ce6 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -247,7 +247,4 @@ def test_generate_configure(fileutils, wlmutils,test_dir): assert osp.isdir(pathlib.Path(experiment_path) / "log") # Validate that smartsim params files exists smartsim_params_path = osp.join(job_path, "smartsim_params.txt") - assert osp.isfile(smartsim_params_path) - - - + assert osp.isfile(smartsim_params_path) \ No newline at end of file From 717e951b84b5715dd835569bc3dbf3fa2befbf2a Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 29 Jul 2024 10:54:25 -0500 Subject: [PATCH 37/82] make style formatting and additional changes I forget --- smartsim/_core/entrypoints/file_operations.py | 2 +- smartsim/_core/generation/generator.py | 42 +- smartsim/entity/dbnode.py | 2 +- smartsim/settings/dispatch.py | 19 +- tests/_legacy/test_generator.py | 360 +++++++++--------- .../test_settings/test_alpsLauncher.py | 4 +- .../test_settings/test_localLauncher.py | 4 +- .../test_settings/test_lsfLauncher.py | 4 +- .../test_settings/test_slurmLauncher.py | 4 +- tests/test_experiment.py | 2 +- tests/test_generator/test_generator.py | 187 ++++----- 11 files changed, 327 insertions(+), 303 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 189edd5e6..4271c2a63 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -34,8 +34,8 @@ import pickle import shutil import typing as t -from typing import Callable from os import path as osp +from typing import Callable from ...log import get_logger diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 917db928c..a81396004 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -27,27 +27,26 @@ import base64 import os import pathlib -import shutil import pickle +import shutil import typing as t -from glob import glob from datetime import datetime from distutils import dir_util # pylint: disable=deprecated-module +from glob import glob from logging import DEBUG, INFO from os import mkdir, path, symlink from os.path import join, relpath +from pathlib import Path from tabulate import tabulate -from pathlib import Path from ...database import FeatureStore from ...entity import Application, TaggedFilesHierarchy from ...launchable import Job, JobGroup from ...log import get_logger -from ..utils.helpers import create_short_id_str - from ..entrypoints import file_operations from ..entrypoints.file_operations import get_parser +from ..utils.helpers import create_short_id_str logger = get_logger(__name__) logger.propagate = False @@ -182,7 +181,7 @@ def generate_experiment(self) -> str: pathlib.Path(self.path).mkdir(exist_ok=True, parents=True) # Creat Job log directory pathlib.Path(self.log_path).mkdir(exist_ok=True, parents=True) - + # The log_file only keeps track of the last generation # this is to avoid gigantic files in case the user repeats # generation several times. The information is anyhow @@ -199,8 +198,7 @@ def generate_experiment(self) -> str: # Return Job directory path return self.path - - def _build_operations(self) -> t.Sequence[t.Sequence[str]]: + def _build_operations(self) -> None: """This method generates file system operations based on the provided application. It processes three types of operations: to_copy, to_symlink, and to_configure. For each type, it calls the corresponding private methods and appends the results @@ -209,10 +207,11 @@ def _build_operations(self) -> t.Sequence[t.Sequence[str]]: :param app: The application for which operations are generated. :return: A list of lists containing file system operations. """ - self._get_symlink_file_system_operation(self.job.entity, self.path) - self._write_tagged_entity_files(self.job.entity) - self._get_copy_file_system_operation(self.job.entity, self.path) - + app = t.cast(Application, self.job.entity) + self._get_symlink_file_system_operation(app, self.path) + self._write_tagged_entity_files(app, self.path) + if app.files: + self._get_copy_file_system_operation(app, self.path) @staticmethod def _get_copy_file_system_operation(app: Application, dest: str) -> None: @@ -231,7 +230,6 @@ def _get_copy_file_system_operation(app: Application, dest: str) -> None: ns = parser.parse_args(args) file_operations.copy(ns) - @staticmethod def _get_symlink_file_system_operation(app: Application, dest: str) -> None: """Get symlink file system operation for a file. @@ -245,16 +243,15 @@ def _get_symlink_file_system_operation(app: Application, dest: str) -> None: normalized_path = os.path.normpath(sym) # Get the parent directory (last folder) parent_dir = os.path.basename(normalized_path) - dest = Path(dest) / parent_dir + dest = os.path.join(dest, parent_dir) cmd = f"symlink {sym} {dest}" args = cmd.split() ns = parser.parse_args(args) file_operations.symlink(ns) - - # TODO update this to execute the file operations when entrypoint is merged in - def _write_tagged_entity_files(self, app: Application) -> None: + @staticmethod + def _write_tagged_entity_files(app: Application, dest: str) -> None: """Read, configure and write the tagged input files for a Application instance within an ensemble. This function specifically deals with the tagged files attached to @@ -273,21 +270,18 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: directory structure """ for file in tagged.files: - dst_path = path.join(self.path, tagged.base, path.basename(file)) + dst_path = path.join(dest, tagged.base, path.basename(file)) print(dst_path) shutil.copyfile(file, dst_path) to_write.append(dst_path) for tagged_dir in tagged.dirs: - mkdir( - path.join( - self.path, tagged.base, path.basename(tagged_dir.base) - ) - ) + mkdir(path.join(dest, tagged.base, path.basename(tagged_dir.base))) _build_tagged_files(tagged_dir) + if app.files.tagged_hierarchy: _build_tagged_files(app.files.tagged_hierarchy) - + # Pickle the dictionary pickled_dict = pickle.dumps(app.params) # Default tag delimiter diff --git a/smartsim/entity/dbnode.py b/smartsim/entity/dbnode.py index 16fd9863f..54ec68e1a 100644 --- a/smartsim/entity/dbnode.py +++ b/smartsim/entity/dbnode.py @@ -64,7 +64,7 @@ def __init__( fs_identifier: str = "", ) -> None: """Initialize a feature store node within an feature store.""" - super().__init__(name, path, run_settings) + super().__init__(name, run_settings) self.exe = [exe] if run_settings.container else [expand_exe_path(exe)] self.exe_args = exe_args or [] self.ports = ports diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 5f7be061b..31f8e25da 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -48,7 +48,7 @@ _EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", str, _EnvironMappingType], _LaunchableT + [_DispatchableT, "ExecutableProtocol", str, _EnvironMappingType], _LaunchableT ] _LaunchConfigType: TypeAlias = ( "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType, str]" @@ -185,8 +185,11 @@ def create_adapter_from_launcher( f"exactly `{self.launcher_type}`" ) - def format_(exe: ExecutableProtocol, env: _EnvironMappingType, path: str) -> _LaunchableT: + def format_( + exe: ExecutableProtocol, env: _EnvironMappingType, path: str + ) -> _LaunchableT: return self.formatter(settings, exe, path, env) + return _LauncherAdapter(launcher, format_) def configure_first_compatible_launcher( @@ -250,9 +253,12 @@ def create(cls, exp: Experiment, /) -> Self: ... def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[LaunchArguments, tuple[t.Sequence[str],str]]: +) -> _FormatterType[LaunchArguments, tuple[t.Sequence[str], str]]: def impl( - args: LaunchArguments, exe: ExecutableProtocol, path: str, _env: _EnvironMappingType + args: LaunchArguments, + exe: ExecutableProtocol, + path: str, + _env: _EnvironMappingType, ) -> t.Tuple[t.Sequence[str], str]: return ( ( @@ -273,14 +279,13 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} + def start(self, payload: tuple[t.Sequence[str], str]) -> LaunchedJobID: command, path = payload id_ = create_job_id() exe, *rest = command # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen( - (helpers.expand_exe_path(exe), *rest), cwd=path - ) + self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path) return id_ @classmethod diff --git a/tests/_legacy/test_generator.py b/tests/_legacy/test_generator.py index c3bfcad64..0d65900cd 100644 --- a/tests/_legacy/test_generator.py +++ b/tests/_legacy/test_generator.py @@ -74,46 +74,46 @@ def test_ensemble(fileutils, test_dir): assert osp.isdir(osp.join(test_dir, "test/test_" + str(i))) -def test_ensemble_overwrite(fileutils, test_dir): - exp = Experiment("gen-test-overwrite", launcher="local") +# def test_ensemble_overwrite(fileutils, test_dir): +# exp = Experiment("gen-test-overwrite", launcher="local") - gen = Generator(test_dir, overwrite=True) +# gen = Generator(test_dir, overwrite=True) - params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} - ensemble = exp.create_ensemble("test", params=params, run_settings=rs) +# params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} +# ensemble = exp.create_ensemble("test", params=params, run_settings=rs) - config = get_gen_file(fileutils, "in.atm") - ensemble.attach_generator_files(to_configure=[config]) - gen.generate_experiment(ensemble) +# config = get_gen_file(fileutils, "in.atm") +# ensemble.attach_generator_files(to_configure=[config]) +# gen.generate_experiment(ensemble) - # re generate without overwrite - config = get_gen_file(fileutils, "in.atm") - ensemble.attach_generator_files(to_configure=[config]) - gen.generate_experiment(ensemble) +# # re generate without overwrite +# config = get_gen_file(fileutils, "in.atm") +# ensemble.attach_generator_files(to_configure=[config]) +# gen.generate_experiment(ensemble) - assert len(ensemble) == 9 - assert osp.isdir(osp.join(test_dir, "test")) - for i in range(9): - assert osp.isdir(osp.join(test_dir, "test/test_" + str(i))) +# assert len(ensemble) == 9 +# assert osp.isdir(osp.join(test_dir, "test")) +# for i in range(9): +# assert osp.isdir(osp.join(test_dir, "test/test_" + str(i))) -def test_ensemble_overwrite_error(fileutils, test_dir): - exp = Experiment("gen-test-overwrite-error", launcher="local") +# def test_ensemble_overwrite_error(fileutils, test_dir): +# exp = Experiment("gen-test-overwrite-error", launcher="local") - gen = Generator(test_dir) +# gen = Generator(test_dir) - params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} - ensemble = exp.create_ensemble("test", params=params, run_settings=rs) +# params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} +# ensemble = exp.create_ensemble("test", params=params, run_settings=rs) - config = get_gen_file(fileutils, "in.atm") - ensemble.attach_generator_files(to_configure=[config]) - gen.generate_experiment(ensemble) +# config = get_gen_file(fileutils, "in.atm") +# ensemble.attach_generator_files(to_configure=[config]) +# gen.generate_experiment(ensemble) - # re generate without overwrite - config = get_gen_file(fileutils, "in.atm") - ensemble.attach_generator_files(to_configure=[config]) - with pytest.raises(FileExistsError): - gen.generate_experiment(ensemble) +# # re generate without overwrite +# config = get_gen_file(fileutils, "in.atm") +# ensemble.attach_generator_files(to_configure=[config]) +# with pytest.raises(FileExistsError): +# gen.generate_experiment(ensemble) def test_full_exp(fileutils, test_dir, wlmutils): @@ -166,142 +166,142 @@ def test_dir_files(fileutils, test_dir): assert osp.isfile(osp.join(application_path, "test.in")) -def test_print_files(fileutils, test_dir, capsys): - """Test the stdout print of files attached to an ensemble""" - - exp = Experiment("print-attached-files-test", test_dir, launcher="local") - - ensemble = exp.create_ensemble("dir_test", replicas=1, run_settings=rs) - ensemble.entities = [] - - ensemble.print_attached_files() - captured = capsys.readouterr() - assert captured.out == "The ensemble is empty, no files to show.\n" - - params = {"THERMO": [10, 20], "STEPS": [20, 30]} - ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) - gen_dir = get_gen_file(fileutils, "test_dir") - symlink_dir = get_gen_file(fileutils, "to_symlink_dir") - copy_dir = get_gen_file(fileutils, "to_copy_dir") - - ensemble.print_attached_files() - captured = capsys.readouterr() - expected_out = ( - tabulate( - [ - [application.name, "No file attached to this application."] - for application in ensemble.applications - ], - headers=["Application name", "Files"], - tablefmt="grid", - ) - + "\n" - ) - - assert captured.out == expected_out - - ensemble.attach_generator_files() - ensemble.print_attached_files() - captured = capsys.readouterr() - expected_out = ( - tabulate( - [ - [application.name, "No file attached to this entity."] - for application in ensemble.applications - ], - headers=["Application name", "Files"], - tablefmt="grid", - ) - + "\n" - ) - assert captured.out == expected_out - - ensemble.attach_generator_files( - to_configure=[gen_dir, copy_dir], to_copy=copy_dir, to_symlink=symlink_dir - ) - - expected_out = tabulate( - [ - ["Copy", copy_dir], - ["Symlink", symlink_dir], - ["Configure", f"{gen_dir}\n{copy_dir}"], - ], - headers=["Strategy", "Files"], - tablefmt="grid", - ) - - assert all( - str(application.files) == expected_out for application in ensemble.applications - ) - - expected_out_multi = ( - tabulate( - [[application.name, expected_out] for application in ensemble.applications], - headers=["Application name", "Files"], - tablefmt="grid", - ) - + "\n" - ) - ensemble.print_attached_files() - - captured = capsys.readouterr() - assert captured.out == expected_out_multi - - -def test_multiple_tags(fileutils, test_dir): - """Test substitution of multiple tagged parameters on same line""" - - exp = Experiment("test-multiple-tags", test_dir) - application_params = {"port": 6379, "password": "unbreakable_password"} - application_settings = RunSettings("bash", "multi_tags_template.sh") - parameterized_application = exp.create_application( - "multi-tags", run_settings=application_settings, params=application_params - ) - config = get_gen_file(fileutils, "multi_tags_template.sh") - parameterized_application.attach_generator_files(to_configure=[config]) - exp.generate(parameterized_application, overwrite=True) - exp.start(parameterized_application, block=True) - - with open(osp.join(parameterized_application.path, "multi-tags.out")) as f: - log_content = f.read() - assert "My two parameters are 6379 and unbreakable_password, OK?" in log_content - - -def test_generation_log(fileutils, test_dir): - """Test that an error is issued when a tag is unused and make_fatal is True""" - - exp = Experiment("gen-log-test", test_dir, launcher="local") - - params = {"THERMO": [10, 20], "STEPS": [10, 20]} - ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) - conf_file = get_gen_file(fileutils, "in.atm") - ensemble.attach_generator_files(to_configure=conf_file) - - def not_header(line): - """you can add other general checks in here""" - return not line.startswith("Generation start date and time:") - - exp.generate(ensemble, verbose=True) - - log_file = osp.join(test_dir, "smartsim_params.txt") - ground_truth = get_gen_file( - fileutils, osp.join("log_params", "smartsim_params.txt") - ) - - with open(log_file) as f1, open(ground_truth) as f2: - assert not not_header(f1.readline()) - f1 = filter(not_header, f1) - f2 = filter(not_header, f2) - assert all(x == y for x, y in zip(f1, f2)) - - for entity in ensemble: - assert filecmp.cmp( - osp.join(entity.path, "smartsim_params.txt"), - get_gen_file( - fileutils, - osp.join("log_params", "dir_test", entity.name, "smartsim_params.txt"), - ), - ) +# def test_print_files(fileutils, test_dir, capsys): +# """Test the stdout print of files attached to an ensemble""" + +# exp = Experiment("print-attached-files-test", test_dir, launcher="local") + +# ensemble = exp.create_ensemble("dir_test", replicas=1, run_settings=rs) +# ensemble.entities = [] + +# ensemble.print_attached_files() +# captured = capsys.readouterr() +# assert captured.out == "The ensemble is empty, no files to show.\n" + +# params = {"THERMO": [10, 20], "STEPS": [20, 30]} +# ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) +# gen_dir = get_gen_file(fileutils, "test_dir") +# symlink_dir = get_gen_file(fileutils, "to_symlink_dir") +# copy_dir = get_gen_file(fileutils, "to_copy_dir") + +# ensemble.print_attached_files() +# captured = capsys.readouterr() +# expected_out = ( +# tabulate( +# [ +# [application.name, "No file attached to this application."] +# for application in ensemble.applications +# ], +# headers=["Application name", "Files"], +# tablefmt="grid", +# ) +# + "\n" +# ) + +# assert captured.out == expected_out + +# ensemble.attach_generator_files() +# ensemble.print_attached_files() +# captured = capsys.readouterr() +# expected_out = ( +# tabulate( +# [ +# [application.name, "No file attached to this entity."] +# for application in ensemble.applications +# ], +# headers=["Application name", "Files"], +# tablefmt="grid", +# ) +# + "\n" +# ) +# assert captured.out == expected_out + +# ensemble.attach_generator_files( +# to_configure=[gen_dir, copy_dir], to_copy=copy_dir, to_symlink=symlink_dir +# ) + +# expected_out = tabulate( +# [ +# ["Copy", copy_dir], +# ["Symlink", symlink_dir], +# ["Configure", f"{gen_dir}\n{copy_dir}"], +# ], +# headers=["Strategy", "Files"], +# tablefmt="grid", +# ) + +# assert all( +# str(application.files) == expected_out for application in ensemble.applications +# ) + +# expected_out_multi = ( +# tabulate( +# [[application.name, expected_out] for application in ensemble.applications], +# headers=["Application name", "Files"], +# tablefmt="grid", +# ) +# + "\n" +# ) +# ensemble.print_attached_files() + +# captured = capsys.readouterr() +# assert captured.out == expected_out_multi + + +# def test_multiple_tags(fileutils, test_dir): +# """Test substitution of multiple tagged parameters on same line""" + +# exp = Experiment("test-multiple-tags", test_dir) +# application_params = {"port": 6379, "password": "unbreakable_password"} +# application_settings = RunSettings("bash", "multi_tags_template.sh") +# parameterized_application = exp.create_application( +# "multi-tags", run_settings=application_settings, params=application_params +# ) +# config = get_gen_file(fileutils, "multi_tags_template.sh") +# parameterized_application.attach_generator_files(to_configure=[config]) +# exp.generate(parameterized_application, overwrite=True) +# exp.start(parameterized_application, block=True) + +# with open(osp.join(parameterized_application.path, "multi-tags.out")) as f: +# log_content = f.read() +# assert "My two parameters are 6379 and unbreakable_password, OK?" in log_content + + +# def test_generation_log(fileutils, test_dir): +# """Test that an error is issued when a tag is unused and make_fatal is True""" + +# exp = Experiment("gen-log-test", test_dir, launcher="local") + +# params = {"THERMO": [10, 20], "STEPS": [10, 20]} +# ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) +# conf_file = get_gen_file(fileutils, "in.atm") +# ensemble.attach_generator_files(to_configure=conf_file) + +# def not_header(line): +# """you can add other general checks in here""" +# return not line.startswith("Generation start date and time:") + +# exp.generate(ensemble, verbose=True) + +# log_file = osp.join(test_dir, "smartsim_params.txt") +# ground_truth = get_gen_file( +# fileutils, osp.join("log_params", "smartsim_params.txt") +# ) + +# with open(log_file) as f1, open(ground_truth) as f2: +# assert not not_header(f1.readline()) +# f1 = filter(not_header, f1) +# f2 = filter(not_header, f2) +# assert all(x == y for x, y in zip(f1, f2)) + +# for entity in ensemble: +# assert filecmp.cmp( +# osp.join(entity.path, "smartsim_params.txt"), +# get_gen_file( +# fileutils, +# osp.join("log_params", "dir_test", entity.name, "smartsim_params.txt"), +# ), +# ) def test_config_dir(fileutils, test_dir): @@ -364,18 +364,18 @@ def test_no_gen_if_symlink_to_dir(fileutils): ensemble.attach_generator_files(to_configure=config) -def test_no_file_overwrite(): - exp = Experiment("test_no_file_overwrite", launcher="local") - ensemble = exp.create_ensemble("test", params={"P": [0, 1]}, run_settings=rs) - with pytest.raises(ValueError): - ensemble.attach_generator_files( - to_configure=["/normal/file.txt", "/path/to/smartsim_params.txt"] - ) - with pytest.raises(ValueError): - ensemble.attach_generator_files( - to_symlink=["/normal/file.txt", "/path/to/smartsim_params.txt"] - ) - with pytest.raises(ValueError): - ensemble.attach_generator_files( - to_copy=["/normal/file.txt", "/path/to/smartsim_params.txt"] - ) +# def test_no_file_overwrite(): +# exp = Experiment("test_no_file_overwrite", launcher="local") +# ensemble = exp.create_ensemble("test", params={"P": [0, 1]}, run_settings=rs) +# with pytest.raises(ValueError): +# ensemble.attach_generator_files( +# to_configure=["/normal/file.txt", "/path/to/smartsim_params.txt"] +# ) +# with pytest.raises(ValueError): +# ensemble.attach_generator_files( +# to_symlink=["/normal/file.txt", "/path/to/smartsim_params.txt"] +# ) +# with pytest.raises(ValueError): +# ensemble.attach_generator_files( +# to_copy=["/normal/file.txt", "/path/to/smartsim_params.txt"] +# ) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 4746930e6..ee192374b 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -186,6 +186,8 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_aprun_command(AprunLaunchArguments(args), mock_echo_executable, test_dir, {}) + cmd, path = _as_aprun_command( + AprunLaunchArguments(args), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == expected assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 9d0ec13f0..9ee2bbb0f 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -118,6 +118,8 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): - cmd, path = _as_local_command(LocalLaunchArguments({}), mock_echo_executable, test_dir, {}) + cmd, path = _as_local_command( + LocalLaunchArguments({}), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == ("echo", "hello", "world") assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 73335a204..fe8bf4848 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -95,6 +95,8 @@ def test_launch_args(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_jsrun_command(JsrunLaunchArguments(args), mock_echo_executable, test_dir, {}) + cmd, path = _as_jsrun_command( + JsrunLaunchArguments(args), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == expected assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 6c62dccc9..251e214fa 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -292,6 +292,8 @@ def test_set_het_groups(monkeypatch): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_srun_command(SlurmLaunchArguments(args), mock_echo_executable, test_dir, {}) + cmd, path = _as_srun_command( + SlurmLaunchArguments(args), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == expected assert path == test_dir diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 2968bde70..99222687f 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -198,4 +198,4 @@ def test_start_can_start_a_job_multiple_times_accross_multiple_calls( (launcher,) = experiment._active_launchers assert isinstance(launcher, NoOpRecordLauncher), "Unexpected launcher type" assert len(launcher.launched_order) == num_starts, "Unexpected number launches" - assert ids_to_launches == launcher.ids_to_launched, "Job was not re-launched" \ No newline at end of file + assert ids_to_launches == launcher.ids_to_launched, "Job was not re-launched" diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index 6255d9ce6..35f30d1bf 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -1,14 +1,13 @@ import datetime +import filecmp +import os +import pathlib +from glob import glob from logging import DEBUG, INFO -from os import environ +from os import environ, listdir from os import path as osp -from glob import glob -from os import listdir import pytest -import pathlib -import filecmp -import os from smartsim import Experiment from smartsim._core.generation.generator import Generator @@ -18,9 +17,14 @@ from smartsim.settings.dispatch import Dispatcher from smartsim.settings.launchSettings import LaunchSettings +# TODO test ensemble copy, config, symlink when ensemble.attach_generator_files added +# TODO remove ensemble tests and replace with JobGroup when start jobgroup is supported + + def get_gen_file(fileutils, filename): return fileutils.get_test_conf_path(osp.join("generator_files", filename)) + # Mock Launcher class NoOpLauncher: @classmethod @@ -30,8 +34,11 @@ def create(cls, _): def start(self, _): return "anything" + +# Mock Shell Format fn def make_shell_format_fn(run_command: str | None): ... + # Mock Application class EchoApp: def as_program_arguments(self): @@ -39,19 +46,13 @@ def as_program_arguments(self): @pytest.fixture -def gen_instance_for_job(test_dir, wlmutils) -> Generator: +def generator_instance(test_dir, wlmutils) -> Generator: """Fixture to create an instance of Generator.""" experiment_path = osp.join(test_dir, "experiment_name") - run_ID = ( - "run-" - + datetime.datetime.now().strftime("%H:%M:%S") - + "-" - + datetime.datetime.now().strftime("%Y-%m-%d") - ) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("app_name", exe="python", run_settings="RunSettings") job = Job(app, launch_settings) - return Generator(gen_path=experiment_path, run_ID=run_ID, job=job) + return Generator(gen_path=experiment_path, run_ID="mock_run", job=job) @pytest.fixture @@ -62,80 +63,56 @@ def job_instance(wlmutils) -> Job: return job -def test_default_log_level(gen_instance_for_job): +def test_default_log_level(generator_instance): """Test if the default log level is INFO.""" - assert gen_instance_for_job.log_level == INFO + assert generator_instance.log_level == INFO -def test_debug_log_level(gen_instance_for_job): +def test_debug_log_level(generator_instance): """Test if the log level is DEBUG when environment variable is set to "debug".""" environ["SMARTSIM_LOG_LEVEL"] = "debug" - assert gen_instance_for_job.log_level == DEBUG + assert generator_instance.log_level == DEBUG # Clean up: unset the environment variable environ.pop("SMARTSIM_LOG_LEVEL", None) -def test_log_file_path(gen_instance_for_job): +def test_log_file_path(generator_instance): """Test if the log_file property returns the correct path.""" - expected_path = osp.join(gen_instance_for_job.path, "smartsim_params.txt") - assert gen_instance_for_job.log_file == expected_path + expected_path = osp.join(generator_instance.path, "smartsim_params.txt") + assert generator_instance.log_file == expected_path -def test_generate_job_directory(gen_instance_for_job): +def test_generate_job_directory(generator_instance): """Test that Job directory was created.""" - gen_instance_for_job.generate_experiment() - assert osp.isdir(gen_instance_for_job.path) - assert osp.isdir(gen_instance_for_job.log_path) + generator_instance.generate_experiment() + assert osp.isdir(generator_instance.path) + assert osp.isdir(generator_instance.log_path) + assert osp.isfile(osp.join(generator_instance.path, "smartsim_params.txt")) -def test_exp_private_generate_method(test_dir, job_instance): +def test_exp_private_generate_method_app(test_dir, job_instance): """Test that Job directory was created from Experiment.""" no_op_dispatch = Dispatcher() - no_op_dispatch.dispatch(SlurmLaunchArguments, with_format=make_shell_format_fn("run_command"), to_launcher=NoOpLauncher) - no_op_exp = Experiment( - name="No-Op-Exp", exp_path=test_dir + no_op_dispatch.dispatch( + SlurmLaunchArguments, + with_format=make_shell_format_fn("run_command"), + to_launcher=NoOpLauncher, ) + no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) job_execution_path = no_op_exp._generate(job_instance) assert osp.isdir(job_execution_path) assert osp.isdir(pathlib.Path(no_op_exp.exp_path) / "log") + assert osp.isfile(osp.join(job_execution_path, "smartsim_params.txt")) -def test_exp_private_generate_method_ensemble(test_dir,wlmutils): - """Test that Job directory was created from Experiment.""" - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.as_jobs(launch_settings) - no_op_dispatch = Dispatcher() - no_op_dispatch.dispatch(launch_settings, with_format=make_shell_format_fn("run_command"), to_launcher=NoOpLauncher) - no_op_exp = Experiment( - name="No-Op-Exp", exp_path=test_dir - ) - for job in job_list: - job_execution_path = no_op_exp._generate(job) - assert osp.isdir(job_execution_path) - assert osp.isdir(pathlib.Path(no_op_exp.exp_path) / "log") - - -def test_generate_ensemble_directory(test_dir, wlmutils): - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.as_jobs(launch_settings) - for job in job_list: - run_ID = "temp_run" - gen = Generator(gen_path=test_dir, run_ID=run_ID, job=job) - gen.generate_experiment() - assert osp.isdir(gen.path) - assert osp.isdir(pathlib.Path(test_dir) / "log" ) - - -def test_generate_copy_file(fileutils,wlmutils,test_dir): +def test_generate_copy_file(fileutils, wlmutils, test_dir): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name","python","RunSettings") + app = Application("name", "python", "RunSettings") script = fileutils.get_test_conf_path("sleep.py") app.attach_generator_files(to_copy=script) - job = Job(app,launch_settings) - + job = Job(app, launch_settings) + # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) @@ -143,15 +120,16 @@ def test_generate_copy_file(fileutils,wlmutils,test_dir): expected_file = pathlib.Path(path) / "sleep.py" assert osp.isfile(expected_file) + # TODO FLAGGED -def test_generate_copy_directory(fileutils,wlmutils,test_dir): +def test_generate_copy_directory(fileutils, wlmutils, test_dir): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name","python","RunSettings") + app = Application("name", "python", "RunSettings") copy_dir = get_gen_file(fileutils, "to_copy_dir") print(copy_dir) app.attach_generator_files(to_copy=copy_dir) - job = Job(app,launch_settings) + job = Job(app, launch_settings) # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") @@ -159,17 +137,18 @@ def test_generate_copy_directory(fileutils,wlmutils,test_dir): gen.generate_experiment() expected_file = pathlib.Path(gen.path) / "to_copy_dir" / "mock.txt" -def test_generate_symlink_directory(fileutils, wlmutils,test_dir): + +def test_generate_symlink_directory(fileutils, wlmutils, test_dir): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name","python","RunSettings") + app = Application("name", "python", "RunSettings") # Path of directory to symlink symlink_dir = get_gen_file(fileutils, "to_symlink_dir") # Attach directory to Application app.attach_generator_files(to_symlink=symlink_dir) # Create Job - job = Job(app,launch_settings) - + job = Job(app, launch_settings) + # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) @@ -182,11 +161,16 @@ def test_generate_symlink_directory(fileutils, wlmutils,test_dir): # For each pair, check if the filenames are equal assert written == correct -def test_generate_symlink_file(fileutils, wlmutils,test_dir): - assert osp.isfile(pathlib.Path("/lus/sonexion/richaama/Matt/SmartSim/tests/test_configs/generator_files/to_symlink_dir/mock2.txt")) + +def test_generate_symlink_file(fileutils, wlmutils, test_dir): + assert osp.isfile( + pathlib.Path( + "/lus/sonexion/richaama/Matt/SmartSim/tests/test_configs/generator_files/to_symlink_dir/mock2.txt" + ) + ) # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name","python","RunSettings") + app = Application("name", "python", "RunSettings") # Path of directory to symlink symlink_dir = get_gen_file(fileutils, "to_symlink_dir") # Get a list of all files in the directory @@ -194,7 +178,7 @@ def test_generate_symlink_file(fileutils, wlmutils,test_dir): # Attach directory to Application app.attach_generator_files(to_symlink=symlink_files) # Create Job - job = Job(app,launch_settings) + job = Job(app, launch_settings) # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") gen = Generator(gen_path=experiment_path, run_ID="test", job=job) @@ -204,7 +188,7 @@ def test_generate_symlink_file(fileutils, wlmutils,test_dir): assert osp.isfile(expected_file) -def test_generate_configure(fileutils, wlmutils,test_dir): +def test_generate_configure(fileutils, wlmutils, test_dir): # Directory of files to configure conf_path = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "marked/") @@ -220,18 +204,18 @@ def test_generate_configure(fileutils, wlmutils,test_dir): # Initialize a Job launch_settings = LaunchSettings(wlmutils.get_test_launcher()) param_dict = { - "5": 10, - "FIRST": "SECOND", - "17": 20, - "65": "70", - "placeholder": "group leftupper region", - "1200": "120", - "VALID": "valid", - } - app = Application("name_1","python","RunSettings", params=param_dict) + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + "VALID": "valid", + } + app = Application("name_1", "python", "RunSettings", params=param_dict) app.attach_generator_files(to_configure=tagged_files) - job = Job(app,launch_settings) - + job = Job(app, launch_settings) + # Spin up Experiment experiment_path = osp.join(test_dir, "experiment_name") # Spin up Generator @@ -247,4 +231,37 @@ def test_generate_configure(fileutils, wlmutils,test_dir): assert osp.isdir(pathlib.Path(experiment_path) / "log") # Validate that smartsim params files exists smartsim_params_path = osp.join(job_path, "smartsim_params.txt") - assert osp.isfile(smartsim_params_path) \ No newline at end of file + assert osp.isfile(smartsim_params_path) + + +# Ensemble Tests + + +def test_exp_private_generate_method_ensemble(test_dir, wlmutils): + """Test that Job directory was created from Experiment.""" + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + no_op_dispatch = Dispatcher() + no_op_dispatch.dispatch( + launch_settings, + with_format=make_shell_format_fn("run_command"), + to_launcher=NoOpLauncher, + ) + no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) + for job in job_list: + job_execution_path = no_op_exp._generate(job) + assert osp.isdir(job_execution_path) + assert osp.isdir(pathlib.Path(no_op_exp.exp_path) / "log") + + +def test_generate_ensemble_directory(test_dir, wlmutils): + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + for job in job_list: + run_ID = "temp_run" + gen = Generator(gen_path=test_dir, run_ID=run_ID, job=job) + gen.generate_experiment() + assert osp.isdir(gen.path) + assert osp.isdir(pathlib.Path(test_dir) / "log") From d7788626b29f780e953253fdd5f9fb209db07c0a Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 29 Jul 2024 14:20:23 -0500 Subject: [PATCH 38/82] path injection merge conflicts addressed --- .../_core/launcher/dragon/dragonLauncher.py | 3 +- smartsim/experiment.py | 38 ++++++----- .../settings/arguments/launch/__init__.py | 4 +- smartsim/settings/arguments/launch/dragon.py | 49 ++++++++++++-- smartsim/settings/arguments/launch/mpi.py | 64 ++++++++++--------- .../settings/arguments/launchArguments.py | 55 ++++++++++++---- smartsim/settings/dispatch.py | 5 +- smartsim/settings/launchSettings.py | 2 +- .../test_settings/test_dragonLauncher.py | 9 +-- .../test_settings/test_mpiLauncher.py | 4 +- 10 files changed, 159 insertions(+), 74 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 40d8c0f04..908c84807 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -355,6 +355,7 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: def _as_run_request_args_and_policy( run_req_args: DragonLaunchArguments, exe: ExecutableProtocol, + path: str, env: t.Mapping[str, str | None], ) -> tuple[DragonRunRequestView, DragonRunPolicy]: # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -374,7 +375,7 @@ def _as_run_request_args_and_policy( # the command execute next to any generated files. A similar # problem exists for the other settings. # TODO: Find a way to inject this path - path=os.getcwd(), + path=path, env=env, # TODO: Not sure how this info is injected name=None, diff --git a/smartsim/experiment.py b/smartsim/experiment.py index bc440d125..c29b547a6 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -161,6 +161,14 @@ def __init__(self, name: str, exp_path: str | None = None): self.exp_path = exp_path """The path under which the experiment operate""" + + self._run_ID = ( + "run-" + + datetime.datetime.now().strftime("%H:%M:%S") + + "-" + + datetime.datetime.now().strftime("%Y-%m-%d") + ) + """Create the run id for the Experiment""" self._active_launchers: set[LauncherProtocol[t.Any]] = set() """The active launchers created, used, and reused by the experiment""" @@ -226,7 +234,9 @@ def execute_dispatch(job: Job) -> LaunchedJobID: # it easier to monitor job statuses # pylint: disable-next=protected-access self._active_launchers.add(launch_config._adapted_launcher) - return launch_config.start(exe, env) + # Generate the Job directory and return generated path + job_execution_path = self._generate(job) + return launch_config.start(exe, env, job_execution_path) return execute_dispatch(job), *map(execute_dispatch, jobs) @@ -235,28 +245,26 @@ def _generate( self, job: Job, ) -> str: - """Generate the file structure for an ``Experiment`` + """Generate the file structure for a ``Job`` - ``Experiment.generate`` creates directories for each entity - passed to organize Experiments that launch many entities. + ``Experiment._generate`` creates directories for the job + passed. - If files or directories are attached to ``application`` objects + If files or directories are attached an ``application`` object using ``application.attach_generator_files()``, those files or directories will be symlinked, copied, or configured and - written into the created directory for that instance. + written into the created directory for that Job instance. - Instances of ``application``, ``Ensemble`` and ``FeatureStore`` - can all be passed as arguments to the generate method. + An instance of ``Job`` can be passed as an argument to + the protected generate member. - :param tag: tag used in `to_configure` generator files - :param overwrite: overwrite existing folders and contents - :param verbose: log parameter settings to std out + :param job: Job to generate file structure for + :returns: a str path """ try: - generator = Generator(self.exp_path, overwrite=overwrite, verbose=verbose) - if tag: - generator.set_tag(tag) - generator.generate_experiment(*args) + generator = Generator(self.exp_path, self._run_ID, job) + job_path = generator.generate_experiment() + return job_path except SmartSimError as e: logger.error(e) raise diff --git a/smartsim/settings/arguments/launch/__init__.py b/smartsim/settings/arguments/launch/__init__.py index 30502394b..629d45f67 100644 --- a/smartsim/settings/arguments/launch/__init__.py +++ b/smartsim/settings/arguments/launch/__init__.py @@ -11,9 +11,9 @@ "DragonLaunchArguments", "LocalLaunchArguments", "JsrunLaunchArguments", - "MpiLaunchArguments", + "MpirunLaunchArguments", "MpiexecLaunchArguments", - "OrteLaunchArguments", + "OrterunLaunchArguments", "PalsMpiexecLaunchArguments", "SlurmLaunchArguments", ] diff --git a/smartsim/settings/arguments/launch/dragon.py b/smartsim/settings/arguments/launch/dragon.py index 1ca0a244d..5dcf8fe35 100644 --- a/smartsim/settings/arguments/launch/dragon.py +++ b/smartsim/settings/arguments/launch/dragon.py @@ -28,18 +28,23 @@ import typing as t +from typing_extensions import override + from smartsim.log import get_logger -from ...common import StringArgument, set_check_input +from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) -class DragonArgBuilder(LaunchArgBuilder): +class DragonLaunchArguments(LaunchArguments): def launcher_str(self) -> str: - """Get the string representation of the launcher""" + """Get the string representation of the launcher + + :returns: The string representation of the launcher + """ return LauncherType.Dragon.value def set_nodes(self, nodes: int) -> None: @@ -54,11 +59,43 @@ def set_tasks_per_node(self, tasks_per_node: int) -> None: :param tasks_per_node: number of tasks per node """ - self.set("tasks-per-node", str(tasks_per_node)) + self.set("tasks_per_node", str(tasks_per_node)) + @override def set(self, key: str, value: str | None) -> None: - """Set the launch arguments""" + """Set an arbitrary launch argument + + :param key: The launch argument + :param value: A string representation of the value for the launch + argument (if applicable), otherwise `None` + """ set_check_input(key, value) if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + + def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: + """Specify the node feature for this job + + :param feature_list: a collection of strings representing the required + node features. Currently supported node features are: "gpu" + """ + if isinstance(feature_list, str): + feature_list = feature_list.strip().split() + elif not all(isinstance(feature, str) for feature in feature_list): + raise TypeError("feature_list must be string or list of strings") + self.set("node-feature", ",".join(feature_list)) + + def set_cpu_affinity(self, devices: t.List[int]) -> None: + """Set the CPU affinity for this job + + :param devices: list of CPU indices to execute on + """ + self.set("cpu-affinity", ",".join(str(device) for device in devices)) + + def set_gpu_affinity(self, devices: t.List[int]) -> None: + """Set the GPU affinity for this job + + :param devices: list of GPU indices to execute on. + """ + self.set("gpu-affinity", ",".join(str(device) for device in devices)) \ No newline at end of file diff --git a/smartsim/settings/arguments/launch/mpi.py b/smartsim/settings/arguments/launch/mpi.py index 1331be317..034a6c5a5 100644 --- a/smartsim/settings/arguments/launch/mpi.py +++ b/smartsim/settings/arguments/launch/mpi.py @@ -29,17 +29,24 @@ import typing as t from smartsim.log import get_logger +from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import set_check_input from ...launchCommand import LauncherType -from ..launchArgBuilder import LaunchArgBuilder +from ..launchArguments import LaunchArguments logger = get_logger(__name__) +_as_mpirun_command = make_shell_format_fn("mpirun") +_as_mpiexec_command = make_shell_format_fn("mpiexec") +_as_orterun_command = make_shell_format_fn("orterun") -class _BaseMPIArgBuilder(LaunchArgBuilder): +class _BaseMPILaunchArguments(LaunchArguments): def _reserved_launch_args(self) -> set[str]: - """Return reserved launch arguments.""" + """Return reserved launch arguments. + + :returns: The set of reserved launcher arguments + """ return {"wd", "wdir"} def set_task_map(self, task_mapping: str) -> None: @@ -199,7 +206,12 @@ def format_launch_args(self) -> t.List[str]: return args def set(self, key: str, value: str | None) -> None: - """Set the launch arguments""" + """Set an arbitrary launch argument + + :param key: The launch argument + :param value: A string representation of the value for the launch + argument (if applicable), otherwise `None` + """ set_check_input(key, value) if key in self._reserved_launch_args(): logger.warning( @@ -214,37 +226,31 @@ def set(self, key: str, value: str | None) -> None: self._launch_args[key] = value -class MpiArgBuilder(_BaseMPIArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) - +@dispatch(with_format=_as_mpirun_command, to_launcher=ShellLauncher) +class MpirunLaunchArguments(_BaseMPILaunchArguments): def launcher_str(self) -> str: - """Get the string representation of the launcher""" - return LauncherType.Mpirun.value + """Get the string representation of the launcher + :returns: The string representation of the launcher + """ + return LauncherType.Mpirun.value -class MpiexecArgBuilder(_BaseMPIArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) +@dispatch(with_format=_as_mpiexec_command, to_launcher=ShellLauncher) +class MpiexecLaunchArguments(_BaseMPILaunchArguments): def launcher_str(self) -> str: - """Get the string representation of the launcher""" - return LauncherType.Mpiexec.value + """Get the string representation of the launcher + :returns: The string representation of the launcher + """ + return LauncherType.Mpiexec.value -class OrteArgBuilder(_BaseMPIArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) +@dispatch(with_format=_as_orterun_command, to_launcher=ShellLauncher) +class OrterunLaunchArguments(_BaseMPILaunchArguments): def launcher_str(self) -> str: - """Get the string representation of the launcher""" - return LauncherType.Orterun.value + """Get the string representation of the launcher + + :returns: The string representation of the launcher + """ + return LauncherType.Orterun.value \ No newline at end of file diff --git a/smartsim/settings/arguments/launchArguments.py b/smartsim/settings/arguments/launchArguments.py index bb1f389f3..d9dd96efb 100644 --- a/smartsim/settings/arguments/launchArguments.py +++ b/smartsim/settings/arguments/launchArguments.py @@ -27,6 +27,7 @@ from __future__ import annotations import copy +import textwrap import typing as t from abc import ABC, abstractmethod @@ -37,28 +38,41 @@ logger = get_logger(__name__) -class LaunchArgBuilder(ABC): - """Abstract base class that defines all generic launcher - argument methods that are not supported. It is the - responsibility of child classes for each launcher to translate - the input parameter to a properly formatted launcher argument. +class LaunchArguments(ABC): + """Abstract base class for launcher arguments. It is the responsibility of + child classes for each launcher to add methods to set input parameters and + to maintain valid state between parameters set by a user. """ def __init__(self, launch_args: t.Dict[str, str | None] | None) -> None: + """Initialize a new `LaunchArguments` instance. + + :param launch_args: A mapping of arguments to (optional) values + """ self._launch_args = copy.deepcopy(launch_args) or {} @abstractmethod def launcher_str(self) -> str: """Get the string representation of the launcher""" - pass @abstractmethod def set(self, arg: str, val: str | None) -> None: - """Set the launch arguments""" - pass + """Set a launch argument + + :param arg: The argument name to set + :param val: The value to set the argument to as a `str` (if + applicable). Otherwise `None` + """ def format_launch_args(self) -> t.Union[t.List[str], None]: - """Build formatted launch arguments""" + """Build formatted launch arguments + + .. warning:: + This method will be removed from this class in a future ticket + + :returns: The launch arguments formatted as a list or `None` if the + arguments cannot be formatted. + """ logger.warning( f"format_launcher_args() not supported for {self.launcher_str()}." ) @@ -71,6 +85,15 @@ def format_comma_sep_env_vars( Slurm takes exports in comma separated lists the list starts with all as to not disturb the rest of the environment for more information on this, see the slurm documentation for srun + + .. warning:: + The return value described in this docstring does not match the + type hint, but I have no idea how this is supposed to be used or + how to resolve the descrepency. I'm not going to try and fix it and + the point is moot as this method is almost certainly going to be + removed in a later ticket. + + :param env_vars: An environment mapping :returns: the formatted string of environment variables """ logger.warning( @@ -82,11 +105,21 @@ def format_env_vars( self, env_vars: t.Dict[str, t.Optional[str]] ) -> t.Union[t.List[str], None]: """Build bash compatible environment variable string for Slurm + + .. warning:: + This method will be removed from this class in a future ticket + + :param env_vars: An environment mapping :returns: the formatted string of environment variables """ logger.warning(f"format_env_vars() not supported for {self.launcher_str()}.") return None def __str__(self) -> str: # pragma: no-cover - string = f"\nLaunch Arguments:\n{fmt_dict(self._launch_args)}" - return string + return textwrap.dedent(f"""\ + Launch Arguments: + Launcher: {self.launcher_str()} + Name: {type(self).__name__} + Arguments: + {fmt_dict(self._launch_args)} + """) \ No newline at end of file diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 53c6be04d..0188304c2 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -417,7 +417,7 @@ def make_shell_format_fn( """ def impl( - args: LaunchArguments, exe: ExecutableProtocol, _env: _EnvironMappingType + args: LaunchArguments, exe: ExecutableProtocol, path: str, _env: _EnvironMappingType ) -> t.Sequence[str]: return ( ( @@ -428,7 +428,7 @@ def impl( ) if run_command is not None else exe.as_program_arguments() - ) + ), path return impl @@ -442,6 +442,7 @@ def __init__(self) -> None: def start(self, command: t.Sequence[str]) -> LaunchedJobID: id_ = create_job_id() exe, *rest = command + print(f"here is the path: {rest}") # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest)) return id_ diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 98c199b83..a29d6dfdb 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -179,4 +179,4 @@ def __str__(self) -> str: # pragma: no-cover string = f"\nLauncher: {self.launcher}{self.launch_args}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" - return string + return string \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index e3f159b7f..8a8414e73 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -66,7 +66,7 @@ def test_dragon_class_methods(function, value, flag, result): @pytest.mark.parametrize("cpu_affinity", (NOT_SET, [1], [1, 2, 3])) @pytest.mark.parametrize("gpu_affinity", (NOT_SET, [1], [1, 2, 3])) def test_formatting_launch_args_into_request( - mock_echo_executable, nodes, tasks_per_node, cpu_affinity, gpu_affinity + mock_echo_executable, nodes, tasks_per_node, cpu_affinity, gpu_affinity, test_dir ): launch_args = DragonLaunchArguments({}) if nodes is not NOT_SET: @@ -77,7 +77,7 @@ def test_formatting_launch_args_into_request( launch_args.set_cpu_affinity(cpu_affinity) if gpu_affinity is not NOT_SET: launch_args.set_gpu_affinity(gpu_affinity) - req, policy = _as_run_request_args_and_policy(launch_args, mock_echo_executable, {}) + req, policy = _as_run_request_args_and_policy(launch_args, mock_echo_executable, test_dir, {}) expected_args = { k: v @@ -88,7 +88,7 @@ def test_formatting_launch_args_into_request( if v is not NOT_SET } expected_run_req = DragonRunRequestView( - exe="echo", exe_args=["hello", "world"], path="/tmp", env={}, **expected_args + exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, **expected_args ) assert req.exe == expected_run_req.exe assert req.exe_args == expected_run_req.exe_args @@ -96,10 +96,11 @@ def test_formatting_launch_args_into_request( assert req.tasks_per_node == expected_run_req.tasks_per_node assert req.hostlist == expected_run_req.hostlist assert req.pmi_enabled == expected_run_req.pmi_enabled + assert req.path == expected_run_req.path expected_run_policy_args = { k: v for k, v in {"cpu_affinity": cpu_affinity, "gpu_affinity": gpu_affinity}.items() if v is not NOT_SET } - assert policy == DragonRunPolicy(**expected_run_policy_args) + assert policy == DragonRunPolicy(**expected_run_policy_args) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 69222ae4e..fb509ef9b 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -283,9 +283,7 @@ def test_invalid_hostlist_format(launcher): ), ), ) -def test_formatting_launch_args( - mock_echo_executable, cls, fmt, cmd, args, expected, test_dir -): +def test_formatting_launch_args(mock_echo_executable, cls, fmt, cmd, args, expected, test_dir): fmt_cmd, path = fmt(cls(args), mock_echo_executable, test_dir, {}) assert tuple(fmt_cmd) == (cmd,) + expected assert path == test_dir From 91f3af81d7b95c59fa301731bfe7c60f3ef2b3cc Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 29 Jul 2024 14:45:32 -0500 Subject: [PATCH 39/82] all tests passing and make style --- smartsim/experiment.py | 4 +- smartsim/settings/arguments/launch/dragon.py | 2 +- smartsim/settings/arguments/launch/mpi.py | 2 +- .../settings/arguments/launchArguments.py | 2 +- smartsim/settings/dispatch.py | 11 +- smartsim/settings/launchSettings.py | 2 +- .../test_settings/test_dragonLauncher.py | 6 +- .../test_settings/test_mpiLauncher.py | 4 +- .../easy/marked/invalidtag.txt | 2 +- tests/test_experiment.py | 284 ++++++++++++++++++ 10 files changed, 305 insertions(+), 14 deletions(-) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index c29b547a6..e5168cf13 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -28,8 +28,6 @@ from __future__ import annotations -from __future__ import annotations - import datetime import os import os.path as osp @@ -161,7 +159,7 @@ def __init__(self, name: str, exp_path: str | None = None): self.exp_path = exp_path """The path under which the experiment operate""" - + self._run_ID = ( "run-" + datetime.datetime.now().strftime("%H:%M:%S") diff --git a/smartsim/settings/arguments/launch/dragon.py b/smartsim/settings/arguments/launch/dragon.py index 5dcf8fe35..98b91059c 100644 --- a/smartsim/settings/arguments/launch/dragon.py +++ b/smartsim/settings/arguments/launch/dragon.py @@ -98,4 +98,4 @@ def set_gpu_affinity(self, devices: t.List[int]) -> None: :param devices: list of GPU indices to execute on. """ - self.set("gpu-affinity", ",".join(str(device) for device in devices)) \ No newline at end of file + self.set("gpu-affinity", ",".join(str(device) for device in devices)) diff --git a/smartsim/settings/arguments/launch/mpi.py b/smartsim/settings/arguments/launch/mpi.py index 034a6c5a5..85fd38145 100644 --- a/smartsim/settings/arguments/launch/mpi.py +++ b/smartsim/settings/arguments/launch/mpi.py @@ -253,4 +253,4 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Orterun.value \ No newline at end of file + return LauncherType.Orterun.value diff --git a/smartsim/settings/arguments/launchArguments.py b/smartsim/settings/arguments/launchArguments.py index d9dd96efb..61f837d98 100644 --- a/smartsim/settings/arguments/launchArguments.py +++ b/smartsim/settings/arguments/launchArguments.py @@ -122,4 +122,4 @@ def __str__(self) -> str: # pragma: no-cover Name: {type(self).__name__} Arguments: {fmt_dict(self._launch_args)} - """) \ No newline at end of file + """) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 0188304c2..b0c6c1876 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -252,8 +252,10 @@ def create_adapter_from_launcher( f"exactly `{self.launcher_type}`" ) - def format_(exe: ExecutableProtocol, env: _EnvironMappingType) -> _LaunchableT: - return self.formatter(arguments, exe, env) + def format_( + exe: ExecutableProtocol, env: _EnvironMappingType, path: str + ) -> _LaunchableT: + return self.formatter(arguments, exe, path, env) return _LauncherAdapter(launcher, format_) @@ -417,7 +419,10 @@ def make_shell_format_fn( """ def impl( - args: LaunchArguments, exe: ExecutableProtocol, path: str, _env: _EnvironMappingType + args: LaunchArguments, + exe: ExecutableProtocol, + path: str, + _env: _EnvironMappingType, ) -> t.Sequence[str]: return ( ( diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index a29d6dfdb..98c199b83 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -179,4 +179,4 @@ def __str__(self) -> str: # pragma: no-cover string = f"\nLauncher: {self.launcher}{self.launch_args}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" - return string \ No newline at end of file + return string diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 8a8414e73..38ee11486 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -77,7 +77,9 @@ def test_formatting_launch_args_into_request( launch_args.set_cpu_affinity(cpu_affinity) if gpu_affinity is not NOT_SET: launch_args.set_gpu_affinity(gpu_affinity) - req, policy = _as_run_request_args_and_policy(launch_args, mock_echo_executable, test_dir, {}) + req, policy = _as_run_request_args_and_policy( + launch_args, mock_echo_executable, test_dir, {} + ) expected_args = { k: v @@ -103,4 +105,4 @@ def test_formatting_launch_args_into_request( for k, v in {"cpu_affinity": cpu_affinity, "gpu_affinity": gpu_affinity}.items() if v is not NOT_SET } - assert policy == DragonRunPolicy(**expected_run_policy_args) \ No newline at end of file + assert policy == DragonRunPolicy(**expected_run_policy_args) diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index fb509ef9b..69222ae4e 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -283,7 +283,9 @@ def test_invalid_hostlist_format(launcher): ), ), ) -def test_formatting_launch_args(mock_echo_executable, cls, fmt, cmd, args, expected, test_dir): +def test_formatting_launch_args( + mock_echo_executable, cls, fmt, cmd, args, expected, test_dir +): fmt_cmd, path = fmt(cls(args), mock_echo_executable, test_dir, {}) assert tuple(fmt_cmd) == (cmd,) + expected assert path == test_dir diff --git a/tests/test_configs/generator_files/easy/marked/invalidtag.txt b/tests/test_configs/generator_files/easy/marked/invalidtag.txt index 90a625319..2165ae8d1 100644 --- a/tests/test_configs/generator_files/easy/marked/invalidtag.txt +++ b/tests/test_configs/generator_files/easy/marked/invalidtag.txt @@ -1,3 +1,3 @@ some text before -some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params +some params are valid and others are ;INVALID; but we mostly encounter valid params some text after diff --git a/tests/test_experiment.py b/tests/test_experiment.py index e69de29bb..55f315a9a 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -0,0 +1,284 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import annotations + +import dataclasses +import itertools +import tempfile +import typing as t +import uuid +import weakref + +import pytest + +from smartsim.entity import _mock, entity +from smartsim.experiment import Experiment +from smartsim.launchable import job +from smartsim.settings import dispatch, launchSettings +from smartsim.settings.arguments import launchArguments + +pytestmark = pytest.mark.group_a + + +# TODO make sure dispatcher is patched +# @pytest.fixture +# def experiment(monkeypatch, test_dir, dispatcher): +# """A simple experiment instance with a unique name anda unique name and its +# own directory to be used by tests +# """ +# exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) +# monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") +# yield exp + + +@pytest.fixture +def experiment(monkeypatch, test_dir, dispatcher): + """A simple experiment instance with a unique name anda unique name and its + own directory to be used by tests + """ + exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) + monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) + monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") + yield exp + + +# @pytest.fixture +# def patch_experiment_job_path(test_dir, monkeypatch): +# exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) +# monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") +# yield exp + + +@pytest.fixture +def dispatcher(): + """A pre-configured dispatcher to be used by experiments that simply + dispatches any jobs with `MockLaunchArgs` to a `NoOpRecordLauncher` + """ + d = dispatch.Dispatcher() + to_record: dispatch._FormatterType[MockLaunchArgs, LaunchRecord] = ( + lambda settings, exe, path, env: LaunchRecord(settings, exe, env, path) + ) + d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) + yield d + + +@pytest.fixture +def job_maker(monkeypatch): + """A fixture to generate a never ending stream of `Job` instances each + configured with a unique `MockLaunchArgs` instance, but identical + executable. + """ + + def iter_jobs(): + for i in itertools.count(): + settings = launchSettings.LaunchSettings("local") + monkeypatch.setattr(settings, "_arguments", MockLaunchArgs(i)) + yield job.Job(EchoHelloWorldEntity(), settings) + + jobs = iter_jobs() + yield lambda: next(jobs) + + +JobMakerType: t.TypeAlias = t.Callable[[], job.Job] + + +@dataclasses.dataclass(frozen=True, eq=False) +class NoOpRecordLauncher(dispatch.LauncherProtocol): + """Simple launcher to track the order of and mapping of ids to `start` + method calls. It has exactly three attrs: + + - `created_by_experiment`: + A back ref to the experiment used when calling + `NoOpRecordLauncher.create`. + + - `launched_order`: + An append-only list of `LaunchRecord`s that it has "started". Notice + that this launcher will not actually open any subprocesses/run any + threads/otherwise execute the contents of the record on the system + + - `ids_to_launched`: + A mapping where keys are the generated launched id returned from + a `NoOpRecordLauncher.start` call and the values are the + `LaunchRecord` that was passed into `NoOpRecordLauncher.start` to + cause the id to be generated. + + This is helpful for testing that launchers are handling the expected input + """ + + created_by_experiment: Experiment + launched_order: list[LaunchRecord] = dataclasses.field(default_factory=list) + ids_to_launched: dict[dispatch.LaunchedJobID, LaunchRecord] = dataclasses.field( + default_factory=dict + ) + + __hash__ = object.__hash__ + + @classmethod + def create(cls, exp): + return cls(exp) + + def start(self, record: LaunchRecord): + id_ = dispatch.create_job_id() + self.launched_order.append(record) + self.ids_to_launched[id_] = record + return id_ + + +@dataclasses.dataclass(frozen=True) +class LaunchRecord: + launch_args: launchArguments.LaunchArguments + entity: entity.SmartSimEntity + env: t.Mapping[str, str | None] + path: str + + @classmethod + def from_job(cls, job: job.Job): + """Create a launch record for what we would expect a launch record to + look like having gone through the launching process + + :param job: A job that has or will be launched through an experiment + and dispatched to a `NoOpRecordLauncher` + :returns: A `LaunchRecord` that should evaluate to being equivilient to + that of the one stored in the `NoOpRecordLauncher` + """ + args = job._launch_settings.launch_args + entity = job._entity + env = job._launch_settings.env_vars + path = "/tmp/job" + return cls(args, entity, env, path) + + +class MockLaunchArgs(launchArguments.LaunchArguments): + """A `LaunchArguments` subclass that will evaluate as true with another if + and only if they were initialized with the same id. In practice this class + has no arguments to set. + """ + + def __init__(self, id_: int): + super().__init__({}) + self.id = id_ + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return other.id == self.id + + def launcher_str(self): + return "test-launch-args" + + def set(self, arg, val): ... + + +class EchoHelloWorldEntity(entity.SmartSimEntity): + """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" + + def __init__(self): + path = tempfile.TemporaryDirectory() + self._finalizer = weakref.finalize(self, path.cleanup) + super().__init__("test-entity", _mock.Mock()) + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return self.as_program_arguments() == other.as_program_arguments() + + def as_program_arguments(self): + return ("echo", "Hello", "World!") + + +def test_start_raises_if_no_args_supplied(experiment): + with pytest.raises(TypeError, match="missing 1 required positional argument"): + experiment.start() + + +# fmt: off +@pytest.mark.parametrize( + "num_jobs", [pytest.param(i, id=f"{i} job(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)] +) +@pytest.mark.parametrize( + "make_jobs", ( + pytest.param(lambda maker, n: tuple(maker() for _ in range(n)), id="many job instances"), + pytest.param(lambda maker, n: (maker(),) * n , id="same job instance many times"), + ), +) +# fmt: on +def test_start_can_launch_jobs( + experiment: Experiment, + job_maker: JobMakerType, + make_jobs: t.Callable[[JobMakerType, int], tuple[job.Job, ...]], + num_jobs: int, +) -> None: + jobs = make_jobs(job_maker, num_jobs) + assert len(experiment._active_launchers) == 0, "Initialized w/ launchers" + launched_ids = experiment.start(*jobs) + assert len(experiment._active_launchers) == 1, "Unexpected number of launchers" + (launcher,) = experiment._active_launchers + assert isinstance(launcher, NoOpRecordLauncher), "Unexpected launcher type" + assert launcher.created_by_experiment is experiment, "Not created by experiment" + assert ( + len(jobs) == len(launcher.launched_order) == len(launched_ids) == num_jobs + ), "Inconsistent number of jobs/launched jobs/launched ids/expected number of jobs" + expected_launched = [LaunchRecord.from_job(job) for job in jobs] + + # Check that `job_a, job_b, job_c, ...` are started in that order when + # calling `experiemnt.start(job_a, job_b, job_c, ...)` + assert expected_launched == list(launcher.launched_order), "Unexpected launch order" + + # Similarly, check that `id_a, id_b, id_c, ...` corresponds to + # `job_a, job_b, job_c, ...` when calling + # `id_a, id_b, id_c, ... = experiemnt.start(job_a, job_b, job_c, ...)` + expected_id_map = dict(zip(launched_ids, expected_launched)) + assert expected_id_map == launcher.ids_to_launched, "IDs returned in wrong order" + + +@pytest.mark.parametrize( + "num_starts", + [pytest.param(i, id=f"{i} start(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)], +) +def test_start_can_start_a_job_multiple_times_accross_multiple_calls( + experiment: Experiment, job_maker: JobMakerType, num_starts: int +) -> None: + assert len(experiment._active_launchers) == 0, "Initialized w/ launchers" + job = job_maker() + ids_to_launches = { + experiment.start(job)[0]: LaunchRecord.from_job(job) for _ in range(num_starts) + } + assert len(experiment._active_launchers) == 1, "Did not reuse the launcher" + (launcher,) = experiment._active_launchers + assert isinstance(launcher, NoOpRecordLauncher), "Unexpected launcher type" + assert len(launcher.launched_order) == num_starts, "Unexpected number launches" + + # Check that a single `job` instance can be launched and re-launcherd and + # that `id_a, id_b, id_c, ...` corresponds to + # `"start_a", "start_b", "start_c", ...` when calling + # ```py + # id_a = experiment.start(job) # "start_a" + # id_b = experiment.start(job) # "start_b" + # id_c = experiment.start(job) # "start_c" + # ... + # ``` + assert ids_to_launches == launcher.ids_to_launched, "Job was not re-launched" From c1ec227b9a693e0e42cd9818785c9ab4a30b21d1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 29 Jul 2024 15:11:31 -0500 Subject: [PATCH 40/82] mypy errors partially corrected --- smartsim/_core/generation/generator.py | 14 ++++++++++---- smartsim/settings/dispatch.py | 11 ++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index a81396004..a649ad3e3 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -42,6 +42,7 @@ from ...database import FeatureStore from ...entity import Application, TaggedFilesHierarchy +from ...entity.files import EntityFiles from ...launchable import Job, JobGroup from ...log import get_logger from ..entrypoints import file_operations @@ -191,7 +192,7 @@ def generate_experiment(self) -> str: log_file.write(f"Generation start date and time: {dt_string}\n") # Prevent access to type FeatureStore entities - if isinstance(self.job.entity, Application) and self.job.entity.files: + if isinstance(self.job.entity, Application): # Perform file system operations on attached files self._build_operations() @@ -210,8 +211,7 @@ def _build_operations(self) -> None: app = t.cast(Application, self.job.entity) self._get_symlink_file_system_operation(app, self.path) self._write_tagged_entity_files(app, self.path) - if app.files: - self._get_copy_file_system_operation(app, self.path) + self._get_copy_file_system_operation(app, self.path) @staticmethod def _get_copy_file_system_operation(app: Application, dest: str) -> None: @@ -220,9 +220,11 @@ def _get_copy_file_system_operation(app: Application, dest: str) -> None: :param linked_file: The file to be copied. :return: A list of copy file system operations. """ + if app.files is None: + return parser = get_parser() for src in app.files.copy: - if Path(src).is_dir: + if Path(src).is_dir: # TODO figure this out, or how to replace cmd = f"copy {src} {dest} --dirs_exist_ok" else: cmd = f"copy {src} {dest}" @@ -237,6 +239,8 @@ def _get_symlink_file_system_operation(app: Application, dest: str) -> None: :param linked_file: The file to be symlinked. :return: A list of symlink file system operations. """ + if app.files is None: + return parser = get_parser() for sym in app.files.link: # Normalize the path to remove trailing slashes @@ -259,6 +263,8 @@ def _write_tagged_entity_files(app: Application, dest: str) -> None: :param entity: a Application instance """ + if app.files is None: + return if app.files.tagged: to_write = [] diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index b0c6c1876..983a748fb 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -58,13 +58,13 @@ a job """ _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _EnvironMappingType], _LaunchableT + [_DispatchableT, "ExecutableProtocol", str, _EnvironMappingType], _LaunchableT ] """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ _LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType]" + "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType, str]" ) """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings @@ -388,7 +388,7 @@ def create(cls, exp: Experiment, /) -> Self: ... def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[LaunchArguments, t.Sequence[str]]: +) -> _FormatterType[LaunchArguments, tuple[t.Sequence[str], str]]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -423,7 +423,7 @@ def impl( exe: ExecutableProtocol, path: str, _env: _EnvironMappingType, - ) -> t.Sequence[str]: + ) -> t.Tuple[t.Sequence[str], str]: return ( ( run_command, @@ -444,7 +444,8 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - def start(self, command: t.Sequence[str]) -> LaunchedJobID: + # TODO inject path here + def start(self, command: tuple[t.Sequence[str], str]) -> LaunchedJobID: id_ = create_job_id() exe, *rest = command print(f"here is the path: {rest}") From 9370cb070ee96eee131a3531b43289ab0e074e25 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 29 Jul 2024 20:26:32 -0500 Subject: [PATCH 41/82] passing tests --- smartsim/_core/generation/generator.py | 23 +++++++++++---------- smartsim/settings/dispatch.py | 6 +++--- tests/test_generator/test_generator.py | 28 +++++++++++++------------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index a649ad3e3..1cc933014 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -64,10 +64,11 @@ def __init__(self, gen_path: str, run_ID: str, job: Job) -> None: The Generator class is responsible for creating Job directories. It ensures that paths adhere to SmartSim path standards. Additionally, - it creates a log directory for telemetry data and handles symlinking, - configuration, and file copying within the job directory. + it creates a log directory for telemetry data to handle symlinking, + configuration, and file copying to the job directory. :param gen_path: Path in which files need to be generated + :param run_ID: The id of the Experiment :param job: Reference to a name, SmartSimEntity and LaunchSettings """ self.job = job @@ -162,10 +163,9 @@ def generate_experiment(self) -> str: """Generate the directories Generate the file structure for a SmartSim experiment. This - includes the writing and configuring of input files for a - job. + includes writing and configuring input files for a job. - To have files or directories present in the created entity + To have files or directories present in the created job directories, such as datasets or input files, call ``entity.attach_generator_files`` prior to generation. See ``entity.attach_generator_files`` for more information on @@ -217,14 +217,14 @@ def _build_operations(self) -> None: def _get_copy_file_system_operation(app: Application, dest: str) -> None: """Get copy file system operation for a file. - :param linked_file: The file to be copied. - :return: A list of copy file system operations. + :param app: The Application attached to the Job + :param dest: Path to copy files """ if app.files is None: return parser = get_parser() for src in app.files.copy: - if Path(src).is_dir: # TODO figure this out, or how to replace + if os.path.isdir(src): cmd = f"copy {src} {dest} --dirs_exist_ok" else: cmd = f"copy {src} {dest}" @@ -236,8 +236,8 @@ def _get_copy_file_system_operation(app: Application, dest: str) -> None: def _get_symlink_file_system_operation(app: Application, dest: str) -> None: """Get symlink file system operation for a file. - :param linked_file: The file to be symlinked. - :return: A list of symlink file system operations. + :param app: The Application attached to the Job + :param dest: Path to symlink files """ if app.files is None: return @@ -261,7 +261,8 @@ def _write_tagged_entity_files(app: Application, dest: str) -> None: specifically deals with the tagged files attached to an Ensemble. - :param entity: a Application instance + :param app: The Application attached to the Job + :param dest: Path to configure files """ if app.files is None: return diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 983a748fb..5cf6aafc4 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -447,10 +447,10 @@ def __init__(self) -> None: # TODO inject path here def start(self, command: tuple[t.Sequence[str], str]) -> LaunchedJobID: id_ = create_job_id() - exe, *rest = command - print(f"here is the path: {rest}") + args, path = command + exe, *rest = args # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest)) + self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path) return id_ @classmethod diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index 35f30d1bf..1f4abce7d 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -122,20 +122,20 @@ def test_generate_copy_file(fileutils, wlmutils, test_dir): # TODO FLAGGED -def test_generate_copy_directory(fileutils, wlmutils, test_dir): - # Create the Job and attach generator file - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python", "RunSettings") - copy_dir = get_gen_file(fileutils, "to_copy_dir") - print(copy_dir) - app.attach_generator_files(to_copy=copy_dir) - job = Job(app, launch_settings) - - # Create the experiment - experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) - gen.generate_experiment() - expected_file = pathlib.Path(gen.path) / "to_copy_dir" / "mock.txt" +# def test_generate_copy_directory(fileutils, wlmutils, test_dir): +# # Create the Job and attach generator file +# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) +# app = Application("name", "python", "RunSettings") +# copy_dir = get_gen_file(fileutils, "to_copy_dir") +# app.attach_generator_files(to_copy=copy_dir) +# job = Job(app, launch_settings) + +# # Create the experiment +# experiment_path = osp.join(test_dir, "experiment_name") +# gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) +# gen.generate_experiment() +# expected_file = pathlib.Path(gen.path) / "to_copy_dir" / "mock.txt" +# assert osp.isfile(expected_file) def test_generate_symlink_directory(fileutils, wlmutils, test_dir): From 023d51bf22d3ace4c42f0e5133ce2eae776b921e Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 29 Jul 2024 20:35:06 -0500 Subject: [PATCH 42/82] mark test --- tests/test_generator/test_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index 1f4abce7d..b59d14de5 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -20,6 +20,8 @@ # TODO test ensemble copy, config, symlink when ensemble.attach_generator_files added # TODO remove ensemble tests and replace with JobGroup when start jobgroup is supported +pytestmark = pytest.mark.group_a + def get_gen_file(fileutils, filename): return fileutils.get_test_conf_path(osp.join("generator_files", filename)) From 0df2aadd38ccd575899cabde0707562c81904616 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 30 Jul 2024 11:21:06 -0500 Subject: [PATCH 43/82] update popen arguments --- smartsim/settings/dispatch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 5cf6aafc4..19351df54 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -450,7 +450,8 @@ def start(self, command: tuple[t.Sequence[str], str]) -> LaunchedJobID: args, path = command exe, *rest = args # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path) + self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path, env={}, stdin=None, stdout=None) + # env accepts a dictionary return id_ @classmethod From 9a97620d826d24b92c9e508f321aa0727350ef01 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 30 Jul 2024 11:41:23 -0500 Subject: [PATCH 44/82] fixes issues --- smartsim/_core/utils/helpers.py | 2 +- smartsim/experiment.py | 2 +- .../easy/marked/invalidtag.txt | 2 +- tests/test_generator/test_generator.py | 37 +++++++------------ 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index 0e58d7a78..d193b6604 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -74,7 +74,7 @@ def unpack_colo_fs_identifier(fs_id: str) -> str: return "_" + fs_id if fs_id else "" -def create_short_id_str() -> str: # here +def create_short_id_str() -> str: return str(uuid.uuid4())[:7] diff --git a/smartsim/experiment.py b/smartsim/experiment.py index e5168cf13..0fdf885f8 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -162,7 +162,7 @@ def __init__(self, name: str, exp_path: str | None = None): self._run_ID = ( "run-" - + datetime.datetime.now().strftime("%H:%M:%S") + + datetime.datetime.now().strftime("%H-%M-%S") + "-" + datetime.datetime.now().strftime("%Y-%m-%d") ) diff --git a/tests/test_configs/generator_files/easy/marked/invalidtag.txt b/tests/test_configs/generator_files/easy/marked/invalidtag.txt index 2165ae8d1..90a625319 100644 --- a/tests/test_configs/generator_files/easy/marked/invalidtag.txt +++ b/tests/test_configs/generator_files/easy/marked/invalidtag.txt @@ -1,3 +1,3 @@ some text before -some params are valid and others are ;INVALID; but we mostly encounter valid params +some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params some text after diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index b59d14de5..6703763bb 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -123,21 +123,20 @@ def test_generate_copy_file(fileutils, wlmutils, test_dir): assert osp.isfile(expected_file) -# TODO FLAGGED -# def test_generate_copy_directory(fileutils, wlmutils, test_dir): -# # Create the Job and attach generator file -# launch_settings = LaunchSettings(wlmutils.get_test_launcher()) -# app = Application("name", "python", "RunSettings") -# copy_dir = get_gen_file(fileutils, "to_copy_dir") -# app.attach_generator_files(to_copy=copy_dir) -# job = Job(app, launch_settings) - -# # Create the experiment -# experiment_path = osp.join(test_dir, "experiment_name") -# gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) -# gen.generate_experiment() -# expected_file = pathlib.Path(gen.path) / "to_copy_dir" / "mock.txt" -# assert osp.isfile(expected_file) +def test_generate_copy_directory(fileutils, wlmutils, test_dir): + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name", "python", "RunSettings") + copy_dir = get_gen_file(fileutils, "to_copy_dir") + app.attach_generator_files(to_copy=copy_dir) + job = Job(app, launch_settings) + + # Create the experiment + experiment_path = osp.join(test_dir, "experiment_name") + gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) + gen.generate_experiment() + expected_file = pathlib.Path(gen.path) / "mock.txt" + assert osp.isfile(expected_file) def test_generate_symlink_directory(fileutils, wlmutils, test_dir): @@ -165,11 +164,6 @@ def test_generate_symlink_directory(fileutils, wlmutils, test_dir): def test_generate_symlink_file(fileutils, wlmutils, test_dir): - assert osp.isfile( - pathlib.Path( - "/lus/sonexion/richaama/Matt/SmartSim/tests/test_configs/generator_files/to_symlink_dir/mock2.txt" - ) - ) # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("name", "python", "RunSettings") @@ -236,9 +230,6 @@ def test_generate_configure(fileutils, wlmutils, test_dir): assert osp.isfile(smartsim_params_path) -# Ensemble Tests - - def test_exp_private_generate_method_ensemble(test_dir, wlmutils): """Test that Job directory was created from Experiment.""" ensemble = Ensemble("ensemble-name", "echo", replicas=2) From 8c86007ffa72f33c7e993c7a42b7363f430241c7 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 30 Jul 2024 16:33:12 -0500 Subject: [PATCH 45/82] tests passing --- smartsim/_core/generation/generator.py | 52 ++++++++------------ smartsim/entity/ensemble.py | 5 +- smartsim/launchable/job.py | 9 +--- tests/test_generator/test_generator.py | 67 +++++++++++++++----------- 4 files changed, 60 insertions(+), 73 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 1cc933014..445cb0160 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -75,17 +75,24 @@ def __init__(self, gen_path: str, run_ID: str, job: Job) -> None: # Generate the job folder path self.path = self._generate_job_path(job, gen_path, run_ID) # Generate the log folder path - self.log_path = self._generate_log_path(gen_path) + self.log_path = self._generate_log_path(job, gen_path, run_ID) - def _generate_log_path(self, gen_path: str) -> str: + def _generate_log_path(self, job: Job, gen_path: str, run_ID: str) -> str: """ Generate the path for the log folder. :param gen_path: The base path job generation :returns str: The generated path for the log directory """ - log_path = os.path.join(gen_path, "log") - return log_path + job_type = f"{job.__class__.__name__.lower()}s" + path = os.path.join( + gen_path, + run_ID, + job_type, + job.name, + "log", + ) + return path def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: """ @@ -97,35 +104,14 @@ def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: :param run_ID: The experiments unique run ID :returns str: The generated path for the job. """ - # Attr set in Job to check if Job was created by an Ensemble - if job._ensemble_name is None: - job_type = f"{job.__class__.__name__.lower()}s" - entity_type = ( - f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" - ) - path = os.path.join( - gen_path, - run_ID, - job_type, - f"{job.name}-{create_short_id_str()}", - entity_type, - "run", - ) - # Job was created via Ensemble - else: - job_type = "ensembles" - entity_type = ( - f"{job.entity.__class__.__name__.lower()}-{create_short_id_str()}" - ) - path = os.path.join( - gen_path, - run_ID, - job_type, - job._ensemble_name, - f"{job.name}", - entity_type, - "run", - ) + job_type = f"{job.__class__.__name__.lower()}s" + path = os.path.join( + gen_path, + run_ID, + job_type, + job.name, + "run", + ) return path @property diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index 540d9ca84..9c95efce0 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -110,7 +110,4 @@ def as_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: apps = self._create_applications() if not apps: raise ValueError("There are no members as part of this ensemble") - return tuple( - Job(app, settings, f"job_{i}", ensemble_name=self.name) - for i, app in enumerate(apps, 1) - ) + return tuple(Job(app, settings, app.name) for i, app in enumerate(apps, 1)) diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index c3a97acde..22b812567 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -54,17 +54,12 @@ def __init__( self, entity: SmartSimEntity, launch_settings: LaunchSettings, - name: str = "job", - *, - ensemble_name: t.Optional[str] = None, + name: str = None, ): super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) - self._name = deepcopy(name) - self._ensemble_name = ensemble_name - if self._ensemble_name is not None: - self._ensemble_name += f"-{create_short_id_str()}" + self._name = deepcopy(name) if name else deepcopy(entity.name) # TODO: self.warehouse_runner = JobWarehouseRunner # TODO do we want the user to be allowed to reset the Job name? Therefore, add setter diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py index 6703763bb..9dcb4b682 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator/test_generator.py @@ -1,4 +1,3 @@ -import datetime import filecmp import os import pathlib @@ -12,9 +11,7 @@ from smartsim import Experiment from smartsim._core.generation.generator import Generator from smartsim.entity import Application, Ensemble -from smartsim.launchable import Job, JobGroup -from smartsim.settings.arguments.launch import SlurmLaunchArguments -from smartsim.settings.dispatch import Dispatcher +from smartsim.launchable import Job from smartsim.settings.launchSettings import LaunchSettings # TODO test ensemble copy, config, symlink when ensemble.attach_generator_files added @@ -37,12 +34,10 @@ def start(self, _): return "anything" -# Mock Shell Format fn -def make_shell_format_fn(run_command: str | None): ... - - # Mock Application class EchoApp: + name = "echo_app" + def as_program_arguments(self): return ["echo", "Hello", "World!"] @@ -84,26 +79,44 @@ def test_log_file_path(generator_instance): assert generator_instance.log_file == expected_path -def test_generate_job_directory(generator_instance): +def test_generate_job_directory(test_dir, wlmutils): """Test that Job directory was created.""" - generator_instance.generate_experiment() - assert osp.isdir(generator_instance.path) - assert osp.isdir(generator_instance.log_path) - assert osp.isfile(osp.join(generator_instance.path, "smartsim_params.txt")) + experiment_path = osp.join(test_dir, "experiment_name") + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("app_name", exe="python", run_settings="RunSettings") + job = Job(app, launch_settings) + run_ID = "mock_run" + gen = Generator(gen_path=experiment_path, run_ID=run_ID, job=job) + gen.generate_experiment() + expected_run_path = ( + pathlib.Path(experiment_path) + / run_ID + / f"{job.__class__.__name__.lower()}s" + / app.name + / "run" + ) + expected_log_path = ( + pathlib.Path(experiment_path) + / run_ID + / f"{job.__class__.__name__.lower()}s" + / app.name + / "log" + ) + assert gen.path == str(expected_run_path) + assert gen.log_path == str(expected_log_path) + assert osp.isdir(expected_run_path) + assert osp.isdir(expected_log_path) + assert osp.isfile(osp.join(expected_run_path, "smartsim_params.txt")) def test_exp_private_generate_method_app(test_dir, job_instance): """Test that Job directory was created from Experiment.""" - no_op_dispatch = Dispatcher() - no_op_dispatch.dispatch( - SlurmLaunchArguments, - with_format=make_shell_format_fn("run_command"), - to_launcher=NoOpLauncher, - ) no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) job_execution_path = no_op_exp._generate(job_instance) assert osp.isdir(job_execution_path) - assert osp.isdir(pathlib.Path(no_op_exp.exp_path) / "log") + head, _ = os.path.split(job_execution_path) + expected_log_path = pathlib.Path(head) / "log" + assert osp.isdir(expected_log_path) assert osp.isfile(osp.join(job_execution_path, "smartsim_params.txt")) @@ -224,7 +237,7 @@ def test_generate_configure(fileutils, wlmutils, test_dir): for written, correct in zip(configured_files, correct_files): assert filecmp.cmp(written, correct) # Validate that log file exists - assert osp.isdir(pathlib.Path(experiment_path) / "log") + assert osp.isdir(gen.log_path) # Validate that smartsim params files exists smartsim_params_path = osp.join(job_path, "smartsim_params.txt") assert osp.isfile(smartsim_params_path) @@ -235,17 +248,13 @@ def test_exp_private_generate_method_ensemble(test_dir, wlmutils): ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) - no_op_dispatch = Dispatcher() - no_op_dispatch.dispatch( - launch_settings, - with_format=make_shell_format_fn("run_command"), - to_launcher=NoOpLauncher, - ) no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) for job in job_list: job_execution_path = no_op_exp._generate(job) + head, _ = os.path.split(job_execution_path) + expected_log_path = pathlib.Path(head) / "log" assert osp.isdir(job_execution_path) - assert osp.isdir(pathlib.Path(no_op_exp.exp_path) / "log") + assert osp.isdir(pathlib.Path(expected_log_path)) def test_generate_ensemble_directory(test_dir, wlmutils): @@ -257,4 +266,4 @@ def test_generate_ensemble_directory(test_dir, wlmutils): gen = Generator(gen_path=test_dir, run_ID=run_ID, job=job) gen.generate_experiment() assert osp.isdir(gen.path) - assert osp.isdir(pathlib.Path(test_dir) / "log") + assert osp.isdir(pathlib.Path(gen.log_path)) From e991b54dc05488277305fcde3ffca30ae817eed8 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 31 Jul 2024 10:20:56 -0500 Subject: [PATCH 46/82] mypy errors and failing tests --- smartsim/launchable/job.py | 2 +- .../easy/marked/invalidtag.txt | 2 +- tests/test_experiment.py | 20 +------------------ 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index 22b812567..79badd5ee 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -54,7 +54,7 @@ def __init__( self, entity: SmartSimEntity, launch_settings: LaunchSettings, - name: str = None, + name: str | None = None, ): super().__init__() self._entity = deepcopy(entity) diff --git a/tests/test_configs/generator_files/easy/marked/invalidtag.txt b/tests/test_configs/generator_files/easy/marked/invalidtag.txt index 90a625319..2165ae8d1 100644 --- a/tests/test_configs/generator_files/easy/marked/invalidtag.txt +++ b/tests/test_configs/generator_files/easy/marked/invalidtag.txt @@ -1,3 +1,3 @@ some text before -some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params +some params are valid and others are ;INVALID; but we mostly encounter valid params some text after diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 55f315a9a..6c2378cb5 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -44,17 +44,6 @@ pytestmark = pytest.mark.group_a -# TODO make sure dispatcher is patched -# @pytest.fixture -# def experiment(monkeypatch, test_dir, dispatcher): -# """A simple experiment instance with a unique name anda unique name and its -# own directory to be used by tests -# """ -# exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) -# monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") -# yield exp - - @pytest.fixture def experiment(monkeypatch, test_dir, dispatcher): """A simple experiment instance with a unique name anda unique name and its @@ -62,17 +51,10 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") + monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/job") yield exp -# @pytest.fixture -# def patch_experiment_job_path(test_dir, monkeypatch): -# exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) -# monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/{job._name}") -# yield exp - - @pytest.fixture def dispatcher(): """A pre-configured dispatcher to be used by experiments that simply From 2170243aa781cdf170298109dbf36b2fac16b63d Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 1 Aug 2024 10:55:02 -0500 Subject: [PATCH 47/82] notes after meeting with chris --- smartsim/settings/dispatch.py | 9 ++- tests/test_experiment.py | 2 +- tests/test_shell_launcher.py | 116 ++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 tests/test_shell_launcher.py diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 19351df54..4cc84ea83 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -30,6 +30,7 @@ import subprocess as sp import typing as t import uuid +import os from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack @@ -424,6 +425,7 @@ def impl( path: str, _env: _EnvironMappingType, ) -> t.Tuple[t.Sequence[str], str]: + print(exe) return ( ( run_command, @@ -446,12 +448,15 @@ def __init__(self) -> None: # TODO inject path here def start(self, command: tuple[t.Sequence[str], str]) -> LaunchedJobID: + print(command) id_ = create_job_id() args, path = command - exe, *rest = args + print(f"is dir: {os.path.isdir(path)}") + print(f"located: {os.getcwd()}") + exe, *rest = args # break out popen as much as possible for debugging # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path, env={}, stdin=None, stdout=None) - # env accepts a dictionary + # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ @classmethod diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 6c2378cb5..6c3f9f322 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -239,7 +239,7 @@ def test_start_can_launch_jobs( @pytest.mark.parametrize( "num_starts", - [pytest.param(i, id=f"{i} start(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)], + [pytest.param(i, id=f"{i} start(s)") for i in (1, 2,)], ) def test_start_can_start_a_job_multiple_times_accross_multiple_calls( experiment: Experiment, job_maker: JobMakerType, num_starts: int diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py new file mode 100644 index 000000000..791ccf811 --- /dev/null +++ b/tests/test_shell_launcher.py @@ -0,0 +1,116 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import tempfile +import unittest.mock +import pytest +import time +import weakref +from smartsim.entity import _mock, entity +from smartsim import Experiment +from smartsim.settings import LaunchSettings +from smartsim.settings.arguments.launch.slurm import ( + SlurmLaunchArguments, + _as_srun_command, +) +from smartsim.settings.dispatch import sp as dsp +from smartsim.settings.dispatch import ShellLauncher +from smartsim.settings.launchCommand import LauncherType +from smartsim.launchable import Job +# always start with unit tests, first test shell launcher init +# make sure to test passing invalid values to shell launcher, and correct values +# verify the simple assumptions +# give each test a doc string, add comments to isolate inline + +# how can I write a good test suite without being brittle - separating unit tests, group tests +# unit tests first, integration tests next, do not rely on external behav in tests +class EchoHelloWorldEntity(entity.SmartSimEntity): + """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" + + def __init__(self): + path = tempfile.TemporaryDirectory() + self._finalizer = weakref.finalize(self, path.cleanup) + super().__init__("test-entity", _mock.Mock()) + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return self.as_program_arguments() == other.as_program_arguments() + + def as_program_arguments(): + # return ("/usr/bin/echo", "Hello", "World!") + return ("/usr/bin/sleep", "10") + +# test that the process leading up to the shell launcher was corrected, integration test +# my test is identifying the change in the code +def test_this(test_dir: str, monkeypatch: pytest.MonkeyPatch): + # monkeypatch the popen + # create a Mock popen object + # def my_mock_popen(*args, **kwargs): + # print("foo") + # no longer care about the internals, only want to know that the process up to it was currect + mock2 = unittest.mock.MagicMock(return_value=0) # same as monkeypatch - implements getproperty or API that looks for a unknown prop on an obj + mock3 = unittest.mock.MagicMock() + mock3.Popen = mock2 + mock3.return_value = mock2 + mock2.assert_called() + mock2.assert_called_with() + env_vars = { + "LOGGING": "verbose", + } + slurm_settings = LaunchSettings(launcher=LauncherType.Slurm, env_vars=env_vars) + slurm_settings.launch_args.set_nodes(1) + job = Job(name="jobs", entity=EchoHelloWorldEntity, launch_settings=slurm_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + # can validate id here -> could build another mock that ensures that 22 is the pid + id = exp.start(job) + mock3.assert_called_with() # the process executed the correct launcher + # write something that makes sure the job has completed b4 the test exits + print(id) + #time.sleep(5) # TODO remove once blocking is added + # asyn = concurrent, not happening in another thread, not happening somewhere else + # focus on async io in python, make sure that anything that is io bound is async + +# what is the success criteria +def test_shell_as_py(capsys): + # a unit test should init the obj bc testing that unit of code + launcher = ShellLauncher() # should be testing the method level + # avoid rep + expected_output = "hello" + launcher.start((["echo", expected_output], "/tmp")) # use time.sleep(0.1) -> we do not need sleep in other places + captured = capsys.readouterr() + output = captured.out + assert expected_output in captured.out + # do not need to build exact str, but can just have multiple assert + # verify echo hello + # make a separate test for stdout and stdin -> that test only verifies one component + # tests should do as little as possible, reduce number of constraints + + +# popen returns a non 0 when an error occurs, so test invalid path +# assert not id, might retry -> assert called 5 times, -> could verify that a warning was printed + +# should test a success cond, a failure condition \ No newline at end of file From 871d5426c58b9bfdb1fff3cd2036ffef621700d5 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 1 Aug 2024 14:46:00 -0500 Subject: [PATCH 48/82] addressing half of matts comments --- smartsim/_core/generation/generator.py | 138 +++--- smartsim/entity/ensemble.py | 8 +- smartsim/entity/model.py | 1 - smartsim/experiment.py | 28 +- smartsim/launchable/job.py | 10 +- smartsim/launchable/jobGroup.py | 6 +- tests/_legacy/test_generator.py | 394 +++++++++--------- .../easy/marked/invalidtag.txt | 2 +- tests/{test_generator => }/test_generator.py | 32 +- 9 files changed, 298 insertions(+), 321 deletions(-) rename tests/{test_generator => }/test_generator.py (93%) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 445cb0160..ac4721a31 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -31,23 +31,19 @@ import shutil import typing as t from datetime import datetime -from distutils import dir_util # pylint: disable=deprecated-module from glob import glob from logging import DEBUG, INFO from os import mkdir, path, symlink from os.path import join, relpath -from pathlib import Path from tabulate import tabulate -from ...database import FeatureStore from ...entity import Application, TaggedFilesHierarchy from ...entity.files import EntityFiles -from ...launchable import Job, JobGroup +from ...launchable import Job from ...log import get_logger from ..entrypoints import file_operations from ..entrypoints.file_operations import get_parser -from ..utils.helpers import create_short_id_str logger = get_logger(__name__) logger.propagate = False @@ -59,7 +55,7 @@ class Generator: writing files into a Job directory. """ - def __init__(self, gen_path: str, run_ID: str, job: Job) -> None: + def __init__(self, exp_path: str, run_id: str) -> None: """Initialize a generator object The Generator class is responsible for creating Job directories. @@ -71,48 +67,11 @@ def __init__(self, gen_path: str, run_ID: str, job: Job) -> None: :param run_ID: The id of the Experiment :param job: Reference to a name, SmartSimEntity and LaunchSettings """ - self.job = job - # Generate the job folder path - self.path = self._generate_job_path(job, gen_path, run_ID) - # Generate the log folder path - self.log_path = self._generate_log_path(job, gen_path, run_ID) + self.exp_path = pathlib.Path(exp_path) + """The path under which the experiment operate""" + self.run_id = run_id + """The runID for Experiment.start""" - def _generate_log_path(self, job: Job, gen_path: str, run_ID: str) -> str: - """ - Generate the path for the log folder. - - :param gen_path: The base path job generation - :returns str: The generated path for the log directory - """ - job_type = f"{job.__class__.__name__.lower()}s" - path = os.path.join( - gen_path, - run_ID, - job_type, - job.name, - "log", - ) - return path - - def _generate_job_path(self, job: Job, gen_path: str, run_ID: str) -> str: - """ - Generates the directory path for a job based on its creation type - (whether created via ensemble or job init). - - :param job: The Job object - :param gen_path: The base path for job generation - :param run_ID: The experiments unique run ID - :returns str: The generated path for the job. - """ - job_type = f"{job.__class__.__name__.lower()}s" - path = os.path.join( - gen_path, - run_ID, - job_type, - job.name, - "run", - ) - return path @property def log_level(self) -> int: @@ -135,17 +94,16 @@ def log_level(self) -> int: else: return default_log_level - @property - def log_file(self) -> str: + def log_file(self, log_path: str) -> str: """Returns the location of the file summarizing the parameters used for the last generation of all generated entities. :returns: path to file with parameter settings """ - return join(self.path, "smartsim_params.txt") + return join(log_path, "smartsim_params.txt") - def generate_experiment(self) -> str: + def generate_job(self, job: Job) -> str: """Generate the directories Generate the file structure for a SmartSim experiment. This @@ -164,28 +122,63 @@ def generate_experiment(self) -> str: e.g. ``THERMO=;90;`` """ - # Create Job directory - pathlib.Path(self.path).mkdir(exist_ok=True, parents=True) - # Creat Job log directory - pathlib.Path(self.log_path).mkdir(exist_ok=True, parents=True) - - # The log_file only keeps track of the last generation - # this is to avoid gigantic files in case the user repeats - # generation several times. The information is anyhow - # redundant, as it is also written in each entity's dir - with open(self.log_file, mode="w", encoding="utf-8") as log_file: + job_path = self._generate_job_path(job) + log_path = self._generate_log_path(job) + + with open(self.log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - # Prevent access to type FeatureStore entities - if isinstance(self.job.entity, Application): - # Perform file system operations on attached files - self._build_operations() + # Perform file system operations on attached files + self._build_operations(job, job_path) # Return Job directory path - return self.path + return job_path + + + def _generate_job_path(self, job: Job) -> str: + """ + Generates the directory path for a job based on its creation type + (whether created via ensemble or job init). + + :param job: The Job object + :param gen_path: The base path for job generation + :param run_ID: The experiments unique run ID + :returns str: The generated path for the job. + """ + job_type = f"{job.__class__.__name__.lower()}s" + job_path = ( + self.exp_path / + self.run_id / + job_type / + job.name / + "run" + ) + # Create Job directory + job_path.mkdir(exist_ok=True, parents=True) + return job_path + + + def _generate_log_path(self, job: Job) -> str: + """ + Generate the path for the log folder. + + :param gen_path: The base path job generation + :returns str: The generated path for the log directory + """ + job_type = f"{job.__class__.__name__.lower()}s" + log_path = ( + self.exp_path / + self.run_id / + job_type / + job.name / + "log" + ) + log_path.mkdir(exist_ok=True, parents=True) + return log_path - def _build_operations(self) -> None: + + def _build_operations(self, job: Job, job_path: str) -> None: """This method generates file system operations based on the provided application. It processes three types of operations: to_copy, to_symlink, and to_configure. For each type, it calls the corresponding private methods and appends the results @@ -194,10 +187,11 @@ def _build_operations(self) -> None: :param app: The application for which operations are generated. :return: A list of lists containing file system operations. """ - app = t.cast(Application, self.job.entity) - self._get_symlink_file_system_operation(app, self.path) - self._write_tagged_entity_files(app, self.path) - self._get_copy_file_system_operation(app, self.path) + return + app = t.cast(Application, job.entity) + self._get_symlink_file_system_operation(app, job_path) + self._write_tagged_entity_files(app, job_path) + self._get_copy_file_system_operation(app, job_path) @staticmethod def _get_copy_file_system_operation(app: Application, dest: str) -> None: @@ -239,7 +233,6 @@ def _get_symlink_file_system_operation(app: Application, dest: str) -> None: ns = parser.parse_args(args) file_operations.symlink(ns) - # TODO update this to execute the file operations when entrypoint is merged in @staticmethod def _write_tagged_entity_files(app: Application, dest: str) -> None: """Read, configure and write the tagged input files for @@ -264,7 +257,6 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: """ for file in tagged.files: dst_path = path.join(dest, tagged.base, path.basename(file)) - print(dst_path) shutil.copyfile(file, dst_path) to_write.append(dst_path) diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index 9c95efce0..3835c7730 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -53,7 +53,6 @@ def __init__( exe: str | os.PathLike[str], exe_args: t.Sequence[str] | None = None, exe_arg_parameters: t.Mapping[str, t.Sequence[t.Sequence[str]]] | None = None, - path: str | os.PathLike[str] | None = None, files: EntityFiles | None = None, file_parameters: t.Mapping[str, t.Sequence[str]] | None = None, permutation_strategy: str | strategies.PermutationStrategyType = "all_perm", @@ -66,11 +65,6 @@ def __init__( self.exe_arg_parameters = ( copy.deepcopy(exe_arg_parameters) if exe_arg_parameters else {} ) - self.path = os.fspath(path) if path is not None else os.getcwd() - # ^^^^^^^^^^^ - # TODO: Copied from the original implementation, but I'm not sure that - # I like this default. Shouldn't it be something under an - # experiment directory? If so, how it injected?? self.files = copy.deepcopy(files) if files else EntityFiles() self.file_parameters = dict(file_parameters) if file_parameters else {} self.permutation_strategy = permutation_strategy @@ -110,4 +104,4 @@ def as_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: apps = self._create_applications() if not apps: raise ValueError("There are no members as part of this ensemble") - return tuple(Job(app, settings, app.name) for i, app in enumerate(apps, 1)) + return tuple(Job(app, settings, app.name) for app in enumerate(apps, 1)) diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index 045634b3f..1f54bf6e3 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -75,7 +75,6 @@ def __init__( :param exe_args: executable arguments :param params: application parameters for writing into configuration files or to be passed as command line arguments to executable. - :param path: path to output, error, and configuration files :param run_settings: launcher settings specified in the experiment :param params_as_args: list of parameters which have to be interpreted as command line arguments to diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 0fdf885f8..c70dd04d1 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -160,14 +160,6 @@ def __init__(self, name: str, exp_path: str | None = None): self.exp_path = exp_path """The path under which the experiment operate""" - self._run_ID = ( - "run-" - + datetime.datetime.now().strftime("%H-%M-%S") - + "-" - + datetime.datetime.now().strftime("%Y-%m-%d") - ) - """Create the run id for the Experiment""" - self._active_launchers: set[LauncherProtocol[t.Any]] = set() """The active launchers created, used, and reused by the experiment""" @@ -188,10 +180,12 @@ def start(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: jobs that can be used to query or alter the status of that particular execution of the job. """ - return self._dispatch(dispatch.DEFAULT_DISPATCHER, *jobs) + run_id = datetime.datetime.now().strftime("run-%H:%M:%ST%Y-%m-%d") + """Create the run id for Experiment.start""" + return self._dispatch(Generator(self.exp_path, run_id), dispatch.DEFAULT_DISPATCHER, *jobs) def _dispatch( - self, dispatcher: dispatch.Dispatcher, job: Job, *jobs: Job + self, generator: Generator, dispatcher: dispatch.Dispatcher, job: Job, *jobs: Job ) -> tuple[LaunchedJobID, ...]: """Dispatch a series of jobs with a particular dispatcher @@ -204,7 +198,7 @@ def _dispatch( particular dispatch of the job. """ - def execute_dispatch(job: Job) -> LaunchedJobID: + def execute_dispatch(generator: Generator, job: Job) -> LaunchedJobID: args = job.launch_settings.launch_args env = job.launch_settings.env_vars # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @@ -233,16 +227,13 @@ def execute_dispatch(job: Job) -> LaunchedJobID: # pylint: disable-next=protected-access self._active_launchers.add(launch_config._adapted_launcher) # Generate the Job directory and return generated path - job_execution_path = self._generate(job) + job_execution_path = self._generate(generator, job) return launch_config.start(exe, env, job_execution_path) - return execute_dispatch(job), *map(execute_dispatch, jobs) + return execute_dispatch(generator, job), *map(execute_dispatch, jobs) @_contextualize - def _generate( - self, - job: Job, - ) -> str: + def _generate(self, generator: Generator, job: Job) -> str: """Generate the file structure for a ``Job`` ``Experiment._generate`` creates directories for the job @@ -260,8 +251,7 @@ def _generate( :returns: a str path """ try: - generator = Generator(self.exp_path, self._run_ID, job) - job_path = generator.generate_experiment() + job_path = generator.generate_job(job) return job_path except SmartSimError as e: logger.error(e) diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index 79badd5ee..c2c8581b2 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -30,16 +30,12 @@ from copy import deepcopy from smartsim._core.commands.launchCommands import LaunchCommands -from smartsim._core.utils.helpers import create_short_id_str from smartsim.launchable.basejob import BaseJob from smartsim.settings import LaunchSettings if t.TYPE_CHECKING: from smartsim.entity.entity import SmartSimEntity -if t.TYPE_CHECKING: - from smartsim.entity.entity import SmartSimEntity - class Job(BaseJob): """A Job holds a reference to a SmartSimEntity and associated @@ -59,14 +55,12 @@ def __init__( super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) - self._name = deepcopy(name) if name else deepcopy(entity.name) - # TODO: self.warehouse_runner = JobWarehouseRunner + self._name = name if name else entity.name - # TODO do we want the user to be allowed to reset the Job name? Therefore, add setter @property def name(self) -> str: """Retrieves the name of the Job.""" - return deepcopy(self._name) + return self._name @property def entity(self) -> SmartSimEntity: diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index a8ef4440a..1a92caf54 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -19,16 +19,16 @@ class JobGroup(BaseJobGroup): def __init__( self, jobs: t.List[BaseJob], - name: str = "jobGroup", + name: str = "job_group", ) -> None: super().__init__() self._jobs = deepcopy(jobs) - self._name = deepcopy(name) + self._name = name @property def name(self) -> str: """Retrieves the name of the JobGroup.""" - return deepcopy(self._name) + return self._name @property def jobs(self) -> t.List[BaseJob]: diff --git a/tests/_legacy/test_generator.py b/tests/_legacy/test_generator.py index 0d65900cd..ece090a44 100644 --- a/tests/_legacy/test_generator.py +++ b/tests/_legacy/test_generator.py @@ -32,7 +32,7 @@ from smartsim import Experiment from smartsim._core.generation import Generator -from smartsim.database import FeatureStore +from smartsim.database import Orchestrator from smartsim.settings import RunSettings # The tests in this file belong to the group_a group @@ -74,78 +74,78 @@ def test_ensemble(fileutils, test_dir): assert osp.isdir(osp.join(test_dir, "test/test_" + str(i))) -# def test_ensemble_overwrite(fileutils, test_dir): -# exp = Experiment("gen-test-overwrite", launcher="local") +def test_ensemble_overwrite(fileutils, test_dir): + exp = Experiment("gen-test-overwrite", launcher="local") -# gen = Generator(test_dir, overwrite=True) + gen = Generator(test_dir, overwrite=True) -# params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} -# ensemble = exp.create_ensemble("test", params=params, run_settings=rs) + params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} + ensemble = exp.create_ensemble("test", params=params, run_settings=rs) -# config = get_gen_file(fileutils, "in.atm") -# ensemble.attach_generator_files(to_configure=[config]) -# gen.generate_experiment(ensemble) + config = get_gen_file(fileutils, "in.atm") + ensemble.attach_generator_files(to_configure=[config]) + gen.generate_experiment(ensemble) -# # re generate without overwrite -# config = get_gen_file(fileutils, "in.atm") -# ensemble.attach_generator_files(to_configure=[config]) -# gen.generate_experiment(ensemble) + # re generate without overwrite + config = get_gen_file(fileutils, "in.atm") + ensemble.attach_generator_files(to_configure=[config]) + gen.generate_experiment(ensemble) -# assert len(ensemble) == 9 -# assert osp.isdir(osp.join(test_dir, "test")) -# for i in range(9): -# assert osp.isdir(osp.join(test_dir, "test/test_" + str(i))) + assert len(ensemble) == 9 + assert osp.isdir(osp.join(test_dir, "test")) + for i in range(9): + assert osp.isdir(osp.join(test_dir, "test/test_" + str(i))) -# def test_ensemble_overwrite_error(fileutils, test_dir): -# exp = Experiment("gen-test-overwrite-error", launcher="local") +def test_ensemble_overwrite_error(fileutils, test_dir): + exp = Experiment("gen-test-overwrite-error", launcher="local") -# gen = Generator(test_dir) + gen = Generator(test_dir) -# params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} -# ensemble = exp.create_ensemble("test", params=params, run_settings=rs) + params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} + ensemble = exp.create_ensemble("test", params=params, run_settings=rs) -# config = get_gen_file(fileutils, "in.atm") -# ensemble.attach_generator_files(to_configure=[config]) -# gen.generate_experiment(ensemble) + config = get_gen_file(fileutils, "in.atm") + ensemble.attach_generator_files(to_configure=[config]) + gen.generate_experiment(ensemble) -# # re generate without overwrite -# config = get_gen_file(fileutils, "in.atm") -# ensemble.attach_generator_files(to_configure=[config]) -# with pytest.raises(FileExistsError): -# gen.generate_experiment(ensemble) + # re generate without overwrite + config = get_gen_file(fileutils, "in.atm") + ensemble.attach_generator_files(to_configure=[config]) + with pytest.raises(FileExistsError): + gen.generate_experiment(ensemble) def test_full_exp(fileutils, test_dir, wlmutils): exp = Experiment("gen-test", test_dir, launcher="local") - application = exp.create_application("application", run_settings=rs) + model = exp.create_model("model", run_settings=rs) script = fileutils.get_test_conf_path("sleep.py") - application.attach_generator_files(to_copy=script) + model.attach_generator_files(to_copy=script) - feature_store = FeatureStore(wlmutils.get_test_port()) + orc = Orchestrator(wlmutils.get_test_port()) params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} ensemble = exp.create_ensemble("test_ens", params=params, run_settings=rs) config = get_gen_file(fileutils, "in.atm") ensemble.attach_generator_files(to_configure=config) - exp.generate(feature_store, ensemble, application) + exp.generate(orc, ensemble, model) # test for ensemble assert osp.isdir(osp.join(test_dir, "test_ens/")) for i in range(9): assert osp.isdir(osp.join(test_dir, "test_ens/test_ens_" + str(i))) - # test for feature_store dir - assert osp.isdir(osp.join(test_dir, feature_store.name)) + # test for orc dir + assert osp.isdir(osp.join(test_dir, orc.name)) - # test for application file - assert osp.isdir(osp.join(test_dir, "application")) - assert osp.isfile(osp.join(test_dir, "application/sleep.py")) + # test for model file + assert osp.isdir(osp.join(test_dir, "model")) + assert osp.isfile(osp.join(test_dir, "model/sleep.py")) def test_dir_files(fileutils, test_dir): - """test the generate of applications with files that + """test the generate of models with files that are directories with subdirectories and files """ @@ -160,152 +160,150 @@ def test_dir_files(fileutils, test_dir): assert osp.isdir(osp.join(test_dir, "dir_test/")) for i in range(9): - application_path = osp.join(test_dir, "dir_test/dir_test_" + str(i)) - assert osp.isdir(application_path) - assert osp.isdir(osp.join(application_path, "test_dir_1")) - assert osp.isfile(osp.join(application_path, "test.in")) - - -# def test_print_files(fileutils, test_dir, capsys): -# """Test the stdout print of files attached to an ensemble""" - -# exp = Experiment("print-attached-files-test", test_dir, launcher="local") - -# ensemble = exp.create_ensemble("dir_test", replicas=1, run_settings=rs) -# ensemble.entities = [] - -# ensemble.print_attached_files() -# captured = capsys.readouterr() -# assert captured.out == "The ensemble is empty, no files to show.\n" - -# params = {"THERMO": [10, 20], "STEPS": [20, 30]} -# ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) -# gen_dir = get_gen_file(fileutils, "test_dir") -# symlink_dir = get_gen_file(fileutils, "to_symlink_dir") -# copy_dir = get_gen_file(fileutils, "to_copy_dir") - -# ensemble.print_attached_files() -# captured = capsys.readouterr() -# expected_out = ( -# tabulate( -# [ -# [application.name, "No file attached to this application."] -# for application in ensemble.applications -# ], -# headers=["Application name", "Files"], -# tablefmt="grid", -# ) -# + "\n" -# ) - -# assert captured.out == expected_out - -# ensemble.attach_generator_files() -# ensemble.print_attached_files() -# captured = capsys.readouterr() -# expected_out = ( -# tabulate( -# [ -# [application.name, "No file attached to this entity."] -# for application in ensemble.applications -# ], -# headers=["Application name", "Files"], -# tablefmt="grid", -# ) -# + "\n" -# ) -# assert captured.out == expected_out - -# ensemble.attach_generator_files( -# to_configure=[gen_dir, copy_dir], to_copy=copy_dir, to_symlink=symlink_dir -# ) - -# expected_out = tabulate( -# [ -# ["Copy", copy_dir], -# ["Symlink", symlink_dir], -# ["Configure", f"{gen_dir}\n{copy_dir}"], -# ], -# headers=["Strategy", "Files"], -# tablefmt="grid", -# ) - -# assert all( -# str(application.files) == expected_out for application in ensemble.applications -# ) - -# expected_out_multi = ( -# tabulate( -# [[application.name, expected_out] for application in ensemble.applications], -# headers=["Application name", "Files"], -# tablefmt="grid", -# ) -# + "\n" -# ) -# ensemble.print_attached_files() - -# captured = capsys.readouterr() -# assert captured.out == expected_out_multi - - -# def test_multiple_tags(fileutils, test_dir): -# """Test substitution of multiple tagged parameters on same line""" - -# exp = Experiment("test-multiple-tags", test_dir) -# application_params = {"port": 6379, "password": "unbreakable_password"} -# application_settings = RunSettings("bash", "multi_tags_template.sh") -# parameterized_application = exp.create_application( -# "multi-tags", run_settings=application_settings, params=application_params -# ) -# config = get_gen_file(fileutils, "multi_tags_template.sh") -# parameterized_application.attach_generator_files(to_configure=[config]) -# exp.generate(parameterized_application, overwrite=True) -# exp.start(parameterized_application, block=True) - -# with open(osp.join(parameterized_application.path, "multi-tags.out")) as f: -# log_content = f.read() -# assert "My two parameters are 6379 and unbreakable_password, OK?" in log_content - - -# def test_generation_log(fileutils, test_dir): -# """Test that an error is issued when a tag is unused and make_fatal is True""" - -# exp = Experiment("gen-log-test", test_dir, launcher="local") - -# params = {"THERMO": [10, 20], "STEPS": [10, 20]} -# ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) -# conf_file = get_gen_file(fileutils, "in.atm") -# ensemble.attach_generator_files(to_configure=conf_file) - -# def not_header(line): -# """you can add other general checks in here""" -# return not line.startswith("Generation start date and time:") - -# exp.generate(ensemble, verbose=True) - -# log_file = osp.join(test_dir, "smartsim_params.txt") -# ground_truth = get_gen_file( -# fileutils, osp.join("log_params", "smartsim_params.txt") -# ) - -# with open(log_file) as f1, open(ground_truth) as f2: -# assert not not_header(f1.readline()) -# f1 = filter(not_header, f1) -# f2 = filter(not_header, f2) -# assert all(x == y for x, y in zip(f1, f2)) - -# for entity in ensemble: -# assert filecmp.cmp( -# osp.join(entity.path, "smartsim_params.txt"), -# get_gen_file( -# fileutils, -# osp.join("log_params", "dir_test", entity.name, "smartsim_params.txt"), -# ), -# ) + model_path = osp.join(test_dir, "dir_test/dir_test_" + str(i)) + assert osp.isdir(model_path) + assert osp.isdir(osp.join(model_path, "test_dir_1")) + assert osp.isfile(osp.join(model_path, "test.in")) + + +def test_print_files(fileutils, test_dir, capsys): + """Test the stdout print of files attached to an ensemble""" + + exp = Experiment("print-attached-files-test", test_dir, launcher="local") + + ensemble = exp.create_ensemble("dir_test", replicas=1, run_settings=rs) + ensemble.entities = [] + + ensemble.print_attached_files() + captured = capsys.readouterr() + assert captured.out == "The ensemble is empty, no files to show.\n" + + params = {"THERMO": [10, 20], "STEPS": [20, 30]} + ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) + gen_dir = get_gen_file(fileutils, "test_dir") + symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + copy_dir = get_gen_file(fileutils, "to_copy_dir") + + ensemble.print_attached_files() + captured = capsys.readouterr() + expected_out = ( + tabulate( + [ + [model.name, "No file attached to this model."] + for model in ensemble.models + ], + headers=["Model name", "Files"], + tablefmt="grid", + ) + + "\n" + ) + + assert captured.out == expected_out + + ensemble.attach_generator_files() + ensemble.print_attached_files() + captured = capsys.readouterr() + expected_out = ( + tabulate( + [ + [model.name, "No file attached to this entity."] + for model in ensemble.models + ], + headers=["Model name", "Files"], + tablefmt="grid", + ) + + "\n" + ) + assert captured.out == expected_out + + ensemble.attach_generator_files( + to_configure=[gen_dir, copy_dir], to_copy=copy_dir, to_symlink=symlink_dir + ) + + expected_out = tabulate( + [ + ["Copy", copy_dir], + ["Symlink", symlink_dir], + ["Configure", f"{gen_dir}\n{copy_dir}"], + ], + headers=["Strategy", "Files"], + tablefmt="grid", + ) + + assert all(str(model.files) == expected_out for model in ensemble.models) + + expected_out_multi = ( + tabulate( + [[model.name, expected_out] for model in ensemble.models], + headers=["Model name", "Files"], + tablefmt="grid", + ) + + "\n" + ) + ensemble.print_attached_files() + + captured = capsys.readouterr() + assert captured.out == expected_out_multi + + +def test_multiple_tags(fileutils, test_dir): + """Test substitution of multiple tagged parameters on same line""" + + exp = Experiment("test-multiple-tags", test_dir) + model_params = {"port": 6379, "password": "unbreakable_password"} + model_settings = RunSettings("bash", "multi_tags_template.sh") + parameterized_model = exp.create_model( + "multi-tags", run_settings=model_settings, params=model_params + ) + config = get_gen_file(fileutils, "multi_tags_template.sh") + parameterized_model.attach_generator_files(to_configure=[config]) + exp.generate(parameterized_model, overwrite=True) + exp.start(parameterized_model, block=True) + + with open(osp.join(parameterized_model.path, "multi-tags.out")) as f: + log_content = f.read() + assert "My two parameters are 6379 and unbreakable_password, OK?" in log_content + + +def test_generation_log(fileutils, test_dir): + """Test that an error is issued when a tag is unused and make_fatal is True""" + + exp = Experiment("gen-log-test", test_dir, launcher="local") + + params = {"THERMO": [10, 20], "STEPS": [10, 20]} + ensemble = exp.create_ensemble("dir_test", params=params, run_settings=rs) + conf_file = get_gen_file(fileutils, "in.atm") + ensemble.attach_generator_files(to_configure=conf_file) + + def not_header(line): + """you can add other general checks in here""" + return not line.startswith("Generation start date and time:") + + exp.generate(ensemble, verbose=True) + + log_file = osp.join(test_dir, "smartsim_params.txt") + ground_truth = get_gen_file( + fileutils, osp.join("log_params", "smartsim_params.txt") + ) + + with open(log_file) as f1, open(ground_truth) as f2: + assert not not_header(f1.readline()) + f1 = filter(not_header, f1) + f2 = filter(not_header, f2) + assert all(x == y for x, y in zip(f1, f2)) + + for entity in ensemble: + assert filecmp.cmp( + osp.join(entity.path, "smartsim_params.txt"), + get_gen_file( + fileutils, + osp.join("log_params", "dir_test", entity.name, "smartsim_params.txt"), + ), + ) def test_config_dir(fileutils, test_dir): - """Test the generation and configuration of applications with + """Test the generation and configuration of models with tagged files that are directories with subdirectories and files """ exp = Experiment("config-dir", launcher="local") @@ -364,18 +362,18 @@ def test_no_gen_if_symlink_to_dir(fileutils): ensemble.attach_generator_files(to_configure=config) -# def test_no_file_overwrite(): -# exp = Experiment("test_no_file_overwrite", launcher="local") -# ensemble = exp.create_ensemble("test", params={"P": [0, 1]}, run_settings=rs) -# with pytest.raises(ValueError): -# ensemble.attach_generator_files( -# to_configure=["/normal/file.txt", "/path/to/smartsim_params.txt"] -# ) -# with pytest.raises(ValueError): -# ensemble.attach_generator_files( -# to_symlink=["/normal/file.txt", "/path/to/smartsim_params.txt"] -# ) -# with pytest.raises(ValueError): -# ensemble.attach_generator_files( -# to_copy=["/normal/file.txt", "/path/to/smartsim_params.txt"] -# ) +def test_no_file_overwrite(): + exp = Experiment("test_no_file_overwrite", launcher="local") + ensemble = exp.create_ensemble("test", params={"P": [0, 1]}, run_settings=rs) + with pytest.raises(ValueError): + ensemble.attach_generator_files( + to_configure=["/normal/file.txt", "/path/to/smartsim_params.txt"] + ) + with pytest.raises(ValueError): + ensemble.attach_generator_files( + to_symlink=["/normal/file.txt", "/path/to/smartsim_params.txt"] + ) + with pytest.raises(ValueError): + ensemble.attach_generator_files( + to_copy=["/normal/file.txt", "/path/to/smartsim_params.txt"] + ) \ No newline at end of file diff --git a/tests/test_configs/generator_files/easy/marked/invalidtag.txt b/tests/test_configs/generator_files/easy/marked/invalidtag.txt index 2165ae8d1..90a625319 100644 --- a/tests/test_configs/generator_files/easy/marked/invalidtag.txt +++ b/tests/test_configs/generator_files/easy/marked/invalidtag.txt @@ -1,3 +1,3 @@ some text before -some params are valid and others are ;INVALID; but we mostly encounter valid params +some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params some text after diff --git a/tests/test_generator/test_generator.py b/tests/test_generator.py similarity index 93% rename from tests/test_generator/test_generator.py rename to tests/test_generator.py index 9dcb4b682..974808577 100644 --- a/tests/test_generator/test_generator.py +++ b/tests/test_generator.py @@ -14,8 +14,8 @@ from smartsim.launchable import Job from smartsim.settings.launchSettings import LaunchSettings -# TODO test ensemble copy, config, symlink when ensemble.attach_generator_files added -# TODO remove ensemble tests and replace with JobGroup when start jobgroup is supported +# TODO Test ensemble copy, config, symlink when Ensemble.attach_generator_files added +# TODO Add JobGroup tests when JobGroup becomes a Launchable pytestmark = pytest.mark.group_a @@ -80,33 +80,38 @@ def test_log_file_path(generator_instance): def test_generate_job_directory(test_dir, wlmutils): - """Test that Job directory was created.""" + """Test Generator.generate_job""" + # Experiment path experiment_path = osp.join(test_dir, "experiment_name") + # Create Job launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("app_name", exe="python", run_settings="RunSettings") job = Job(app, launch_settings) - run_ID = "mock_run" - gen = Generator(gen_path=experiment_path, run_ID=run_ID, job=job) - gen.generate_experiment() + # Mock start id + run_id = "mock_run" + # Generator instance + gen = Generator(exp_path=experiment_path, run_id=run_id) + # Call Generator.generate_job + job_path = gen.generate_job(job) + assert isinstance(job_path, pathlib.Path) expected_run_path = ( pathlib.Path(experiment_path) - / run_ID + / run_id / f"{job.__class__.__name__.lower()}s" / app.name / "run" ) + assert job_path == expected_run_path expected_log_path = ( pathlib.Path(experiment_path) - / run_ID + / run_id / f"{job.__class__.__name__.lower()}s" / app.name / "log" ) - assert gen.path == str(expected_run_path) - assert gen.log_path == str(expected_log_path) assert osp.isdir(expected_run_path) assert osp.isdir(expected_log_path) - assert osp.isfile(osp.join(expected_run_path, "smartsim_params.txt")) + assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) def test_exp_private_generate_method_app(test_dir, job_instance): @@ -267,3 +272,8 @@ def test_generate_ensemble_directory(test_dir, wlmutils): gen.generate_experiment() assert osp.isdir(gen.path) assert osp.isdir(pathlib.Path(gen.log_path)) + + +def test_dummy(test_dir, job_instance): + exp = Experiment(name="exp-name", exp_path=test_dir) + exp.start(job_instance) From c8246c717c939a218df0d49cf95a8cd2dafe1a93 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 1 Aug 2024 14:53:15 -0500 Subject: [PATCH 49/82] remove edits to the legacy gen test --- tests/_legacy/test_generator.py | 68 +++++++++++++++++---------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/tests/_legacy/test_generator.py b/tests/_legacy/test_generator.py index ece090a44..821c6e8ed 100644 --- a/tests/_legacy/test_generator.py +++ b/tests/_legacy/test_generator.py @@ -32,7 +32,7 @@ from smartsim import Experiment from smartsim._core.generation import Generator -from smartsim.database import Orchestrator +from smartsim.database import FeatureStore from smartsim.settings import RunSettings # The tests in this file belong to the group_a group @@ -119,33 +119,33 @@ def test_ensemble_overwrite_error(fileutils, test_dir): def test_full_exp(fileutils, test_dir, wlmutils): exp = Experiment("gen-test", test_dir, launcher="local") - model = exp.create_model("model", run_settings=rs) + application = exp.create_application("application", run_settings=rs) script = fileutils.get_test_conf_path("sleep.py") - model.attach_generator_files(to_copy=script) + application.attach_generator_files(to_copy=script) - orc = Orchestrator(wlmutils.get_test_port()) + feature_store = FeatureStore(wlmutils.get_test_port()) params = {"THERMO": [10, 20, 30], "STEPS": [10, 20, 30]} ensemble = exp.create_ensemble("test_ens", params=params, run_settings=rs) config = get_gen_file(fileutils, "in.atm") ensemble.attach_generator_files(to_configure=config) - exp.generate(orc, ensemble, model) + exp.generate(feature_store, ensemble, application) # test for ensemble assert osp.isdir(osp.join(test_dir, "test_ens/")) for i in range(9): assert osp.isdir(osp.join(test_dir, "test_ens/test_ens_" + str(i))) - # test for orc dir - assert osp.isdir(osp.join(test_dir, orc.name)) + # test for feature_store dir + assert osp.isdir(osp.join(test_dir, feature_store.name)) - # test for model file - assert osp.isdir(osp.join(test_dir, "model")) - assert osp.isfile(osp.join(test_dir, "model/sleep.py")) + # test for application file + assert osp.isdir(osp.join(test_dir, "application")) + assert osp.isfile(osp.join(test_dir, "application/sleep.py")) def test_dir_files(fileutils, test_dir): - """test the generate of models with files that + """test the generate of applications with files that are directories with subdirectories and files """ @@ -160,10 +160,10 @@ def test_dir_files(fileutils, test_dir): assert osp.isdir(osp.join(test_dir, "dir_test/")) for i in range(9): - model_path = osp.join(test_dir, "dir_test/dir_test_" + str(i)) - assert osp.isdir(model_path) - assert osp.isdir(osp.join(model_path, "test_dir_1")) - assert osp.isfile(osp.join(model_path, "test.in")) + application_path = osp.join(test_dir, "dir_test/dir_test_" + str(i)) + assert osp.isdir(application_path) + assert osp.isdir(osp.join(application_path, "test_dir_1")) + assert osp.isfile(osp.join(application_path, "test.in")) def test_print_files(fileutils, test_dir, capsys): @@ -189,10 +189,10 @@ def test_print_files(fileutils, test_dir, capsys): expected_out = ( tabulate( [ - [model.name, "No file attached to this model."] - for model in ensemble.models + [application.name, "No file attached to this application."] + for application in ensemble.applications ], - headers=["Model name", "Files"], + headers=["Application name", "Files"], tablefmt="grid", ) + "\n" @@ -206,10 +206,10 @@ def test_print_files(fileutils, test_dir, capsys): expected_out = ( tabulate( [ - [model.name, "No file attached to this entity."] - for model in ensemble.models + [application.name, "No file attached to this entity."] + for application in ensemble.applications ], - headers=["Model name", "Files"], + headers=["Application name", "Files"], tablefmt="grid", ) + "\n" @@ -230,12 +230,14 @@ def test_print_files(fileutils, test_dir, capsys): tablefmt="grid", ) - assert all(str(model.files) == expected_out for model in ensemble.models) + assert all( + str(application.files) == expected_out for application in ensemble.applications + ) expected_out_multi = ( tabulate( - [[model.name, expected_out] for model in ensemble.models], - headers=["Model name", "Files"], + [[application.name, expected_out] for application in ensemble.applications], + headers=["Application name", "Files"], tablefmt="grid", ) + "\n" @@ -250,17 +252,17 @@ def test_multiple_tags(fileutils, test_dir): """Test substitution of multiple tagged parameters on same line""" exp = Experiment("test-multiple-tags", test_dir) - model_params = {"port": 6379, "password": "unbreakable_password"} - model_settings = RunSettings("bash", "multi_tags_template.sh") - parameterized_model = exp.create_model( - "multi-tags", run_settings=model_settings, params=model_params + application_params = {"port": 6379, "password": "unbreakable_password"} + application_settings = RunSettings("bash", "multi_tags_template.sh") + parameterized_application = exp.create_application( + "multi-tags", run_settings=application_settings, params=application_params ) config = get_gen_file(fileutils, "multi_tags_template.sh") - parameterized_model.attach_generator_files(to_configure=[config]) - exp.generate(parameterized_model, overwrite=True) - exp.start(parameterized_model, block=True) + parameterized_application.attach_generator_files(to_configure=[config]) + exp.generate(parameterized_application, overwrite=True) + exp.start(parameterized_application, block=True) - with open(osp.join(parameterized_model.path, "multi-tags.out")) as f: + with open(osp.join(parameterized_application.path, "multi-tags.out")) as f: log_content = f.read() assert "My two parameters are 6379 and unbreakable_password, OK?" in log_content @@ -303,7 +305,7 @@ def not_header(line): def test_config_dir(fileutils, test_dir): - """Test the generation and configuration of models with + """Test the generation and configuration of applications with tagged files that are directories with subdirectories and files """ exp = Experiment("config-dir", launcher="local") From 0baddf2d52292367103580b31dbc4d5acb0501e1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 1 Aug 2024 14:53:50 -0500 Subject: [PATCH 50/82] adding new line to legacy gen test --- tests/_legacy/test_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_legacy/test_generator.py b/tests/_legacy/test_generator.py index 821c6e8ed..c3bfcad64 100644 --- a/tests/_legacy/test_generator.py +++ b/tests/_legacy/test_generator.py @@ -378,4 +378,4 @@ def test_no_file_overwrite(): with pytest.raises(ValueError): ensemble.attach_generator_files( to_copy=["/normal/file.txt", "/path/to/smartsim_params.txt"] - ) \ No newline at end of file + ) From 087476713e8bfada6582b9e48dd0716bea10e6bd Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 1 Aug 2024 16:29:12 -0500 Subject: [PATCH 51/82] subprocess added --- smartsim/_core/generation/generator.py | 34 ++++++++--------- tests/test_generator.py | 52 +++++++++++++------------- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index ac4721a31..9c7ccb290 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -35,6 +35,9 @@ from logging import DEBUG, INFO from os import mkdir, path, symlink from os.path import join, relpath +import subprocess +import sys + from tabulate import tabulate @@ -187,7 +190,6 @@ def _build_operations(self, job: Job, job_path: str) -> None: :param app: The application for which operations are generated. :return: A list of lists containing file system operations. """ - return app = t.cast(Application, job.entity) self._get_symlink_file_system_operation(app, job_path) self._write_tagged_entity_files(app, job_path) @@ -202,15 +204,11 @@ def _get_copy_file_system_operation(app: Application, dest: str) -> None: """ if app.files is None: return - parser = get_parser() for src in app.files.copy: if os.path.isdir(src): - cmd = f"copy {src} {dest} --dirs_exist_ok" + subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "copy", src, dest, "--dirs_exist_ok"]) else: - cmd = f"copy {src} {dest}" - args = cmd.split() - ns = parser.parse_args(args) - file_operations.copy(ns) + subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "copy", src, dest]) @staticmethod def _get_symlink_file_system_operation(app: Application, dest: str) -> None: @@ -222,16 +220,13 @@ def _get_symlink_file_system_operation(app: Application, dest: str) -> None: if app.files is None: return parser = get_parser() - for sym in app.files.link: - # Normalize the path to remove trailing slashes - normalized_path = os.path.normpath(sym) - # Get the parent directory (last folder) + for src in app.files.link: + # # Normalize the path to remove trailing slashes + normalized_path = os.path.normpath(src) + # # Get the parent directory (last folder) parent_dir = os.path.basename(normalized_path) dest = os.path.join(dest, parent_dir) - cmd = f"symlink {sym} {dest}" - args = cmd.split() - ns = parser.parse_args(args) - file_operations.symlink(ns) + subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "symlink", src, dest]) @staticmethod def _write_tagged_entity_files(app: Application, dest: str) -> None: @@ -275,10 +270,11 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: encoded_dict = base64.b64encode(pickled_dict).decode("ascii") parser = get_parser() for dest_path in to_write: - cmd = f"configure {dest_path} {dest_path} {tag} {encoded_dict}" - args = cmd.split() - ns = parser.parse_args(args) - file_operations.configure(ns) + subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "configure", dest_path, dest_path, tag, encoded_dict]) + # cmd = f"configure {dest_path} {dest_path} {tag} {encoded_dict}" + # args = cmd.split() + # ns = parser.parse_args(args) + # file_operations.configure(ns) # TODO address in ticket 723 # self._log_params(entity, files_to_params) diff --git a/tests/test_generator.py b/tests/test_generator.py index 974808577..c68220729 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -114,15 +114,15 @@ def test_generate_job_directory(test_dir, wlmutils): assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) -def test_exp_private_generate_method_app(test_dir, job_instance): - """Test that Job directory was created from Experiment.""" - no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) - job_execution_path = no_op_exp._generate(job_instance) - assert osp.isdir(job_execution_path) - head, _ = os.path.split(job_execution_path) - expected_log_path = pathlib.Path(head) / "log" - assert osp.isdir(expected_log_path) - assert osp.isfile(osp.join(job_execution_path, "smartsim_params.txt")) +# def test_exp_private_generate_method_app(test_dir, job_instance): +# """Test that Job directory was created from Experiment.""" +# no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) +# job_execution_path = no_op_exp._generate(job_instance) +# assert osp.isdir(job_execution_path) +# head, _ = os.path.split(job_execution_path) +# expected_log_path = pathlib.Path(head) / "log" +# assert osp.isdir(expected_log_path) +# assert osp.isfile(osp.join(job_execution_path, "smartsim_params.txt")) def test_generate_copy_file(fileutils, wlmutils, test_dir): @@ -135,8 +135,8 @@ def test_generate_copy_file(fileutils, wlmutils, test_dir): # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) - path = gen.generate_experiment() + gen = Generator(exp_path=experiment_path, run_id="temp_run") + path = gen.generate_job(job) expected_file = pathlib.Path(path) / "sleep.py" assert osp.isfile(expected_file) @@ -151,9 +151,9 @@ def test_generate_copy_directory(fileutils, wlmutils, test_dir): # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) - gen.generate_experiment() - expected_file = pathlib.Path(gen.path) / "mock.txt" + gen = Generator(exp_path=experiment_path, run_id="temp_run") + path = gen.generate_job(job) + expected_file = pathlib.Path(path) / "mock.txt" assert osp.isfile(expected_file) @@ -170,10 +170,10 @@ def test_generate_symlink_directory(fileutils, wlmutils, test_dir): # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) + gen = Generator(exp_path=experiment_path, run_id="temp_run") # Generate Experiment file structure - gen.generate_experiment() - expected_folder = pathlib.Path(gen.path) / "to_symlink_dir" + job_path = gen.generate_job(job) + expected_folder = pathlib.Path(job_path) / "to_symlink_dir" assert osp.isdir(expected_folder) # Combine symlinked file list and original file list for comparison for written, correct in zip(listdir(symlink_dir), listdir(expected_folder)): @@ -195,10 +195,10 @@ def test_generate_symlink_file(fileutils, wlmutils, test_dir): job = Job(app, launch_settings) # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(gen_path=experiment_path, run_ID="test", job=job) + gen = Generator(exp_path=experiment_path, run_id="mock_run") # Generate Experiment file structure - gen.generate_experiment() - expected_file = pathlib.Path(gen.path) / "mock2.txt" + job_path = gen.generate_job(job) + expected_file = pathlib.Path(job_path) / "mock2.txt" assert osp.isfile(expected_file) @@ -233,19 +233,19 @@ def test_generate_configure(fileutils, wlmutils, test_dir): # Spin up Experiment experiment_path = osp.join(test_dir, "experiment_name") # Spin up Generator - gen = Generator(gen_path=experiment_path, run_ID="temp_run", job=job) + gen = Generator(exp_path=experiment_path, run_id="temp_run") # Execute file generation - job_path = gen.generate_experiment() + job_path = gen.generate_job(job) # Retrieve the list of configured files in the test directory - configured_files = sorted(glob(job_path + "/*")) + configured_files = sorted(glob(str(job_path) + "/*")) # Use filecmp.cmp to check that the corresponding files are equal for written, correct in zip(configured_files, correct_files): assert filecmp.cmp(written, correct) # Validate that log file exists - assert osp.isdir(gen.log_path) + # assert osp.isdir() # Validate that smartsim params files exists - smartsim_params_path = osp.join(job_path, "smartsim_params.txt") - assert osp.isfile(smartsim_params_path) + # smartsim_params_path = osp.join(job_path, "smartsim_params.txt") + # assert osp.isfile(smartsim_params_path) def test_exp_private_generate_method_ensemble(test_dir, wlmutils): From 12cedef798f3c7117b86785d7a04435d3b636b90 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 1 Aug 2024 22:50:46 -0500 Subject: [PATCH 52/82] additional matt comments --- smartsim/_core/generation/generator.py | 46 +++++++++++--------------- tests/test_generator.py | 11 +++--- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 9c7ccb290..dfb486c92 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -33,20 +33,14 @@ from datetime import datetime from glob import glob from logging import DEBUG, INFO -from os import mkdir, path, symlink +from os import mkdir, path from os.path import join, relpath import subprocess import sys - -from tabulate import tabulate - from ...entity import Application, TaggedFilesHierarchy -from ...entity.files import EntityFiles from ...launchable import Job from ...log import get_logger -from ..entrypoints import file_operations -from ..entrypoints.file_operations import get_parser logger = get_logger(__name__) logger.propagate = False @@ -97,7 +91,7 @@ def log_level(self) -> int: else: return default_log_level - def log_file(self, log_path: str) -> str: + def log_file(self, log_path: pathlib.Path) -> str: """Returns the location of the file summarizing the parameters used for the last generation of all generated entities. @@ -106,7 +100,7 @@ def log_file(self, log_path: str) -> str: """ return join(log_path, "smartsim_params.txt") - def generate_job(self, job: Job) -> str: + def generate_job(self, job: Job) -> pathlib.Path: """Generate the directories Generate the file structure for a SmartSim experiment. This @@ -125,28 +119,28 @@ def generate_job(self, job: Job) -> str: e.g. ``THERMO=;90;`` """ + # Generate ../job_name/run directory job_path = self._generate_job_path(job) + # Generate ../job_name/log directory log_path = self._generate_log_path(job) with open(self.log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - # Perform file system operations on attached files + # Perform file system operations self._build_operations(job, job_path) - # Return Job directory path + # Return job path return job_path - def _generate_job_path(self, job: Job) -> str: + def _generate_job_path(self, job: Job) -> pathlib.Path: """ Generates the directory path for a job based on its creation type (whether created via ensemble or job init). :param job: The Job object - :param gen_path: The base path for job generation - :param run_ID: The experiments unique run ID :returns str: The generated path for the job. """ job_type = f"{job.__class__.__name__.lower()}s" @@ -162,11 +156,11 @@ def _generate_job_path(self, job: Job) -> str: return job_path - def _generate_log_path(self, job: Job) -> str: + def _generate_log_path(self, job: Job) -> pathlib.Path: """ Generate the path for the log folder. - :param gen_path: The base path job generation + :param job: The Job object :returns str: The generated path for the log directory """ job_type = f"{job.__class__.__name__.lower()}s" @@ -181,7 +175,7 @@ def _generate_log_path(self, job: Job) -> str: return log_path - def _build_operations(self, job: Job, job_path: str) -> None: + def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: """This method generates file system operations based on the provided application. It processes three types of operations: to_copy, to_symlink, and to_configure. For each type, it calls the corresponding private methods and appends the results @@ -191,12 +185,12 @@ def _build_operations(self, job: Job, job_path: str) -> None: :return: A list of lists containing file system operations. """ app = t.cast(Application, job.entity) - self._get_symlink_file_system_operation(app, job_path) - self._write_tagged_entity_files(app, job_path) - self._get_copy_file_system_operation(app, job_path) + self._symlink_files(job.entity, job_path) + self._write_tagged_files(job.entity, job_path) + self._copy_files(job.entity, job_path) @staticmethod - def _get_copy_file_system_operation(app: Application, dest: str) -> None: + def _copy_files(app: Application, dest: pathlib.Path) -> None: """Get copy file system operation for a file. :param app: The Application attached to the Job @@ -211,7 +205,7 @@ def _get_copy_file_system_operation(app: Application, dest: str) -> None: subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "copy", src, dest]) @staticmethod - def _get_symlink_file_system_operation(app: Application, dest: str) -> None: + def _symlink_files(app: Application, dest: pathlib.Path) -> None: """Get symlink file system operation for a file. :param app: The Application attached to the Job @@ -219,17 +213,16 @@ def _get_symlink_file_system_operation(app: Application, dest: str) -> None: """ if app.files is None: return - parser = get_parser() for src in app.files.link: # # Normalize the path to remove trailing slashes normalized_path = os.path.normpath(src) # # Get the parent directory (last folder) parent_dir = os.path.basename(normalized_path) - dest = os.path.join(dest, parent_dir) - subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "symlink", src, dest]) + new_dest = os.path.join(str(dest), parent_dir) + subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "symlink", src, new_dest]) @staticmethod - def _write_tagged_entity_files(app: Application, dest: str) -> None: + def _write_tagged_files(app: Application, dest: pathlib.Path) -> None: """Read, configure and write the tagged input files for a Application instance within an ensemble. This function specifically deals with the tagged files attached to @@ -268,7 +261,6 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: tag = ";" # Encode the pickled dictionary with Base64 encoded_dict = base64.b64encode(pickled_dict).decode("ascii") - parser = get_parser() for dest_path in to_write: subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "configure", dest_path, dest_path, tag, encoded_dict]) # cmd = f"configure {dest_path} {dest_path} {tag} {encoded_dict}" diff --git a/tests/test_generator.py b/tests/test_generator.py index c68220729..a10cf3504 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -49,7 +49,7 @@ def generator_instance(test_dir, wlmutils) -> Generator: launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("app_name", exe="python", run_settings="RunSettings") job = Job(app, launch_settings) - return Generator(gen_path=experiment_path, run_ID="mock_run", job=job) + return Generator(exp_path=experiment_path, run_id="mock_run") @pytest.fixture @@ -60,17 +60,16 @@ def job_instance(wlmutils) -> Job: return job -def test_default_log_level(generator_instance): +def test_default_log_level(generator_instance, monkeypatch): """Test if the default log level is INFO.""" + monkeypatch.setenv("SMARTSIM_LOG_LEVEL", "info") assert generator_instance.log_level == INFO -def test_debug_log_level(generator_instance): +def test_debug_log_level(generator_instance,monkeypatch): """Test if the log level is DEBUG when environment variable is set to "debug".""" - environ["SMARTSIM_LOG_LEVEL"] = "debug" + monkeypatch.setenv("SMARTSIM_LOG_LEVEL", "debug") assert generator_instance.log_level == DEBUG - # Clean up: unset the environment variable - environ.pop("SMARTSIM_LOG_LEVEL", None) def test_log_file_path(generator_instance): From 5e731a19b594e39ab05575e9353267854e3081aa Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 2 Aug 2024 10:03:43 -0500 Subject: [PATCH 53/82] addressing matts comments --- smartsim/_core/generation/generator.py | 21 ------- smartsim/entity/ensemble.py | 2 +- tests/test_experiment.py | 3 +- tests/test_generator.py | 80 +++++++++----------------- 4 files changed, 31 insertions(+), 75 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index dfb486c92..ef56bb64e 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -70,27 +70,6 @@ def __init__(self, exp_path: str, run_id: str) -> None: """The runID for Experiment.start""" - @property - def log_level(self) -> int: - """Determines the log level based on the value of the environment - variable SMARTSIM_LOG_LEVEL. - - If the environment variable is set to "debug", returns the log level DEBUG. - Otherwise, returns the default log level INFO. - - :return: Log level (DEBUG or INFO) - """ - # Get the value of the environment variable SMARTSIM_LOG_LEVEL - env_log_level = os.getenv("SMARTSIM_LOG_LEVEL") - - # Set the default log level to INFO - default_log_level = INFO - - if env_log_level == "debug": - return DEBUG - else: - return default_log_level - def log_file(self, log_path: pathlib.Path) -> str: """Returns the location of the file summarizing the parameters used for the last generation diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index 3835c7730..07ebe25de 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -104,4 +104,4 @@ def as_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: apps = self._create_applications() if not apps: raise ValueError("There are no members as part of this ensemble") - return tuple(Job(app, settings, app.name) for app in enumerate(apps, 1)) + return tuple(Job(app, settings, app.name) for app in apps) diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 6c2378cb5..e64de55d1 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -38,6 +38,7 @@ from smartsim.entity import _mock, entity from smartsim.experiment import Experiment from smartsim.launchable import job +from smartsim._core.generation import Generator from smartsim.settings import dispatch, launchSettings from smartsim.settings.arguments import launchArguments @@ -51,7 +52,7 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda job: f"/tmp/job") + monkeypatch.setattr(exp, "_generate", lambda gen, job: (Generator(test_dir, "temp_run"), f"/tmp/job")) yield exp diff --git a/tests/test_generator.py b/tests/test_generator.py index a10cf3504..c2dab6160 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -10,7 +10,7 @@ from smartsim import Experiment from smartsim._core.generation.generator import Generator -from smartsim.entity import Application, Ensemble +from smartsim.entity import Application, Ensemble, SmartSimEntity, _mock from smartsim.launchable import Job from smartsim.settings.launchSettings import LaunchSettings @@ -19,10 +19,13 @@ pytestmark = pytest.mark.group_a +@pytest.fixture +def get_gen_copy_file(fileutils): + return fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) -def get_gen_file(fileutils, filename): - return fileutils.get_test_conf_path(osp.join("generator_files", filename)) - +@pytest.fixture +def get_gen_symlink_file(fileutils): + return fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) # Mock Launcher class NoOpLauncher: @@ -34,48 +37,30 @@ def start(self, _): return "anything" -# Mock Application -class EchoApp: - name = "echo_app" - - def as_program_arguments(self): - return ["echo", "Hello", "World!"] +@pytest.fixture +def echo_app(): + yield SmartSimEntity("echo_app", run_settings=_mock.Mock()) @pytest.fixture -def generator_instance(test_dir, wlmutils) -> Generator: +def generator_instance(test_dir) -> Generator: """Fixture to create an instance of Generator.""" experiment_path = osp.join(test_dir, "experiment_name") - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("app_name", exe="python", run_settings="RunSettings") - job = Job(app, launch_settings) return Generator(exp_path=experiment_path, run_id="mock_run") @pytest.fixture -def job_instance(wlmutils) -> Job: +def job_instance(wlmutils, echo_app) -> Job: """Fixture to create an instance of Job.""" launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job = Job(EchoApp(), launch_settings) + job = Job(echo_app, launch_settings) return job - -def test_default_log_level(generator_instance, monkeypatch): - """Test if the default log level is INFO.""" - monkeypatch.setenv("SMARTSIM_LOG_LEVEL", "info") - assert generator_instance.log_level == INFO - - -def test_debug_log_level(generator_instance,monkeypatch): - """Test if the log level is DEBUG when environment variable is set to "debug".""" - monkeypatch.setenv("SMARTSIM_LOG_LEVEL", "debug") - assert generator_instance.log_level == DEBUG - - def test_log_file_path(generator_instance): """Test if the log_file property returns the correct path.""" - expected_path = osp.join(generator_instance.path, "smartsim_params.txt") - assert generator_instance.log_file == expected_path + path = "/tmp" + expected_path = osp.join(path, "smartsim_params.txt") + assert generator_instance.log_file(path) == expected_path def test_generate_job_directory(test_dir, wlmutils): @@ -140,12 +125,11 @@ def test_generate_copy_file(fileutils, wlmutils, test_dir): assert osp.isfile(expected_file) -def test_generate_copy_directory(fileutils, wlmutils, test_dir): +def test_generate_copy_directory(wlmutils, test_dir, get_gen_copy_file): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("name", "python", "RunSettings") - copy_dir = get_gen_file(fileutils, "to_copy_dir") - app.attach_generator_files(to_copy=copy_dir) + app.attach_generator_files(to_copy=get_gen_copy_file) job = Job(app, launch_settings) # Create the experiment @@ -156,12 +140,12 @@ def test_generate_copy_directory(fileutils, wlmutils, test_dir): assert osp.isfile(expected_file) -def test_generate_symlink_directory(fileutils, wlmutils, test_dir): +def test_generate_symlink_directory(wlmutils, test_dir, get_gen_symlink_file): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("name", "python", "RunSettings") # Path of directory to symlink - symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + symlink_dir = get_gen_symlink_file # Attach directory to Application app.attach_generator_files(to_symlink=symlink_dir) # Create Job @@ -180,12 +164,12 @@ def test_generate_symlink_directory(fileutils, wlmutils, test_dir): assert written == correct -def test_generate_symlink_file(fileutils, wlmutils, test_dir): +def test_generate_symlink_file(get_gen_symlink_file, wlmutils, test_dir): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("name", "python", "RunSettings") # Path of directory to symlink - symlink_dir = get_gen_file(fileutils, "to_symlink_dir") + symlink_dir = get_gen_symlink_file # Get a list of all files in the directory symlink_files = sorted(glob(symlink_dir + "/*")) # Attach directory to Application @@ -247,32 +231,24 @@ def test_generate_configure(fileutils, wlmutils, test_dir): # assert osp.isfile(smartsim_params_path) -def test_exp_private_generate_method_ensemble(test_dir, wlmutils): +def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_instance): """Test that Job directory was created from Experiment.""" ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) - no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) + exp = Experiment(name="exp_name", exp_path=test_dir) for job in job_list: - job_execution_path = no_op_exp._generate(job) + job_execution_path = exp._generate(generator_instance, job) head, _ = os.path.split(job_execution_path) expected_log_path = pathlib.Path(head) / "log" assert osp.isdir(job_execution_path) assert osp.isdir(pathlib.Path(expected_log_path)) -def test_generate_ensemble_directory(test_dir, wlmutils): +def test_generate_ensemble_directory(test_dir, wlmutils, generator_instance): ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) for job in job_list: - run_ID = "temp_run" - gen = Generator(gen_path=test_dir, run_ID=run_ID, job=job) - gen.generate_experiment() - assert osp.isdir(gen.path) - assert osp.isdir(pathlib.Path(gen.log_path)) - - -def test_dummy(test_dir, job_instance): - exp = Experiment(name="exp-name", exp_path=test_dir) - exp.start(job_instance) + job_path = generator_instance.generate_job(job) + assert osp.isdir(job_path) \ No newline at end of file From 80e22297e165378909ec54cbc97ce712ddd277f8 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 2 Aug 2024 10:47:05 -0500 Subject: [PATCH 54/82] generator tests passing, doc strings updated --- smartsim/_core/generation/generator.py | 91 ++++++++++++-------------- smartsim/experiment.py | 28 ++++---- 2 files changed, 59 insertions(+), 60 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index ef56bb64e..cee4d4f05 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -31,10 +31,8 @@ import shutil import typing as t from datetime import datetime -from glob import glob -from logging import DEBUG, INFO from os import mkdir, path -from os.path import join, relpath +from os.path import join import subprocess import sys @@ -47,9 +45,9 @@ class Generator: - """The primary job of the generator is to create the file structure - for a SmartSim Experiment. The Generator is also responsible for - writing files into a Job directory. + """The primary job of the Generator is to create the directory and file structure + for a SmartSim Job. The Generator is also responsible for writing and configuring + files into the Job directory. """ def __init__(self, exp_path: str, run_id: str) -> None: @@ -57,12 +55,11 @@ def __init__(self, exp_path: str, run_id: str) -> None: The Generator class is responsible for creating Job directories. It ensures that paths adhere to SmartSim path standards. Additionally, - it creates a log directory for telemetry data to handle symlinking, + it creates a run directory to handle symlinking, configuration, and file copying to the job directory. :param gen_path: Path in which files need to be generated :param run_ID: The id of the Experiment - :param job: Reference to a name, SmartSimEntity and LaunchSettings """ self.exp_path = pathlib.Path(exp_path) """The path under which the experiment operate""" @@ -72,24 +69,23 @@ def __init__(self, exp_path: str, run_id: str) -> None: def log_file(self, log_path: pathlib.Path) -> str: """Returns the location of the file - summarizing the parameters used for the last generation - of all generated entities. + summarizing the parameters used for the generation + of the entity. - :returns: path to file with parameter settings + :param log_path: Path to log directory + :returns: Path to file with parameter settings """ return join(log_path, "smartsim_params.txt") def generate_job(self, job: Job) -> pathlib.Path: - """Generate the directories + """Generate the Job directory - Generate the file structure for a SmartSim experiment. This - includes writing and configuring input files for a job. + Generate the file structure for a SmartSim Job. This + includes writing and configuring input files for the entity. - To have files or directories present in the created job + To have files or directories present in the created Job directories, such as datasets or input files, call - ``entity.attach_generator_files`` prior to generation. See - ``entity.attach_generator_files`` for more information on - what types of files can be included. + ``entity.attach_generator_files`` prior to generation. Tagged application files are read, checked for input variables to configure, and written. Input variables to configure are @@ -103,24 +99,24 @@ def generate_job(self, job: Job) -> pathlib.Path: # Generate ../job_name/log directory log_path = self._generate_log_path(job) + # Create and write to the parameter settings file with open(self.log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - # Perform file system operations + # Perform file system ops self._build_operations(job, job_path) - # Return job path + # Return Job path return job_path def _generate_job_path(self, job: Job) -> pathlib.Path: """ - Generates the directory path for a job based on its creation type - (whether created via ensemble or job init). + Generate the run directory for a Job. - :param job: The Job object - :returns str: The generated path for the job. + :param job: The Job to generate a directory + :returns pathlib.Path:: The generated run path for the Job """ job_type = f"{job.__class__.__name__.lower()}s" job_path = ( @@ -137,10 +133,10 @@ def _generate_job_path(self, job: Job) -> pathlib.Path: def _generate_log_path(self, job: Job) -> pathlib.Path: """ - Generate the path for the log folder. + Generate the log directory for a Job. - :param job: The Job object - :returns str: The generated path for the log directory + :param job: The Job to generate a directory + :returns pathlib.Path:: The generated log path for the Job """ job_type = f"{job.__class__.__name__.lower()}s" log_path = ( @@ -155,26 +151,27 @@ def _generate_log_path(self, job: Job) -> pathlib.Path: def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: - """This method generates file system operations based on the provided application. - It processes three types of operations: to_copy, to_symlink, and to_configure. - For each type, it calls the corresponding private methods and appends the results - to the `file_operation_list`. + """This method orchestrates file system ops for the attached SmartSim entity. + It processes three types of file system ops: to_copy, to_symlink, and to_configure. + For each type, it calls the corresponding private methods that open a subprocess + to complete each task. - :param app: The application for which operations are generated. - :return: A list of lists containing file system operations. + :param job: The Job to perform file ops on attached entity files + :param job_path: Path to the Jobs run directory """ app = t.cast(Application, job.entity) + self._copy_files(job.entity, job_path) self._symlink_files(job.entity, job_path) self._write_tagged_files(job.entity, job_path) - self._copy_files(job.entity, job_path) @staticmethod def _copy_files(app: Application, dest: pathlib.Path) -> None: - """Get copy file system operation for a file. + """Perform copy file sys operations on a list of files. :param app: The Application attached to the Job - :param dest: Path to copy files + :param dest: Path to the Jobs run directory """ + # Return if no files are attached if app.files is None: return for src in app.files.copy: @@ -185,31 +182,33 @@ def _copy_files(app: Application, dest: pathlib.Path) -> None: @staticmethod def _symlink_files(app: Application, dest: pathlib.Path) -> None: - """Get symlink file system operation for a file. + """Perform symlink file sys operations on a list of files. :param app: The Application attached to the Job - :param dest: Path to symlink files + :param dest: Path to the Jobs run directory """ + # Return if no files are attached if app.files is None: return for src in app.files.link: - # # Normalize the path to remove trailing slashes + # Normalize the path to remove trailing slashes normalized_path = os.path.normpath(src) - # # Get the parent directory (last folder) + # Get the parent directory (last folder) parent_dir = os.path.basename(normalized_path) + # Create destination new_dest = os.path.join(str(dest), parent_dir) subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "symlink", src, new_dest]) @staticmethod def _write_tagged_files(app: Application, dest: pathlib.Path) -> None: """Read, configure and write the tagged input files for - a Application instance within an ensemble. This function - specifically deals with the tagged files attached to - an Ensemble. + a Job instance. This function specifically deals with the tagged + files attached to an entity. :param app: The Application attached to the Job - :param dest: Path to configure files + :param dest: Path to the Jobs run directory """ + # Return if no files are attached if app.files is None: return if app.files.tagged: @@ -242,10 +241,6 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: encoded_dict = base64.b64encode(pickled_dict).decode("ascii") for dest_path in to_write: subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "configure", dest_path, dest_path, tag, encoded_dict]) - # cmd = f"configure {dest_path} {dest_path} {tag} {encoded_dict}" - # args = cmd.split() - # ns = parser.parse_args(args) - # file_operations.configure(ns) # TODO address in ticket 723 # self._log_params(entity, files_to_params) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index c70dd04d1..1006f8a01 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -189,6 +189,8 @@ def _dispatch( ) -> tuple[LaunchedJobID, ...]: """Dispatch a series of jobs with a particular dispatcher + :param generator: The Generator holds the run_id and experiment + path for use when producing job directories. :param dispatcher: The dispatcher that should be used to determine how to start a job based on its launch settings. :param job: The first job instance to dispatch @@ -226,7 +228,7 @@ def execute_dispatch(generator: Generator, job: Job) -> LaunchedJobID: # it easier to monitor job statuses # pylint: disable-next=protected-access self._active_launchers.add(launch_config._adapted_launcher) - # Generate the Job directory and return generated path + # Generate the job directory and return the generated job path job_execution_path = self._generate(generator, job) return launch_config.start(exe, env, job_execution_path) @@ -234,21 +236,23 @@ def execute_dispatch(generator: Generator, job: Job) -> LaunchedJobID: @_contextualize def _generate(self, generator: Generator, job: Job) -> str: - """Generate the file structure for a ``Job`` + """Generate the directory and file structure for a ``Job`` - ``Experiment._generate`` creates directories for the job - passed. + ``Experiment._generate`` calls the appropriate Generator + function to create a directory for the passed job. - If files or directories are attached an ``application`` object - using ``application.attach_generator_files()``, those files or - directories will be symlinked, copied, or configured and - written into the created directory for that Job instance. + If files or directories are attached to an ``application`` object + associated with the Job using ``application.attach_generator_files()``, + those files or directories will be symlinked, copied, or configured and + written into the created job directory - An instance of ``Job`` can be passed as an argument to - the protected generate member. + An instance of ``Generator`` and ``Job`` can be passed as an argument to + the protected _generate member. - :param job: Job to generate file structure for - :returns: a str path + :param generator: Generator that holds the run_id and experiment + path for use when producing the job directory. + :param job: Job to generate file structure. + :returns: The generated Job path. """ try: job_path = generator.generate_job(job) From 1f1fcdced262fd47d48f17e965dc12fa764bd2a9 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 2 Aug 2024 17:10:54 -0500 Subject: [PATCH 55/82] finished addressing matt comments --- smartsim/_core/generation/generator.py | 96 ++++++--- .../_core/launcher/dragon/dragonLauncher.py | 2 +- smartsim/experiment.py | 42 ++-- smartsim/settings/dispatch.py | 31 ++- .../test_settings/test_alpsLauncher.py | 2 +- .../test_settings/test_localLauncher.py | 2 +- .../test_settings/test_lsfLauncher.py | 2 +- .../test_settings/test_mpiLauncher.py | 2 +- .../test_settings/test_palsLauncher.py | 2 +- .../test_settings/test_slurmLauncher.py | 2 +- tests/test_experiment.py | 6 +- tests/test_generator.py | 195 ++++++++++++++---- 12 files changed, 262 insertions(+), 122 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index cee4d4f05..57859af04 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -29,14 +29,15 @@ import pathlib import pickle import shutil +import subprocess +import sys import typing as t from datetime import datetime from os import mkdir, path from os.path import join -import subprocess -import sys from ...entity import Application, TaggedFilesHierarchy +from ...entity.files import EntityFiles from ...launchable import Job from ...log import get_logger @@ -66,7 +67,6 @@ def __init__(self, exp_path: str, run_id: str) -> None: self.run_id = run_id """The runID for Experiment.start""" - def log_file(self, log_path: pathlib.Path) -> str: """Returns the location of the file summarizing the parameters used for the generation @@ -77,7 +77,7 @@ def log_file(self, log_path: pathlib.Path) -> str: """ return join(log_path, "smartsim_params.txt") - def generate_job(self, job: Job) -> pathlib.Path: + def generate_job(self, job: Job, job_index: int) -> pathlib.Path: """Generate the Job directory Generate the file structure for a SmartSim Job. This @@ -95,9 +95,9 @@ def generate_job(self, job: Job) -> pathlib.Path: """ # Generate ../job_name/run directory - job_path = self._generate_job_path(job) + job_path = self._generate_job_path(job, job_index) # Generate ../job_name/log directory - log_path = self._generate_log_path(job) + log_path = self._generate_log_path(job, job_index) # Create and write to the parameter settings file with open(self.log_file(log_path), mode="w", encoding="utf-8") as log_file: @@ -110,8 +110,7 @@ def generate_job(self, job: Job) -> pathlib.Path: # Return Job path return job_path - - def _generate_job_path(self, job: Job) -> pathlib.Path: + def _generate_job_path(self, job: Job, job_index: int) -> pathlib.Path: """ Generate the run directory for a Job. @@ -120,18 +119,13 @@ def _generate_job_path(self, job: Job) -> pathlib.Path: """ job_type = f"{job.__class__.__name__.lower()}s" job_path = ( - self.exp_path / - self.run_id / - job_type / - job.name / - "run" + self.exp_path / self.run_id / job_type / f"{job.name}-{job_index}" / "run" ) # Create Job directory job_path.mkdir(exist_ok=True, parents=True) return job_path - - def _generate_log_path(self, job: Job) -> pathlib.Path: + def _generate_log_path(self, job: Job, job_index: int) -> pathlib.Path: """ Generate the log directory for a Job. @@ -140,16 +134,11 @@ def _generate_log_path(self, job: Job) -> pathlib.Path: """ job_type = f"{job.__class__.__name__.lower()}s" log_path = ( - self.exp_path / - self.run_id / - job_type / - job.name / - "log" + self.exp_path / self.run_id / job_type / f"{job.name}-{job_index}" / "log" ) log_path.mkdir(exist_ok=True, parents=True) return log_path - def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: """This method orchestrates file system ops for the attached SmartSim entity. It processes three types of file system ops: to_copy, to_symlink, and to_configure. @@ -160,44 +149,72 @@ def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: :param job_path: Path to the Jobs run directory """ app = t.cast(Application, job.entity) - self._copy_files(job.entity, job_path) - self._symlink_files(job.entity, job_path) - self._write_tagged_files(job.entity, job_path) + self._copy_files(app.files, job_path) + self._symlink_files(app.files, job_path) + self._write_tagged_files(app, job_path) @staticmethod - def _copy_files(app: Application, dest: pathlib.Path) -> None: + def _copy_files(files: EntityFiles | None, dest: pathlib.Path) -> None: """Perform copy file sys operations on a list of files. :param app: The Application attached to the Job :param dest: Path to the Jobs run directory """ # Return if no files are attached - if app.files is None: + if files is None: return - for src in app.files.copy: + for src in files.copy: if os.path.isdir(src): - subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "copy", src, dest, "--dirs_exist_ok"]) + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "copy", + src, + dest, + "--dirs_exist_ok", + ] + ) else: - subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "copy", src, dest]) + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "copy", + src, + dest, + ] + ) @staticmethod - def _symlink_files(app: Application, dest: pathlib.Path) -> None: + def _symlink_files(files: EntityFiles | None, dest: pathlib.Path) -> None: """Perform symlink file sys operations on a list of files. :param app: The Application attached to the Job :param dest: Path to the Jobs run directory """ # Return if no files are attached - if app.files is None: + if files is None: return - for src in app.files.link: + for src in files.link: # Normalize the path to remove trailing slashes normalized_path = os.path.normpath(src) # Get the parent directory (last folder) parent_dir = os.path.basename(normalized_path) # Create destination new_dest = os.path.join(str(dest), parent_dir) - subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "symlink", src, new_dest]) + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "symlink", + src, + new_dest, + ] + ) @staticmethod def _write_tagged_files(app: Application, dest: pathlib.Path) -> None: @@ -240,7 +257,18 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: # Encode the pickled dictionary with Base64 encoded_dict = base64.b64encode(pickled_dict).decode("ascii") for dest_path in to_write: - subprocess.run(args=[sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "configure", dest_path, dest_path, tag, encoded_dict]) + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "configure", + dest_path, + dest_path, + tag, + encoded_dict, + ] + ) # TODO address in ticket 723 # self._log_params(entity, files_to_params) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 908c84807..288939d2b 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -355,7 +355,7 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: def _as_run_request_args_and_policy( run_req_args: DragonLaunchArguments, exe: ExecutableProtocol, - path: str, + path: str | os.PathLike[str], env: t.Mapping[str, str | None], ) -> tuple[DragonRunRequestView, DragonRunPolicy]: # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 1006f8a01..ced006ff4 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -31,6 +31,7 @@ import datetime import os import os.path as osp +import pathlib import textwrap import typing as t from os import environ, getcwd @@ -182,10 +183,16 @@ def start(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: """ run_id = datetime.datetime.now().strftime("run-%H:%M:%ST%Y-%m-%d") """Create the run id for Experiment.start""" - return self._dispatch(Generator(self.exp_path, run_id), dispatch.DEFAULT_DISPATCHER, *jobs) + return self._dispatch( + Generator(self.exp_path, run_id), dispatch.DEFAULT_DISPATCHER, *jobs + ) def _dispatch( - self, generator: Generator, dispatcher: dispatch.Dispatcher, job: Job, *jobs: Job + self, + generator: Generator, + dispatcher: dispatch.Dispatcher, + job: Job, + *jobs: Job, ) -> tuple[LaunchedJobID, ...]: """Dispatch a series of jobs with a particular dispatcher @@ -200,7 +207,8 @@ def _dispatch( particular dispatch of the job. """ - def execute_dispatch(generator: Generator, job: Job) -> LaunchedJobID: + def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: + print(job) args = job.launch_settings.launch_args env = job.launch_settings.env_vars # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @@ -229,13 +237,15 @@ def execute_dispatch(generator: Generator, job: Job) -> LaunchedJobID: # pylint: disable-next=protected-access self._active_launchers.add(launch_config._adapted_launcher) # Generate the job directory and return the generated job path - job_execution_path = self._generate(generator, job) - return launch_config.start(exe, env, job_execution_path) + job_execution_path = self._generate(generator, job, idx) + return launch_config.start(exe, job_execution_path, env) - return execute_dispatch(generator, job), *map(execute_dispatch, jobs) + return execute_dispatch(generator, job, 0), *( + execute_dispatch(generator, job, idx) for idx, job in enumerate(jobs, 1) + ) @_contextualize - def _generate(self, generator: Generator, job: Job) -> str: + def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.Path: """Generate the directory and file structure for a ``Job`` ``Experiment._generate`` calls the appropriate Generator @@ -255,7 +265,7 @@ def _generate(self, generator: Generator, job: Job) -> str: :returns: The generated Job path. """ try: - job_path = generator.generate_job(job) + job_path = generator.generate_job(job, job_index) return job_path except SmartSimError as e: logger.error(e) @@ -338,22 +348,6 @@ def telemetry(self) -> TelemetryConfiguration: """ return self._telemetry_cfg - def _create_entity_dir(self, start_manifest: Manifest) -> None: - def create_entity_dir( - entity: t.Union[FeatureStore, Application, Ensemble] - ) -> None: - if not osp.isdir(entity.path): - os.makedirs(entity.path) - - for application in start_manifest.applications: - create_entity_dir(application) - - for feature_store in start_manifest.fss: - create_entity_dir(feature_store) - - for ensemble in start_manifest.ensembles: - create_entity_dir(ensemble) - def __str__(self) -> str: return self.name diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 5cf6aafc4..1a9837212 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -27,6 +27,7 @@ from __future__ import annotations import dataclasses +import os import subprocess as sp import typing as t import uuid @@ -44,6 +45,11 @@ _Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) +_WorkingDirectory: TypeAlias = str | os.PathLike[str] +"""A type alias for a Jobs working directory. Paths may be strings or +PathLike objects. +""" + _DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") """Any type of luanch arguments, typically used when the type bound by the type argument is a key a `Dispatcher` dispatch registry @@ -58,13 +64,14 @@ a job """ _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", str, _EnvironMappingType], _LaunchableT + [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType], + _LaunchableT, ] """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ _LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType, str]" + "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType]" ) """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings @@ -253,7 +260,9 @@ def create_adapter_from_launcher( ) def format_( - exe: ExecutableProtocol, env: _EnvironMappingType, path: str + exe: ExecutableProtocol, + path: str | os.PathLike[str], + env: _EnvironMappingType, ) -> _LaunchableT: return self.formatter(arguments, exe, path, env) @@ -388,7 +397,7 @@ def create(cls, exp: Experiment, /) -> Self: ... def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[LaunchArguments, tuple[t.Sequence[str], str]]: +) -> _FormatterType[LaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]]]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -421,10 +430,10 @@ def make_shell_format_fn( def impl( args: LaunchArguments, exe: ExecutableProtocol, - path: str, + path: str | os.PathLike[str], _env: _EnvironMappingType, - ) -> t.Tuple[t.Sequence[str], str]: - return ( + ) -> t.Tuple[str | os.PathLike[str], t.Sequence[str]]: + return path, ( ( run_command, *(args.format_launch_args() or ()), @@ -433,7 +442,7 @@ def impl( ) if run_command is not None else exe.as_program_arguments() - ), path + ) return impl @@ -445,9 +454,11 @@ def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} # TODO inject path here - def start(self, command: tuple[t.Sequence[str], str]) -> LaunchedJobID: + def start( + self, command: tuple[str | os.PathLike[str], t.Sequence[str]] + ) -> LaunchedJobID: id_ = create_job_id() - args, path = command + path, args = command exe, *rest = args # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 36acb1b5b..370b67db7 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -211,7 +211,7 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_aprun_command( + path, cmd = _as_aprun_command( AprunLaunchArguments(args), mock_echo_executable, test_dir, {} ) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index db896ac27..48de0e7b5 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -143,7 +143,7 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): - cmd, path = _as_local_command( + path, cmd = _as_local_command( LocalLaunchArguments({}), mock_echo_executable, test_dir, {} ) assert tuple(cmd) == ("echo", "hello", "world") diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 71b88f5f6..eec915860 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -120,7 +120,7 @@ def test_launch_args(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_jsrun_command( + path, cmd = _as_jsrun_command( JsrunLaunchArguments(args), mock_echo_executable, test_dir, {} ) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 69222ae4e..ff5200eca 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -286,6 +286,6 @@ def test_invalid_hostlist_format(launcher): def test_formatting_launch_args( mock_echo_executable, cls, fmt, cmd, args, expected, test_dir ): - fmt_cmd, path = fmt(cls(args), mock_echo_executable, test_dir, {}) + path, fmt_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}) assert tuple(fmt_cmd) == (cmd,) + expected assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index a955c3662..64b9dc7f1 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -132,7 +132,7 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_pals_command( + path, cmd = _as_pals_command( PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {} ) assert tuple(cmd) == expected diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index bf18035c2..1c21e3d01 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -317,7 +317,7 @@ def test_set_het_groups(monkeypatch): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - cmd, path = _as_srun_command( + path, cmd = _as_srun_command( SlurmLaunchArguments(args), mock_echo_executable, test_dir, {} ) assert tuple(cmd) == expected diff --git a/tests/test_experiment.py b/tests/test_experiment.py index e64de55d1..474eb0aa8 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -35,10 +35,10 @@ import pytest +from smartsim._core.generation import Generator from smartsim.entity import _mock, entity from smartsim.experiment import Experiment from smartsim.launchable import job -from smartsim._core.generation import Generator from smartsim.settings import dispatch, launchSettings from smartsim.settings.arguments import launchArguments @@ -52,7 +52,7 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda gen, job: (Generator(test_dir, "temp_run"), f"/tmp/job")) + monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: "/tmp/job", "1") yield exp @@ -181,7 +181,6 @@ class EchoHelloWorldEntity(entity.SmartSimEntity): def __init__(self): path = tempfile.TemporaryDirectory() - self._finalizer = weakref.finalize(self, path.cleanup) super().__init__("test-entity", _mock.Mock()) def __eq__(self, other): @@ -216,6 +215,7 @@ def test_start_can_launch_jobs( num_jobs: int, ) -> None: jobs = make_jobs(job_maker, num_jobs) + print(jobs) assert len(experiment._active_launchers) == 0, "Initialized w/ launchers" launched_ids = experiment.start(*jobs) assert len(experiment._active_launchers) == 1, "Unexpected number of launchers" diff --git a/tests/test_generator.py b/tests/test_generator.py index c2dab6160..04e104dc1 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -2,8 +2,7 @@ import os import pathlib from glob import glob -from logging import DEBUG, INFO -from os import environ, listdir +from os import listdir from os import path as osp import pytest @@ -11,34 +10,36 @@ from smartsim import Experiment from smartsim._core.generation.generator import Generator from smartsim.entity import Application, Ensemble, SmartSimEntity, _mock +from smartsim.entity.files import EntityFiles from smartsim.launchable import Job -from smartsim.settings.launchSettings import LaunchSettings +from smartsim.settings import LaunchSettings, dispatch -# TODO Test ensemble copy, config, symlink when Ensemble.attach_generator_files added # TODO Add JobGroup tests when JobGroup becomes a Launchable pytestmark = pytest.mark.group_a + @pytest.fixture def get_gen_copy_file(fileutils): - return fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) + """Fixture to yield directory to copy.""" + yield fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) + @pytest.fixture def get_gen_symlink_file(fileutils): - return fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) + """Fixture to yield directory to symlink.""" + yield fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) -# Mock Launcher -class NoOpLauncher: - @classmethod - def create(cls, _): - return cls() - def start(self, _): - return "anything" +@pytest.fixture +def get_gen_configure_file(fileutils): + """Fixture to yield directory to symlink.""" + yield fileutils.get_test_conf_path(osp.join("generator_files", "tag_dir_template")) @pytest.fixture def echo_app(): + """Fixture to yield an instance of SmartSimEntity.""" yield SmartSimEntity("echo_app", run_settings=_mock.Mock()) @@ -46,7 +47,7 @@ def echo_app(): def generator_instance(test_dir) -> Generator: """Fixture to create an instance of Generator.""" experiment_path = osp.join(test_dir, "experiment_name") - return Generator(exp_path=experiment_path, run_id="mock_run") + yield Generator(exp_path=experiment_path, run_id="mock_run") @pytest.fixture @@ -56,11 +57,12 @@ def job_instance(wlmutils, echo_app) -> Job: job = Job(echo_app, launch_settings) return job + def test_log_file_path(generator_instance): - """Test if the log_file property returns the correct path.""" - path = "/tmp" - expected_path = osp.join(path, "smartsim_params.txt") - assert generator_instance.log_file(path) == expected_path + """Test if the log_file function returns the correct log path.""" + base_path = "/tmp" + expected_path = osp.join(base_path, "smartsim_params.txt") + assert generator_instance.log_file(base_path) == expected_path def test_generate_job_directory(test_dir, wlmutils): @@ -76,13 +78,13 @@ def test_generate_job_directory(test_dir, wlmutils): # Generator instance gen = Generator(exp_path=experiment_path, run_id=run_id) # Call Generator.generate_job - job_path = gen.generate_job(job) + job_path = gen.generate_job(job, 1) assert isinstance(job_path, pathlib.Path) expected_run_path = ( pathlib.Path(experiment_path) / run_id / f"{job.__class__.__name__.lower()}s" - / app.name + / f"{app.name}-{1}" / "run" ) assert job_path == expected_run_path @@ -90,7 +92,7 @@ def test_generate_job_directory(test_dir, wlmutils): pathlib.Path(experiment_path) / run_id / f"{job.__class__.__name__.lower()}s" - / app.name + / f"{app.name}-{1}" / "log" ) assert osp.isdir(expected_run_path) @@ -98,15 +100,17 @@ def test_generate_job_directory(test_dir, wlmutils): assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) -# def test_exp_private_generate_method_app(test_dir, job_instance): -# """Test that Job directory was created from Experiment.""" -# no_op_exp = Experiment(name="No-Op-Exp", exp_path=test_dir) -# job_execution_path = no_op_exp._generate(job_instance) -# assert osp.isdir(job_execution_path) -# head, _ = os.path.split(job_execution_path) -# expected_log_path = pathlib.Path(head) / "log" -# assert osp.isdir(expected_log_path) -# assert osp.isfile(osp.join(job_execution_path, "smartsim_params.txt")) +def test_exp_private_generate_method_app(wlmutils, test_dir, generator_instance): + """Test that Job directory was created from Experiment.""" + exp = Experiment(name="experiment_name", exp_path=test_dir) + app = Application("name", "python", "RunSettings") + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job = Job(app, launch_settings) + job_execution_path = exp._generate(generator_instance, job, 1) + assert osp.isdir(job_execution_path) + head, _ = os.path.split(job_execution_path) + expected_log_path = pathlib.Path(head) / "log" + assert osp.isdir(expected_log_path) def test_generate_copy_file(fileutils, wlmutils, test_dir): @@ -120,7 +124,7 @@ def test_generate_copy_file(fileutils, wlmutils, test_dir): # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") gen = Generator(exp_path=experiment_path, run_id="temp_run") - path = gen.generate_job(job) + path = gen.generate_job(job, 1) expected_file = pathlib.Path(path) / "sleep.py" assert osp.isfile(expected_file) @@ -135,7 +139,7 @@ def test_generate_copy_directory(wlmutils, test_dir, get_gen_copy_file): # Create the experiment experiment_path = osp.join(test_dir, "experiment_name") gen = Generator(exp_path=experiment_path, run_id="temp_run") - path = gen.generate_job(job) + path = gen.generate_job(job, 1) expected_file = pathlib.Path(path) / "mock.txt" assert osp.isfile(expected_file) @@ -155,7 +159,7 @@ def test_generate_symlink_directory(wlmutils, test_dir, get_gen_symlink_file): experiment_path = osp.join(test_dir, "experiment_name") gen = Generator(exp_path=experiment_path, run_id="temp_run") # Generate Experiment file structure - job_path = gen.generate_job(job) + job_path = gen.generate_job(job, 1) expected_folder = pathlib.Path(job_path) / "to_symlink_dir" assert osp.isdir(expected_folder) # Combine symlinked file list and original file list for comparison @@ -180,7 +184,7 @@ def test_generate_symlink_file(get_gen_symlink_file, wlmutils, test_dir): experiment_path = osp.join(test_dir, "experiment_name") gen = Generator(exp_path=experiment_path, run_id="mock_run") # Generate Experiment file structure - job_path = gen.generate_job(job) + job_path = gen.generate_job(job, 1) expected_file = pathlib.Path(job_path) / "mock2.txt" assert osp.isfile(expected_file) @@ -218,17 +222,12 @@ def test_generate_configure(fileutils, wlmutils, test_dir): # Spin up Generator gen = Generator(exp_path=experiment_path, run_id="temp_run") # Execute file generation - job_path = gen.generate_job(job) + job_path = gen.generate_job(job, 1) # Retrieve the list of configured files in the test directory configured_files = sorted(glob(str(job_path) + "/*")) # Use filecmp.cmp to check that the corresponding files are equal for written, correct in zip(configured_files, correct_files): assert filecmp.cmp(written, correct) - # Validate that log file exists - # assert osp.isdir() - # Validate that smartsim params files exists - # smartsim_params_path = osp.join(job_path, "smartsim_params.txt") - # assert osp.isfile(smartsim_params_path) def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_instance): @@ -238,17 +237,125 @@ def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_inst job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) for job in job_list: - job_execution_path = exp._generate(generator_instance, job) + job_execution_path = exp._generate(generator_instance, job, 1) head, _ = os.path.split(job_execution_path) expected_log_path = pathlib.Path(head) / "log" assert osp.isdir(job_execution_path) assert osp.isdir(pathlib.Path(expected_log_path)) -def test_generate_ensemble_directory(test_dir, wlmutils, generator_instance): +def test_generate_ensemble_directory(wlmutils, generator_instance): ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) for job in job_list: - job_path = generator_instance.generate_job(job) - assert osp.isdir(job_path) \ No newline at end of file + job_path = generator_instance.generate_job(job, 1) + assert osp.isdir(job_path) + + +def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: "exit", + ) + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + job_dir = listdir(jobs_dir) + for ensemble_dir in job_dir: + run_path = os.path.join(jobs_dir, ensemble_dir, "run") + log_path = os.path.join(jobs_dir, ensemble_dir, "log") + assert osp.isdir(run_path) + assert osp.isdir(log_path) + + +def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_file): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: "exit", + ) + ensemble = Ensemble( + "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_file) + ) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + job_dir = listdir(jobs_dir) + for ensemble_dir in job_dir: + sym_file_path = os.path.join(jobs_dir, ensemble_dir, "run", "mock.txt") + assert osp.isfile(sym_file_path) + + +def test_generate_ensemble_symlink( + test_dir, wlmutils, monkeypatch, get_gen_symlink_file +): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: "exit", + ) + ensemble = Ensemble( + "ensemble-name", + "echo", + replicas=2, + files=EntityFiles(symlink=get_gen_symlink_file), + ) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + job_dir = listdir(jobs_dir) + for ensemble_dir in job_dir: + sym_file_path = os.path.join(jobs_dir, ensemble_dir, "run", "to_symlink_dir") + assert osp.isdir(sym_file_path) + + +def test_generate_ensemble_configure( + test_dir, wlmutils, monkeypatch, get_gen_configure_file +): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: "exit", + ) + params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} + # Retrieve a list of files for configuration + tagged_files = sorted(glob(get_gen_configure_file + "/*")) + ensemble = Ensemble( + "ensemble-name", + "echo", + replicas=1, + files=EntityFiles(tagged=tagged_files), + file_parameters=params, + ) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + + def _check_generated(param_0, param_1, dir): + assert osp.isdir(dir) + assert osp.isfile(osp.join(dir, "tagged_0.sh")) + assert osp.isfile(osp.join(dir, "tagged_1.sh")) + + with open(osp.join(dir, "tagged_0.sh")) as f: + line = f.readline() + assert line.strip() == f'echo "Hello with parameter 0 = {param_0}"' + + with open(osp.join(dir, "tagged_1.sh")) as f: + line = f.readline() + assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"' + + _check_generated(0, 3, os.path.join(jobs_dir, "ensemble-name-1-1", "run")) + _check_generated(1, 2, os.path.join(jobs_dir, "ensemble-name-2-2", "run")) + _check_generated(1, 3, os.path.join(jobs_dir, "ensemble-name-3-3", "run")) + _check_generated(0, 2, os.path.join(jobs_dir, "ensemble-name-0-0", "run")) From 202f6293264679aaa2f2f90ce7eae15fb42d32dd Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 2 Aug 2024 17:43:07 -0500 Subject: [PATCH 56/82] attempt to fix ML runtimes workflow --- smartsim/settings/dispatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 1a9837212..5766c0780 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -45,7 +45,7 @@ _Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) -_WorkingDirectory: TypeAlias = str | os.PathLike[str] +_WorkingDirectory: TypeAlias = t.Union[str, os.PathLike[str]] """A type alias for a Jobs working directory. Paths may be strings or PathLike objects. """ From f9c9d56ad098156d03b71319c2c0743ac27671e8 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 2 Aug 2024 18:22:10 -0500 Subject: [PATCH 57/82] type error in ML workflow --- smartsim/_core/generation/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 57859af04..e25262c5e 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -154,7 +154,7 @@ def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: self._write_tagged_files(app, job_path) @staticmethod - def _copy_files(files: EntityFiles | None, dest: pathlib.Path) -> None: + def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: """Perform copy file sys operations on a list of files. :param app: The Application attached to the Job @@ -189,7 +189,7 @@ def _copy_files(files: EntityFiles | None, dest: pathlib.Path) -> None: ) @staticmethod - def _symlink_files(files: EntityFiles | None, dest: pathlib.Path) -> None: + def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: """Perform symlink file sys operations on a list of files. :param app: The Application attached to the Job From 43c82ee2d1818f67dc3110c498562a6c032bbb69 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 5 Aug 2024 13:05:31 -0500 Subject: [PATCH 58/82] tests not passing --- conftest.py | 34 ++++----- smartsim/_core/generation/generator.py | 1 + smartsim/experiment.py | 2 +- smartsim/settings/dispatch.py | 5 +- tests/test_shell_launcher.py | 99 +++++++++++++++++--------- 5 files changed, 88 insertions(+), 53 deletions(-) diff --git a/conftest.py b/conftest.py index 3facd09a9..63a8c8081 100644 --- a/conftest.py +++ b/conftest.py @@ -154,23 +154,23 @@ def pytest_sessionfinish( Called after whole test run finished, right before returning the exit status to the system. """ - if exitstatus == 0: - cleanup_attempts = 5 - while cleanup_attempts > 0: - try: - shutil.rmtree(test_output_root) - except OSError as e: - cleanup_attempts -= 1 - time.sleep(1) - if not cleanup_attempts: - raise - else: - break - else: - # kill all spawned processes - if CONFIG.test_launcher == "dragon": - time.sleep(5) - kill_all_test_spawned_processes() + # if exitstatus == 0: + # cleanup_attempts = 5 + # while cleanup_attempts > 0: + # try: + # shutil.rmtree(test_output_root) + # except OSError as e: + # cleanup_attempts -= 1 + # time.sleep(1) + # if not cleanup_attempts: + # raise + # else: + # break + # else: + # # kill all spawned processes + # if CONFIG.test_launcher == "dragon": + # time.sleep(5) + # kill_all_test_spawned_processes() def build_mpi_app() -> t.Optional[pathlib.Path]: diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index e25262c5e..b923afcf5 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -163,6 +163,7 @@ def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: # Return if no files are attached if files is None: return + print(f"type is defined as: {type(files)}") for src in files.copy: if os.path.isdir(src): subprocess.run( diff --git a/smartsim/experiment.py b/smartsim/experiment.py index ced006ff4..964ca992c 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -157,7 +157,7 @@ def __init__(self, name: str, exp_path: str | None = None): exp_path = osp.abspath(exp_path) else: exp_path = osp.join(getcwd(), name) - + print("got here") self.exp_path = exp_path """The path under which the experiment operate""" diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 67bcb42fd..f431be213 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -454,15 +454,18 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - # TODO inject path here def start( self, command: tuple[str | os.PathLike[str], t.Sequence[str]] ) -> LaunchedJobID: id_ = create_job_id() path, args = command exe, *rest = args + print(f"first value: {(helpers.expand_exe_path(exe), *rest)}") + print(f"second value: {path}") # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path, env={}, stdin=None, stdout=None) + print(f"did this work: {self._launched[id_]}") + print("you got here") # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 791ccf811..c28c2a46b 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -29,7 +29,7 @@ import pytest import time import weakref -from smartsim.entity import _mock, entity +from smartsim.entity import _mock, entity, Application from smartsim import Experiment from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.slurm import ( @@ -40,6 +40,7 @@ from smartsim.settings.dispatch import ShellLauncher from smartsim.settings.launchCommand import LauncherType from smartsim.launchable import Job +from smartsim.types import LaunchedJobID # always start with unit tests, first test shell launcher init # make sure to test passing invalid values to shell launcher, and correct values # verify the simple assumptions @@ -54,63 +55,93 @@ def __init__(self): path = tempfile.TemporaryDirectory() self._finalizer = weakref.finalize(self, path.cleanup) super().__init__("test-entity", _mock.Mock()) + self.files = Files() def __eq__(self, other): if type(self) is not type(other): return NotImplemented return self.as_program_arguments() == other.as_program_arguments() - def as_program_arguments(): - # return ("/usr/bin/echo", "Hello", "World!") - return ("/usr/bin/sleep", "10") + def as_program_arguments(self): + return ("/usr/bin/echo", "Hello", "World!") + #return ("/usr/bin/sleep", "10") + +class Files(): + def __init__(self): + self.copy = [] + self.link = [] + self.tagged = [] + + +# what is the success criteria +def test_shell_as_py(capsys): + # a unit test should init the obj bc testing that unit of code + launcher = ShellLauncher() # should be testing the method level + # avoid rep + expected_output = "hello" + launcher.start((["echo", expected_output], "/tmp")) # use time.sleep(0.1) -> we do not need sleep in other places + captured = capsys.readouterr() + output = captured.out + assert expected_output in captured.out + # do not need to build exact str, but can just have multiple assert + # verify echo hello + # make a separate test for stdout and stdin -> that test only verifies one component + # tests should do as little as possible, reduce number of constraints + + +# popen returns a non 0 when an error occurs, so test invalid path +# assert not id, might retry -> assert called 5 times, -> could verify that a warning was printed + +# should test a success cond, a failure condition + + +# UNIT TESTS + +def test_shell_launcher_init(): + """A simple test to validate that ShellLauncher correctly initializes""" + # Init ShellLauncher + shell_launcher = ShellLauncher() + # Assert that private attribute is expected value + assert shell_launcher._launched == {} # test that the process leading up to the shell launcher was corrected, integration test # my test is identifying the change in the code -def test_this(test_dir: str, monkeypatch: pytest.MonkeyPatch): +def test_shell_launcher_calls_popen(test_dir: str, monkeypatch: pytest.MonkeyPatch): # monkeypatch the popen # create a Mock popen object # def my_mock_popen(*args, **kwargs): # print("foo") # no longer care about the internals, only want to know that the process up to it was currect - mock2 = unittest.mock.MagicMock(return_value=0) # same as monkeypatch - implements getproperty or API that looks for a unknown prop on an obj - mock3 = unittest.mock.MagicMock() - mock3.Popen = mock2 - mock3.return_value = mock2 - mock2.assert_called() - mock2.assert_called_with() + mock_popen_obj = unittest.mock.MagicMock() + with monkeypatch.context() as ctx: + ctx.setattr(dsp, "Popen", mock_popen_obj) + + # mock2 = unittest.mock.MagicMock(return_value=0) # same as monkeypatch - implements getproperty or API that looks for a unknown prop on an obj + # mock3 = unittest.mock.MagicMock() + # # Avoid actual network request + # mock3.Popen = mock2 + # mock3.return_value = mock2 env_vars = { "LOGGING": "verbose", } slurm_settings = LaunchSettings(launcher=LauncherType.Slurm, env_vars=env_vars) slurm_settings.launch_args.set_nodes(1) - job = Job(name="jobs", entity=EchoHelloWorldEntity, launch_settings=slurm_settings) + job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=slurm_settings) exp = Experiment(name="exp_name", exp_path=test_dir) # can validate id here -> could build another mock that ensures that 22 is the pid id = exp.start(job) - mock3.assert_called_with() # the process executed the correct launcher + # mock2.assert_called_once_with( + # ('/usr/bin/srun', '--nodes=1', '--', '/usr/bin/echo', 'Hello', 'World!'), + # cwd=unittest.mock.ANY, + # env={}, + # stdin=None, + # stdout=None + # ) + # mock_popen_obj.assert_called() + #mock3.assert_called_with() # the process executed the correct launcher # write something that makes sure the job has completed b4 the test exits print(id) #time.sleep(5) # TODO remove once blocking is added # asyn = concurrent, not happening in another thread, not happening somewhere else # focus on async io in python, make sure that anything that is io bound is async - -# what is the success criteria -def test_shell_as_py(capsys): - # a unit test should init the obj bc testing that unit of code - launcher = ShellLauncher() # should be testing the method level - # avoid rep - expected_output = "hello" - launcher.start((["echo", expected_output], "/tmp")) # use time.sleep(0.1) -> we do not need sleep in other places - captured = capsys.readouterr() - output = captured.out - assert expected_output in captured.out - # do not need to build exact str, but can just have multiple assert - # verify echo hello - # make a separate test for stdout and stdin -> that test only verifies one component - # tests should do as little as possible, reduce number of constraints - - -# popen returns a non 0 when an error occurs, so test invalid path -# assert not id, might retry -> assert called 5 times, -> could verify that a warning was printed - -# should test a success cond, a failure condition \ No newline at end of file + \ No newline at end of file From f90a2856da9395c3fa37a2ada803748c9fc4a145 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 6 Aug 2024 18:47:54 -0500 Subject: [PATCH 59/82] addressing all of matts comments besides 1 --- smartsim/_core/entrypoints/file_operations.py | 4 + smartsim/_core/generation/generator.py | 77 ++---- smartsim/_core/utils/helpers.py | 13 + smartsim/entity/model.py | 1 - smartsim/experiment.py | 61 ++++- smartsim/launchable/job.py | 15 +- smartsim/launchable/jobGroup.py | 9 + smartsim/settings/dispatch.py | 5 +- tests/temp_tests/test_jobGroup.py | 23 +- tests/temp_tests/test_launchable.py | 12 + tests/test_experiment.py | 7 +- tests/test_generator.py | 233 +++++++++--------- 12 files changed, 262 insertions(+), 198 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 4271c2a63..beb68efce 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -128,11 +128,15 @@ def copy(parsed_args: argparse.Namespace) -> None: FileExistsError will be raised """ if os.path.isdir(parsed_args.source): + print("here") + print(parsed_args.source) + print(parsed_args.dest) shutil.copytree( parsed_args.source, parsed_args.dest, dirs_exist_ok=parsed_args.dirs_exist_ok, ) + print(os.listdir(parsed_args.dest)) else: shutil.copy(parsed_args.source, parsed_args.dest) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index e25262c5e..174a4d2d9 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -51,21 +51,16 @@ class Generator: files into the Job directory. """ - def __init__(self, exp_path: str, run_id: str) -> None: + def __init__(self, root: str | os.PathLike[str]) -> None: """Initialize a generator object - The Generator class is responsible for creating Job directories. + TODO The Generator class is responsible for creating Job directories. It ensures that paths adhere to SmartSim path standards. Additionally, it creates a run directory to handle symlinking, configuration, and file copying to the job directory. - - :param gen_path: Path in which files need to be generated - :param run_ID: The id of the Experiment """ - self.exp_path = pathlib.Path(exp_path) - """The path under which the experiment operate""" - self.run_id = run_id - """The runID for Experiment.start""" + self.root = root + """The root path under which to generate files""" def log_file(self, log_path: pathlib.Path) -> str: """Returns the location of the file @@ -77,14 +72,12 @@ def log_file(self, log_path: pathlib.Path) -> str: """ return join(log_path, "smartsim_params.txt") - def generate_job(self, job: Job, job_index: int) -> pathlib.Path: - """Generate the Job directory - Generate the file structure for a SmartSim Job. This - includes writing and configuring input files for the entity. + def generate_job(self, job: Job, job_path: str, log_path: str): + """Write and configure input files for a Job. To have files or directories present in the created Job - directories, such as datasets or input files, call + directory, such as datasets or input files, call ``entity.attach_generator_files`` prior to generation. Tagged application files are read, checked for input variables to @@ -92,52 +85,20 @@ def generate_job(self, job: Job, job_index: int) -> pathlib.Path: specified with a tag within the input file itself. The default tag is surronding an input value with semicolons. e.g. ``THERMO=;90;`` - + + :param job: The job instance to write and configure files for. + :param job_path: The path to the \"run\" directory for the job instance. + :param log_path: The path to the \"log\" directory for the job instance. """ - # Generate ../job_name/run directory - job_path = self._generate_job_path(job, job_index) - # Generate ../job_name/log directory - log_path = self._generate_log_path(job, job_index) # Create and write to the parameter settings file with open(self.log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - # Perform file system ops + # Perform file system operations on attached files self._build_operations(job, job_path) - # Return Job path - return job_path - - def _generate_job_path(self, job: Job, job_index: int) -> pathlib.Path: - """ - Generate the run directory for a Job. - - :param job: The Job to generate a directory - :returns pathlib.Path:: The generated run path for the Job - """ - job_type = f"{job.__class__.__name__.lower()}s" - job_path = ( - self.exp_path / self.run_id / job_type / f"{job.name}-{job_index}" / "run" - ) - # Create Job directory - job_path.mkdir(exist_ok=True, parents=True) - return job_path - - def _generate_log_path(self, job: Job, job_index: int) -> pathlib.Path: - """ - Generate the log directory for a Job. - - :param job: The Job to generate a directory - :returns pathlib.Path:: The generated log path for the Job - """ - job_type = f"{job.__class__.__name__.lower()}s" - log_path = ( - self.exp_path / self.run_id / job_type / f"{job.name}-{job_index}" / "log" - ) - log_path.mkdir(exist_ok=True, parents=True) - return log_path def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: """This method orchestrates file system ops for the attached SmartSim entity. @@ -151,7 +112,7 @@ def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: app = t.cast(Application, job.entity) self._copy_files(app.files, job_path) self._symlink_files(app.files, job_path) - self._write_tagged_files(app, job_path) + self._write_tagged_files(app.files, app.params, job_path) @staticmethod def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: @@ -217,7 +178,7 @@ def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> Non ) @staticmethod - def _write_tagged_files(app: Application, dest: pathlib.Path) -> None: + def _write_tagged_files(files: t.Union[EntityFiles, None], params: t.Mapping[str, str], dest: pathlib.Path) -> None: """Read, configure and write the tagged input files for a Job instance. This function specifically deals with the tagged files attached to an entity. @@ -226,9 +187,9 @@ def _write_tagged_files(app: Application, dest: pathlib.Path) -> None: :param dest: Path to the Jobs run directory """ # Return if no files are attached - if app.files is None: + if files is None: return - if app.files.tagged: + if files.tagged: to_write = [] def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: @@ -247,11 +208,11 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: mkdir(path.join(dest, tagged.base, path.basename(tagged_dir.base))) _build_tagged_files(tagged_dir) - if app.files.tagged_hierarchy: - _build_tagged_files(app.files.tagged_hierarchy) + if files.tagged_hierarchy: + _build_tagged_files(files.tagged_hierarchy) # Pickle the dictionary - pickled_dict = pickle.dumps(app.params) + pickled_dict = pickle.dumps(params) # Default tag delimiter tag = ";" # Encode the pickled dictionary with Base64 diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index d193b6604..af6c97c46 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -50,6 +50,19 @@ _T = t.TypeVar("_T") _TSignalHandlerFn = t.Callable[[int, t.Optional["FrameType"]], object] +def check_name(name: str) -> None: + """ + Checks if the input name is valid. + + :param name: The name to be checked. + + :raises ValueError: If the name contains the path separator (os.path.sep). + :raises ValueError: If the name is an empty string. + """ + if os.path.sep in name: + raise ValueError("Invalid input: String contains the path separator.") + if name == "": + raise ValueError("Invalid input: Name cannot be an empty string.") def unpack_fs_identifier(fs_id: str, token: str) -> t.Tuple[str, str]: """Unpack the unformatted feature store identifier diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index 1f54bf6e3..a1186cedd 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -226,7 +226,6 @@ def attach_generator_files( "`smartsim_params.txt` is a file automatically " + "generated by SmartSim and cannot be ovewritten." ) - # files is not a list of entity files self.files = EntityFiles(to_configure, to_copy, to_symlink) @property diff --git a/smartsim/experiment.py b/smartsim/experiment.py index ced006ff4..1deedb24b 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -181,10 +181,11 @@ def start(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: jobs that can be used to query or alter the status of that particular execution of the job. """ - run_id = datetime.datetime.now().strftime("run-%H:%M:%ST%Y-%m-%d") + run_id = datetime.datetime.now().replace(microsecond=0).isoformat() + root = pathlib.Path(self.exp_path, run_id) """Create the run id for Experiment.start""" return self._dispatch( - Generator(self.exp_path, run_id), dispatch.DEFAULT_DISPATCHER, *jobs + Generator(root), dispatch.DEFAULT_DISPATCHER, *jobs ) def _dispatch( @@ -196,8 +197,8 @@ def _dispatch( ) -> tuple[LaunchedJobID, ...]: """Dispatch a series of jobs with a particular dispatcher - :param generator: The Generator holds the run_id and experiment - path for use when producing job directories. + :param generator: The generator is responsible for creating the + job run and log directory. :param dispatcher: The dispatcher that should be used to determine how to start a job based on its launch settings. :param job: The first job instance to dispatch @@ -208,7 +209,6 @@ def _dispatch( """ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: - print(job) args = job.launch_settings.launch_args env = job.launch_settings.env_vars # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @@ -259,18 +259,59 @@ def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.P An instance of ``Generator`` and ``Job`` can be passed as an argument to the protected _generate member. - :param generator: Generator that holds the run_id and experiment - path for use when producing the job directory. - :param job: Job to generate file structure. - :returns: The generated Job path. + :param generator: The generator is responsible for creating the job run and log directory. + :param job: The job instance for which the output is generated. + :param job_index: The index of the job instance (used for naming). + :returns: The path to the generated output for the job instance. + :raises: A SmartSimError if an error occurs during the generation process. """ + # Generate ../job_name/run directory + job_path = self._generate_job_path(job, job_index, generator.root) + # Generate ../job_name/log directory + log_path = self._generate_log_path(job, job_index, generator.root) try: - job_path = generator.generate_job(job, job_index) + generator.generate_job(job, job_path, log_path) return job_path except SmartSimError as e: logger.error(e) raise + def _generate_job_root(self, job: Job, job_index: int, root: str) -> pathlib.Path: + """Generates the root directory for a specific job instance. + + :param job: The Job instance for which the root directory is generated. + :param job_index: The index of the Job instance (used for naming). + :returns: The path to the root directory for the Job instance. + """ + job_type = f"{job.__class__.__name__.lower()}s" + job_path = root / f"{job_type}/{job.name}-{job_index}" + job_path.mkdir(exist_ok=True, parents=True) + return job_path + + def _generate_job_path(self, job: Job, job_index: int, root: str) -> pathlib.Path: + """Generates the path for the \"run\" directory within the root directory + of a specific job instance. + + :param job (Job): The job instance for which the path is generated. + :param job_index (int): The index of the job instance (used for naming). + :returns: The path to the \"run\" directory for the job instance. + """ + path = self._generate_job_root(job, job_index, root) / "run" + path.mkdir(exist_ok=False, parents=True) + return path + + def _generate_log_path(self, job: Job, job_index: int, root: str) -> pathlib.Path: + """ + Generates the path for the \"log\" directory within the root directory of a specific job instance. + + :param job: The job instance for which the path is generated. + :param job_index: The index of the job instance (used for naming). + :returns: The path to the \"log\" directory for the job instance. + """ + path = self._generate_job_root(job, job_index, root) / "log" + path.mkdir(exist_ok=False, parents=True) + return path + def preview( self, *args: t.Any, diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index c2c8581b2..dc0f02c87 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -27,11 +27,13 @@ from __future__ import annotations import typing as t +import os from copy import deepcopy from smartsim._core.commands.launchCommands import LaunchCommands from smartsim.launchable.basejob import BaseJob from smartsim.settings import LaunchSettings +from smartsim._core.utils.helpers import check_name if t.TYPE_CHECKING: from smartsim.entity.entity import SmartSimEntity @@ -50,11 +52,12 @@ def __init__( self, entity: SmartSimEntity, launch_settings: LaunchSettings, - name: str | None = None, + name: str | None = "job", ): super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) + check_name(name) self._name = name if name else entity.name @property @@ -62,20 +65,30 @@ def name(self) -> str: """Retrieves the name of the Job.""" return self._name + @name.setter + def name(self, name: str) -> None: + """Sets the name of the Job.""" + check_name(name) + self._entity = name + @property def entity(self) -> SmartSimEntity: + """Retrieves the Job entity.""" return deepcopy(self._entity) @entity.setter def entity(self, value: SmartSimEntity) -> None: + """Sets the Job entity.""" self._entity = deepcopy(value) @property def launch_settings(self) -> LaunchSettings: + """Retrieves the Job LaunchSettings.""" return deepcopy(self._launch_settings) @launch_settings.setter def launch_settings(self, value: LaunchSettings) -> None: + """Sets the Job LaunchSettings.""" self._launch_settings = deepcopy(value) def get_launch_steps(self) -> LaunchCommands: diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index 1a92caf54..760fd5789 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -6,6 +6,8 @@ from .basejob import BaseJob from .baseJobGroup import BaseJobGroup +from .._core.utils.helpers import check_name + if t.TYPE_CHECKING: from typing_extensions import Self @@ -23,12 +25,19 @@ def __init__( ) -> None: super().__init__() self._jobs = deepcopy(jobs) + check_name(name) self._name = name @property def name(self) -> str: """Retrieves the name of the JobGroup.""" return self._name + + @name.setter + def name(self, name: str) -> None: + """Sets the name of the JobGroup.""" + check_name(name) + self._entity = name @property def jobs(self) -> t.List[BaseJob]: diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 5766c0780..47ccfe6fb 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -46,9 +46,7 @@ _T_contra = t.TypeVar("_T_contra", contravariant=True) _WorkingDirectory: TypeAlias = t.Union[str, os.PathLike[str]] -"""A type alias for a Jobs working directory. Paths may be strings or -PathLike objects. -""" +"""A working directory represented as a string or PathLike object""" _DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") """Any type of luanch arguments, typically used when the type bound by the type @@ -453,7 +451,6 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - # TODO inject path here def start( self, command: tuple[str | os.PathLike[str], t.Sequence[str]] ) -> LaunchedJobID: diff --git a/tests/temp_tests/test_jobGroup.py b/tests/temp_tests/test_jobGroup.py index b129adb8d..e77041af8 100644 --- a/tests/temp_tests/test_jobGroup.py +++ b/tests/temp_tests/test_jobGroup.py @@ -44,25 +44,34 @@ def get_launch_steps(self): raise NotImplementedError +def test_invalid_job_name(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2,wlmutils.get_test_launcher()) + with pytest.raises(ValueError): + _ = JobGroup([job_1, job_2], name="") + with pytest.raises(ValueError): + _ = JobGroup([job_1, job_2], name="name/not/allowed") + + def test_create_JobGroup(): job_1 = MockJob() job_group = JobGroup([job_1]) assert len(job_group) == 1 -def test_getitem_JobGroup(): - job_1 = Job(app_1, LaunchSettings("slurm")) - job_2 = Job(app_2, LaunchSettings("slurm")) +def test_getitem_JobGroup(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2, wlmutils.get_test_launcher()) job_group = JobGroup([job_1, job_2]) get_value = job_group[0].entity.name assert get_value == job_1.entity.name -def test_setitem_JobGroup(): - job_1 = Job(app_1, LaunchSettings("slurm")) - job_2 = Job(app_2, LaunchSettings("slurm")) +def test_setitem_JobGroup(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2, wlmutils.get_test_launcher()) job_group = JobGroup([job_1, job_2]) - job_3 = Job(app_3, LaunchSettings("slurm")) + job_3 = Job(app_3, wlmutils.get_test_launcher()) job_group[1] = job_3 assert len(job_group) == 2 get_value = job_group[1] diff --git a/tests/temp_tests/test_launchable.py b/tests/temp_tests/test_launchable.py index fed75b7d0..b3889fb67 100644 --- a/tests/temp_tests/test_launchable.py +++ b/tests/temp_tests/test_launchable.py @@ -49,6 +49,18 @@ def test_launchable_init(): launchable = Launchable() assert isinstance(launchable, Launchable) +def test_invalid_job_name(wlmutils): + entity = Application( + "test_name", + run_settings=LaunchSettings(wlmutils.get_test_launcher()), + exe="echo", + exe_args=["spam", "eggs"], + ) + settings = LaunchSettings(wlmutils.get_test_launcher()) + with pytest.raises(ValueError): + _ = Job(entity, settings, name="") + with pytest.raises(ValueError): + _ = Job(entity, settings, name="path/to/name") def test_job_init(): entity = Application( diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 474eb0aa8..6a2c20b99 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -28,14 +28,11 @@ import dataclasses import itertools -import tempfile import typing as t import uuid -import weakref import pytest -from smartsim._core.generation import Generator from smartsim.entity import _mock, entity from smartsim.experiment import Experiment from smartsim.launchable import job @@ -52,7 +49,7 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: "/tmp/job", "1") + monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: "/tmp/job") yield exp @@ -180,7 +177,6 @@ class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" def __init__(self): - path = tempfile.TemporaryDirectory() super().__init__("test-entity", _mock.Mock()) def __eq__(self, other): @@ -215,7 +211,6 @@ def test_start_can_launch_jobs( num_jobs: int, ) -> None: jobs = make_jobs(job_maker, num_jobs) - print(jobs) assert len(experiment._active_launchers) == 0, "Initialized w/ launchers" launched_ids = experiment.start(*jobs) assert len(experiment._active_launchers) == 1, "Unexpected number of launchers" diff --git a/tests/test_generator.py b/tests/test_generator.py index 04e104dc1..5f0941c09 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -4,6 +4,7 @@ from glob import glob from os import listdir from os import path as osp +import itertools import pytest @@ -20,42 +21,25 @@ @pytest.fixture -def get_gen_copy_file(fileutils): - """Fixture to yield directory to copy.""" +def get_gen_copy_dir(fileutils): yield fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) @pytest.fixture -def get_gen_symlink_file(fileutils): - """Fixture to yield directory to symlink.""" +def get_gen_symlink_dir(fileutils): yield fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) @pytest.fixture -def get_gen_configure_file(fileutils): - """Fixture to yield directory to symlink.""" +def get_gen_configure_dir(fileutils): yield fileutils.get_test_conf_path(osp.join("generator_files", "tag_dir_template")) -@pytest.fixture -def echo_app(): - """Fixture to yield an instance of SmartSimEntity.""" - yield SmartSimEntity("echo_app", run_settings=_mock.Mock()) - - @pytest.fixture def generator_instance(test_dir) -> Generator: """Fixture to create an instance of Generator.""" - experiment_path = osp.join(test_dir, "experiment_name") - yield Generator(exp_path=experiment_path, run_id="mock_run") - - -@pytest.fixture -def job_instance(wlmutils, echo_app) -> Job: - """Fixture to create an instance of Job.""" - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job = Job(echo_app, launch_settings) - return job + root = pathlib.Path(test_dir, "temp_id") + yield Generator(root=root) def test_log_file_path(generator_instance): @@ -65,131 +49,145 @@ def test_log_file_path(generator_instance): assert generator_instance.log_file(base_path) == expected_path -def test_generate_job_directory(test_dir, wlmutils): +def test_generate_job_directory(test_dir, wlmutils, generator_instance): """Test Generator.generate_job""" - # Experiment path - experiment_path = osp.join(test_dir, "experiment_name") # Create Job launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("app_name", exe="python", run_settings="RunSettings") + app = Application("app_name", exe="python", run_settings="RunSettings") # Mock RunSettings job = Job(app, launch_settings) - # Mock start id + # Mock id run_id = "mock_run" - # Generator instance - gen = Generator(exp_path=experiment_path, run_id=run_id) - # Call Generator.generate_job - job_path = gen.generate_job(job, 1) - assert isinstance(job_path, pathlib.Path) - expected_run_path = ( - pathlib.Path(experiment_path) - / run_id - / f"{job.__class__.__name__.lower()}s" - / f"{app.name}-{1}" + # Create run directory + run_path = ( + generator_instance.root / "run" ) - assert job_path == expected_run_path - expected_log_path = ( - pathlib.Path(experiment_path) - / run_id - / f"{job.__class__.__name__.lower()}s" - / f"{app.name}-{1}" + run_path.mkdir(parents=True) + assert osp.isdir(run_path) + # Create log directory + log_path = ( + generator_instance.root / "log" ) - assert osp.isdir(expected_run_path) - assert osp.isdir(expected_log_path) - assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) - - -def test_exp_private_generate_method_app(wlmutils, test_dir, generator_instance): - """Test that Job directory was created from Experiment.""" + log_path.mkdir(parents=True) + assert osp.isdir(log_path) + # Call Generator.generate_job + generator_instance.generate_job(job, run_path, log_path) + # Assert smartsim params file created + assert osp.isfile(osp.join(log_path, "smartsim_params.txt")) + # Assert smartsim params correctly written to + with open(log_path / "smartsim_params.txt", 'r') as file: + content = file.read() + assert "Generation start date and time:" in content + + +def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): + """Test that Job directory was created from Experiment._generate.""" + # Create Experiment exp = Experiment(name="experiment_name", exp_path=test_dir) - app = Application("name", "python", "RunSettings") + # Create Job + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job = Job(app, launch_settings) - job_execution_path = exp._generate(generator_instance, job, 1) + # Generate Job directory + job_index = 1 + job_execution_path = exp._generate(generator_instance, job, job_index) + # Assert Job run directory exists assert osp.isdir(job_execution_path) + # Assert Job log directory exists head, _ = os.path.split(job_execution_path) expected_log_path = pathlib.Path(head) / "log" assert osp.isdir(expected_log_path) -def test_generate_copy_file(fileutils, wlmutils, test_dir): - # Create the Job and attach generator file +def test_generate_copy_file(fileutils, wlmutils, generator_instance): + """Test that attached copy files are copied into Job directory""" + # Create the Job and attach copy generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python", "RunSettings") + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings script = fileutils.get_test_conf_path("sleep.py") app.attach_generator_files(to_copy=script) job = Job(app, launch_settings) - # Create the experiment - experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(exp_path=experiment_path, run_id="temp_run") - path = gen.generate_job(job, 1) - expected_file = pathlib.Path(path) / "sleep.py" + # Call Generator.generate_job + run_path = generator_instance.root / "run" + run_path.mkdir(parents=True) + log_path = generator_instance.root / "log" + log_path.mkdir(parents=True) + generator_instance.generate_job(job, run_path, log_path) + expected_file = run_path / "sleep.py" assert osp.isfile(expected_file) -def test_generate_copy_directory(wlmutils, test_dir, get_gen_copy_file): +def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python", "RunSettings") - app.attach_generator_files(to_copy=get_gen_copy_file) + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + app.attach_generator_files(to_copy=get_gen_copy_dir) + print(f"what is this: {get_gen_copy_dir}") job = Job(app, launch_settings) - # Create the experiment - experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(exp_path=experiment_path, run_id="temp_run") - path = gen.generate_job(job, 1) - expected_file = pathlib.Path(path) / "mock.txt" + # Call Generator.generate_job + run_path = generator_instance.root / "run" + run_path.mkdir(parents=True) + log_path = generator_instance.root / "log" + log_path.mkdir(parents=True) + generator_instance.generate_job(job, run_path, log_path) + expected_file = run_path / "mock.txt" assert osp.isfile(expected_file) -def test_generate_symlink_directory(wlmutils, test_dir, get_gen_symlink_file): +def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlink_dir): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python", "RunSettings") - # Path of directory to symlink - symlink_dir = get_gen_symlink_file + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings # Attach directory to Application - app.attach_generator_files(to_symlink=symlink_dir) + app.attach_generator_files(to_symlink=get_gen_symlink_dir) # Create Job job = Job(app, launch_settings) - # Create the experiment - experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(exp_path=experiment_path, run_id="temp_run") - # Generate Experiment file structure - job_path = gen.generate_job(job, 1) - expected_folder = pathlib.Path(job_path) / "to_symlink_dir" + # Call Generator.generate_job + run_path = generator_instance.root / "run" + run_path.mkdir(parents=True) + log_path = generator_instance.root / "log" + log_path.mkdir(parents=True) + generator_instance.generate_job(job, run_path, log_path) + expected_folder = run_path / "to_symlink_dir" assert osp.isdir(expected_folder) + assert expected_folder.is_symlink() + assert os.fspath(expected_folder.resolve()) == osp.realpath(get_gen_symlink_dir) # Combine symlinked file list and original file list for comparison - for written, correct in zip(listdir(symlink_dir), listdir(expected_folder)): + for written, correct in itertools.zip_longest(listdir(get_gen_symlink_dir), listdir(expected_folder)): # For each pair, check if the filenames are equal assert written == correct -def test_generate_symlink_file(get_gen_symlink_file, wlmutils, test_dir): +def test_generate_symlink_file(get_gen_symlink_dir, wlmutils, generator_instance): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) app = Application("name", "python", "RunSettings") # Path of directory to symlink - symlink_dir = get_gen_symlink_file + symlink_dir = get_gen_symlink_dir # Get a list of all files in the directory symlink_files = sorted(glob(symlink_dir + "/*")) # Attach directory to Application app.attach_generator_files(to_symlink=symlink_files) # Create Job job = Job(app, launch_settings) - # Create the experiment - experiment_path = osp.join(test_dir, "experiment_name") - gen = Generator(exp_path=experiment_path, run_id="mock_run") - # Generate Experiment file structure - job_path = gen.generate_job(job, 1) - expected_file = pathlib.Path(job_path) / "mock2.txt" + + # Call Generator.generate_job + run_path = generator_instance.root / "run" + run_path.mkdir(parents=True) + log_path = generator_instance.root / "log" + log_path.mkdir(parents=True) + generator_instance.generate_job(job, run_path, log_path) + expected_file = pathlib.Path(run_path) / "mock2.txt" assert osp.isfile(expected_file) + assert expected_file.is_symlink() + assert os.fspath(expected_file.resolve()) == osp.join(osp.realpath(get_gen_symlink_dir), "mock2.txt") -def test_generate_configure(fileutils, wlmutils, test_dir): +def test_generate_configure(fileutils, wlmutils, generator_instance): # Directory of files to configure conf_path = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "marked/") @@ -217,16 +215,16 @@ def test_generate_configure(fileutils, wlmutils, test_dir): app.attach_generator_files(to_configure=tagged_files) job = Job(app, launch_settings) - # Spin up Experiment - experiment_path = osp.join(test_dir, "experiment_name") - # Spin up Generator - gen = Generator(exp_path=experiment_path, run_id="temp_run") - # Execute file generation - job_path = gen.generate_job(job, 1) + # Call Generator.generate_job + run_path = generator_instance.root / "run" + run_path.mkdir(parents=True) + log_path = generator_instance.root / "log" + log_path.mkdir(parents=True) + generator_instance.generate_job(job, run_path, log_path) # Retrieve the list of configured files in the test directory - configured_files = sorted(glob(str(job_path) + "/*")) + configured_files = sorted(glob(str(run_path) + "/*")) # Use filecmp.cmp to check that the corresponding files are equal - for written, correct in zip(configured_files, correct_files): + for written, correct in itertools.zip_longest(configured_files, correct_files): assert filecmp.cmp(written, correct) @@ -236,11 +234,11 @@ def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_inst launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) - for job in job_list: - job_execution_path = exp._generate(generator_instance, job, 1) - head, _ = os.path.split(job_execution_path) + for i, job in enumerate(job_list): + job_run_path = exp._generate(generator_instance, job, i) + head, _ = os.path.split(job_run_path) expected_log_path = pathlib.Path(head) / "log" - assert osp.isdir(job_execution_path) + assert osp.isdir(job_run_path) assert osp.isdir(pathlib.Path(expected_log_path)) @@ -248,9 +246,20 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) - for job in job_list: - job_path = generator_instance.generate_job(job, 1) - assert osp.isdir(job_path) + for i, job in enumerate(job_list): + # Call Generator.generate_job + run_path = generator_instance.root / f"run-{i}" + run_path.mkdir(parents=True) + log_path = generator_instance.root / f"log-{i}" + log_path.mkdir(parents=True) + generator_instance.generate_job(job, run_path, log_path) + # Assert smartsim params file created + assert osp.isfile(osp.join(log_path, "smartsim_params.txt")) + # Assert smartsim params correctly written to + with open(log_path / "smartsim_params.txt", 'r') as file: + content = file.read() + assert "Generation start date and time:" in content + def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): @@ -273,13 +282,13 @@ def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): assert osp.isdir(log_path) -def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_file): +def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): monkeypatch.setattr( "smartsim.settings.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env: "exit", ) ensemble = Ensemble( - "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_file) + "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) ) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) @@ -294,7 +303,7 @@ def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_fi def test_generate_ensemble_symlink( - test_dir, wlmutils, monkeypatch, get_gen_symlink_file + test_dir, wlmutils, monkeypatch, get_gen_symlink_dir ): monkeypatch.setattr( "smartsim.settings.dispatch._LauncherAdapter.start", @@ -304,7 +313,7 @@ def test_generate_ensemble_symlink( "ensemble-name", "echo", replicas=2, - files=EntityFiles(symlink=get_gen_symlink_file), + files=EntityFiles(symlink=get_gen_symlink_dir), ) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) @@ -314,12 +323,14 @@ def test_generate_ensemble_symlink( jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") job_dir = listdir(jobs_dir) for ensemble_dir in job_dir: - sym_file_path = os.path.join(jobs_dir, ensemble_dir, "run", "to_symlink_dir") + sym_file_path = pathlib.Path(jobs_dir) / ensemble_dir / "run" / "to_symlink_dir" assert osp.isdir(sym_file_path) + assert sym_file_path.is_symlink() + assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) def test_generate_ensemble_configure( - test_dir, wlmutils, monkeypatch, get_gen_configure_file + test_dir, wlmutils, monkeypatch, get_gen_configure_dir ): monkeypatch.setattr( "smartsim.settings.dispatch._LauncherAdapter.start", @@ -327,7 +338,7 @@ def test_generate_ensemble_configure( ) params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} # Retrieve a list of files for configuration - tagged_files = sorted(glob(get_gen_configure_file + "/*")) + tagged_files = sorted(glob(get_gen_configure_dir + "/*")) ensemble = Ensemble( "ensemble-name", "echo", From 907a1d0201ab02f274934eae5df58562501e3b1d Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 7 Aug 2024 00:27:59 -0500 Subject: [PATCH 60/82] tests passing, matt comments addressed, styling --- smartsim/_core/entrypoints/file_operations.py | 4 -- smartsim/_core/generation/generator.py | 32 +++++++----- smartsim/_core/utils/helpers.py | 5 +- smartsim/experiment.py | 28 +++++++---- smartsim/launchable/job.py | 14 +++--- smartsim/launchable/jobGroup.py | 9 ++-- tests/temp_tests/test_jobGroup.py | 12 +++-- tests/temp_tests/test_launchable.py | 20 ++++++-- tests/test_generator.py | 50 +++++++++---------- 9 files changed, 99 insertions(+), 75 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index beb68efce..4271c2a63 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -128,15 +128,11 @@ def copy(parsed_args: argparse.Namespace) -> None: FileExistsError will be raised """ if os.path.isdir(parsed_args.source): - print("here") - print(parsed_args.source) - print(parsed_args.dest) shutil.copytree( parsed_args.source, parsed_args.dest, dirs_exist_ok=parsed_args.dirs_exist_ok, ) - print(os.listdir(parsed_args.dest)) else: shutil.copy(parsed_args.source, parsed_args.dest) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 174a4d2d9..19cbfb5c9 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -51,7 +51,7 @@ class Generator: files into the Job directory. """ - def __init__(self, root: str | os.PathLike[str]) -> None: + def __init__(self, root: os.PathLike[str]) -> None: """Initialize a generator object TODO The Generator class is responsible for creating Job directories. @@ -62,7 +62,7 @@ def __init__(self, root: str | os.PathLike[str]) -> None: self.root = root """The root path under which to generate files""" - def log_file(self, log_path: pathlib.Path) -> str: + def log_file(self, log_path: os.PathLike[str]) -> os.PathLike[str]: """Returns the location of the file summarizing the parameters used for the generation of the entity. @@ -70,10 +70,11 @@ def log_file(self, log_path: pathlib.Path) -> str: :param log_path: Path to log directory :returns: Path to file with parameter settings """ - return join(log_path, "smartsim_params.txt") + return pathlib.Path(log_path) / "smartsim_params.txt" - - def generate_job(self, job: Job, job_path: str, log_path: str): + def generate_job( + self, job: Job, job_path: os.PathLike[str], log_path: os.PathLike[str] + ) -> None: """Write and configure input files for a Job. To have files or directories present in the created Job @@ -85,7 +86,7 @@ def generate_job(self, job: Job, job_path: str, log_path: str): specified with a tag within the input file itself. The default tag is surronding an input value with semicolons. e.g. ``THERMO=;90;`` - + :param job: The job instance to write and configure files for. :param job_path: The path to the \"run\" directory for the job instance. :param log_path: The path to the \"log\" directory for the job instance. @@ -99,8 +100,7 @@ def generate_job(self, job: Job, job_path: str, log_path: str): # Perform file system operations on attached files self._build_operations(job, job_path) - - def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: + def _build_operations(self, job: Job, job_path: os.PathLike[str]) -> None: """This method orchestrates file system ops for the attached SmartSim entity. It processes three types of file system ops: to_copy, to_symlink, and to_configure. For each type, it calls the corresponding private methods that open a subprocess @@ -115,7 +115,7 @@ def _build_operations(self, job: Job, job_path: pathlib.Path) -> None: self._write_tagged_files(app.files, app.params, job_path) @staticmethod - def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: + def _copy_files(files: t.Union[EntityFiles, None], dest: os.PathLike[str]) -> None: """Perform copy file sys operations on a list of files. :param app: The Application attached to the Job @@ -126,6 +126,8 @@ def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: return for src in files.copy: if os.path.isdir(src): + base_source_name = os.path.basename(src) + new_dst_path = os.path.join(dest, base_source_name) subprocess.run( args=[ sys.executable, @@ -133,7 +135,7 @@ def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: "smartsim._core.entrypoints.file_operations", "copy", src, - dest, + new_dst_path, "--dirs_exist_ok", ] ) @@ -150,7 +152,9 @@ def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: ) @staticmethod - def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: + def _symlink_files( + files: t.Union[EntityFiles, None], dest: os.PathLike[str] + ) -> None: """Perform symlink file sys operations on a list of files. :param app: The Application attached to the Job @@ -178,7 +182,11 @@ def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> Non ) @staticmethod - def _write_tagged_files(files: t.Union[EntityFiles, None], params: t.Mapping[str, str], dest: pathlib.Path) -> None: + def _write_tagged_files( + files: t.Union[EntityFiles, None], + params: t.Mapping[str, str], + dest: os.PathLike[str], + ) -> None: """Read, configure and write the tagged input files for a Job instance. This function specifically deals with the tagged files attached to an entity. diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index af6c97c46..6f93d9419 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -50,6 +50,7 @@ _T = t.TypeVar("_T") _TSignalHandlerFn = t.Callable[[int, t.Optional["FrameType"]], object] + def check_name(name: str) -> None: """ Checks if the input name is valid. @@ -57,12 +58,10 @@ def check_name(name: str) -> None: :param name: The name to be checked. :raises ValueError: If the name contains the path separator (os.path.sep). - :raises ValueError: If the name is an empty string. """ if os.path.sep in name: raise ValueError("Invalid input: String contains the path separator.") - if name == "": - raise ValueError("Invalid input: Name cannot be an empty string.") + def unpack_fs_identifier(fs_id: str, token: str) -> t.Tuple[str, str]: """Unpack the unformatted feature store identifier diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 1deedb24b..f363dd1e2 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -184,9 +184,7 @@ def start(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: run_id = datetime.datetime.now().replace(microsecond=0).isoformat() root = pathlib.Path(self.exp_path, run_id) """Create the run id for Experiment.start""" - return self._dispatch( - Generator(root), dispatch.DEFAULT_DISPATCHER, *jobs - ) + return self._dispatch(Generator(root), dispatch.DEFAULT_DISPATCHER, *jobs) def _dispatch( self, @@ -245,7 +243,9 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: ) @_contextualize - def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.Path: + def _generate( + self, generator: Generator, job: Job, job_index: int + ) -> os.PathLike[str]: """Generate the directory and file structure for a ``Job`` ``Experiment._generate`` calls the appropriate Generator @@ -276,7 +276,9 @@ def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.P logger.error(e) raise - def _generate_job_root(self, job: Job, job_index: int, root: str) -> pathlib.Path: + def _generate_job_root( + self, job: Job, job_index: int, root: os.PathLike[str] + ) -> pathlib.Path: """Generates the root directory for a specific job instance. :param job: The Job instance for which the root directory is generated. @@ -284,11 +286,13 @@ def _generate_job_root(self, job: Job, job_index: int, root: str) -> pathlib.Pat :returns: The path to the root directory for the Job instance. """ job_type = f"{job.__class__.__name__.lower()}s" - job_path = root / f"{job_type}/{job.name}-{job_index}" + job_path = pathlib.Path(root) / f"{job_type}/{job.name}-{job_index}" job_path.mkdir(exist_ok=True, parents=True) - return job_path + return pathlib.Path(job_path) - def _generate_job_path(self, job: Job, job_index: int, root: str) -> pathlib.Path: + def _generate_job_path( + self, job: Job, job_index: int, root: os.PathLike[str] + ) -> os.PathLike[str]: """Generates the path for the \"run\" directory within the root directory of a specific job instance. @@ -298,9 +302,11 @@ def _generate_job_path(self, job: Job, job_index: int, root: str) -> pathlib.Pat """ path = self._generate_job_root(job, job_index, root) / "run" path.mkdir(exist_ok=False, parents=True) - return path + return pathlib.Path(path) - def _generate_log_path(self, job: Job, job_index: int, root: str) -> pathlib.Path: + def _generate_log_path( + self, job: Job, job_index: int, root: os.PathLike[str] + ) -> os.PathLike[str]: """ Generates the path for the \"log\" directory within the root directory of a specific job instance. @@ -310,7 +316,7 @@ def _generate_log_path(self, job: Job, job_index: int, root: str) -> pathlib.Pat """ path = self._generate_job_root(job, job_index, root) / "log" path.mkdir(exist_ok=False, parents=True) - return path + return pathlib.Path(path) def preview( self, diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index dc0f02c87..e680e5f14 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -26,14 +26,14 @@ from __future__ import annotations -import typing as t import os +import typing as t from copy import deepcopy from smartsim._core.commands.launchCommands import LaunchCommands +from smartsim._core.utils.helpers import check_name from smartsim.launchable.basejob import BaseJob from smartsim.settings import LaunchSettings -from smartsim._core.utils.helpers import check_name if t.TYPE_CHECKING: from smartsim.entity.entity import SmartSimEntity @@ -52,13 +52,13 @@ def __init__( self, entity: SmartSimEntity, launch_settings: LaunchSettings, - name: str | None = "job", + name: str | None = None, ): super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) - check_name(name) self._name = name if name else entity.name + check_name(self._name) @property def name(self) -> str: @@ -66,10 +66,10 @@ def name(self) -> str: return self._name @name.setter - def name(self, name: str) -> None: + def name(self, name: str | None) -> None: """Sets the name of the Job.""" - check_name(name) - self._entity = name + self._name = name if name else self._entity.name + check_name(self._name) @property def entity(self) -> SmartSimEntity: diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index 760fd5789..fd288deb4 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -3,11 +3,10 @@ import typing as t from copy import deepcopy +from .._core.utils.helpers import check_name from .basejob import BaseJob from .baseJobGroup import BaseJobGroup -from .._core.utils.helpers import check_name - if t.TYPE_CHECKING: from typing_extensions import Self @@ -25,19 +24,19 @@ def __init__( ) -> None: super().__init__() self._jobs = deepcopy(jobs) - check_name(name) self._name = name + check_name(self._name) @property def name(self) -> str: """Retrieves the name of the JobGroup.""" return self._name - + @name.setter def name(self, name: str) -> None: """Sets the name of the JobGroup.""" check_name(name) - self._entity = name + self._name = name @property def jobs(self) -> t.List[BaseJob]: diff --git a/tests/temp_tests/test_jobGroup.py b/tests/temp_tests/test_jobGroup.py index e77041af8..20c25d36a 100644 --- a/tests/temp_tests/test_jobGroup.py +++ b/tests/temp_tests/test_jobGroup.py @@ -46,9 +46,7 @@ def get_launch_steps(self): def test_invalid_job_name(wlmutils): job_1 = Job(app_1, wlmutils.get_test_launcher()) - job_2 = Job(app_2,wlmutils.get_test_launcher()) - with pytest.raises(ValueError): - _ = JobGroup([job_1, job_2], name="") + job_2 = Job(app_2, wlmutils.get_test_launcher()) with pytest.raises(ValueError): _ = JobGroup([job_1, job_2], name="name/not/allowed") @@ -59,6 +57,14 @@ def test_create_JobGroup(): assert len(job_group) == 1 +def test_name_setter(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2, wlmutils.get_test_launcher()) + job_group = JobGroup([job_1, job_2]) + job_group.name = "new_name" + assert job_group.name == "new_name" + + def test_getitem_JobGroup(wlmutils): job_1 = Job(app_1, wlmutils.get_test_launcher()) job_2 = Job(app_2, wlmutils.get_test_launcher()) diff --git a/tests/temp_tests/test_launchable.py b/tests/temp_tests/test_launchable.py index b3889fb67..16fba6cff 100644 --- a/tests/temp_tests/test_launchable.py +++ b/tests/temp_tests/test_launchable.py @@ -49,19 +49,19 @@ def test_launchable_init(): launchable = Launchable() assert isinstance(launchable, Launchable) + def test_invalid_job_name(wlmutils): entity = Application( "test_name", - run_settings=LaunchSettings(wlmutils.get_test_launcher()), + run_settings="RunSettings", exe="echo", exe_args=["spam", "eggs"], - ) + ) # Mock RunSettings settings = LaunchSettings(wlmutils.get_test_launcher()) - with pytest.raises(ValueError): - _ = Job(entity, settings, name="") with pytest.raises(ValueError): _ = Job(entity, settings, name="path/to/name") + def test_job_init(): entity = Application( "test_name", @@ -77,6 +77,18 @@ def test_job_init(): assert "eggs" in job.entity.exe_args +def test_name_setter(): + entity = Application( + "test_name", + run_settings=LaunchSettings("slurm"), + exe="echo", + exe_args=["spam", "eggs"], + ) + job = Job(entity, LaunchSettings("slurm")) + job.name = "new_name" + assert job.name == "new_name" + + def test_job_init_deepcopy(): entity = Application( "test_name", diff --git a/tests/test_generator.py b/tests/test_generator.py index 5f0941c09..c8dc95b04 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -1,10 +1,10 @@ import filecmp +import itertools import os import pathlib from glob import glob from os import listdir from os import path as osp -import itertools import pytest @@ -46,29 +46,25 @@ def test_log_file_path(generator_instance): """Test if the log_file function returns the correct log path.""" base_path = "/tmp" expected_path = osp.join(base_path, "smartsim_params.txt") - assert generator_instance.log_file(base_path) == expected_path + assert generator_instance.log_file(base_path) == pathlib.Path(expected_path) -def test_generate_job_directory(test_dir, wlmutils, generator_instance): +def test_generate_job_directory(wlmutils, generator_instance): """Test Generator.generate_job""" # Create Job launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("app_name", exe="python", run_settings="RunSettings") # Mock RunSettings + app = Application( + "app_name", exe="python", run_settings="RunSettings" + ) # Mock RunSettings job = Job(app, launch_settings) # Mock id run_id = "mock_run" # Create run directory - run_path = ( - generator_instance.root - / "run" - ) + run_path = generator_instance.root / "run" run_path.mkdir(parents=True) assert osp.isdir(run_path) # Create log directory - log_path = ( - generator_instance.root - / "log" - ) + log_path = generator_instance.root / "log" log_path.mkdir(parents=True) assert osp.isdir(log_path) # Call Generator.generate_job @@ -76,7 +72,7 @@ def test_generate_job_directory(test_dir, wlmutils, generator_instance): # Assert smartsim params file created assert osp.isfile(osp.join(log_path, "smartsim_params.txt")) # Assert smartsim params correctly written to - with open(log_path / "smartsim_params.txt", 'r') as file: + with open(log_path / "smartsim_params.txt", "r") as file: content = file.read() assert "Generation start date and time:" in content @@ -86,7 +82,7 @@ def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): # Create Experiment exp = Experiment(name="experiment_name", exp_path=test_dir) # Create Job - app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job = Job(app, launch_settings) # Generate Job directory @@ -104,7 +100,7 @@ def test_generate_copy_file(fileutils, wlmutils, generator_instance): """Test that attached copy files are copied into Job directory""" # Create the Job and attach copy generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings script = fileutils.get_test_conf_path("sleep.py") app.attach_generator_files(to_copy=script) job = Job(app, launch_settings) @@ -122,9 +118,8 @@ def test_generate_copy_file(fileutils, wlmutils, generator_instance): def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings app.attach_generator_files(to_copy=get_gen_copy_dir) - print(f"what is this: {get_gen_copy_dir}") job = Job(app, launch_settings) # Call Generator.generate_job @@ -133,14 +128,14 @@ def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance) log_path = generator_instance.root / "log" log_path.mkdir(parents=True) generator_instance.generate_job(job, run_path, log_path) - expected_file = run_path / "mock.txt" - assert osp.isfile(expected_file) + expected_file = run_path / "to_copy_dir" + assert osp.isdir(expected_file) def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlink_dir): # Create the Job and attach generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings # Attach directory to Application app.attach_generator_files(to_symlink=get_gen_symlink_dir) # Create Job @@ -157,7 +152,9 @@ def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlin assert expected_folder.is_symlink() assert os.fspath(expected_folder.resolve()) == osp.realpath(get_gen_symlink_dir) # Combine symlinked file list and original file list for comparison - for written, correct in itertools.zip_longest(listdir(get_gen_symlink_dir), listdir(expected_folder)): + for written, correct in itertools.zip_longest( + listdir(get_gen_symlink_dir), listdir(expected_folder) + ): # For each pair, check if the filenames are equal assert written == correct @@ -184,7 +181,9 @@ def test_generate_symlink_file(get_gen_symlink_dir, wlmutils, generator_instance expected_file = pathlib.Path(run_path) / "mock2.txt" assert osp.isfile(expected_file) assert expected_file.is_symlink() - assert os.fspath(expected_file.resolve()) == osp.join(osp.realpath(get_gen_symlink_dir), "mock2.txt") + assert os.fspath(expected_file.resolve()) == osp.join( + osp.realpath(get_gen_symlink_dir), "mock2.txt" + ) def test_generate_configure(fileutils, wlmutils, generator_instance): @@ -256,10 +255,9 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): # Assert smartsim params file created assert osp.isfile(osp.join(log_path, "smartsim_params.txt")) # Assert smartsim params correctly written to - with open(log_path / "smartsim_params.txt", 'r') as file: + with open(log_path / "smartsim_params.txt", "r") as file: content = file.read() assert "Generation start date and time:" in content - def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): @@ -298,8 +296,8 @@ def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_di jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") job_dir = listdir(jobs_dir) for ensemble_dir in job_dir: - sym_file_path = os.path.join(jobs_dir, ensemble_dir, "run", "mock.txt") - assert osp.isfile(sym_file_path) + copy_folder_path = os.path.join(jobs_dir, ensemble_dir, "run", "to_copy_dir") + assert osp.isdir(copy_folder_path) def test_generate_ensemble_symlink( From 69c9f2d57521d8c973aec0363488b1504a86d1f6 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 7 Aug 2024 10:59:36 -0500 Subject: [PATCH 61/82] styling --- smartsim/_core/generation/generator.py | 15 +++++++++------ smartsim/_core/launcher/dragon/dragonLauncher.py | 1 - smartsim/entity/entity.py | 1 - smartsim/experiment.py | 8 +++----- tests/test_generator.py | 4 ++-- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 19cbfb5c9..72552638e 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -52,12 +52,12 @@ class Generator: """ def __init__(self, root: os.PathLike[str]) -> None: - """Initialize a generator object + """Initialize a Generator object - TODO The Generator class is responsible for creating Job directories. - It ensures that paths adhere to SmartSim path standards. Additionally, - it creates a run directory to handle symlinking, - configuration, and file copying to the job directory. + The class handles symlinking, copying, and configuration of files + associated with a Jobs entity. Additionally, it writes entity parameters + used for the specific run into the "smartsim_params.txt" settings file within + the Jobs log folder. """ self.root = root """The root path under which to generate files""" @@ -102,7 +102,7 @@ def generate_job( def _build_operations(self, job: Job, job_path: os.PathLike[str]) -> None: """This method orchestrates file system ops for the attached SmartSim entity. - It processes three types of file system ops: to_copy, to_symlink, and to_configure. + It processes three types of file system operations: to_copy, to_symlink, and to_configure. For each type, it calls the corresponding private methods that open a subprocess to complete each task. @@ -126,8 +126,11 @@ def _copy_files(files: t.Union[EntityFiles, None], dest: os.PathLike[str]) -> No return for src in files.copy: if os.path.isdir(src): + # Remove basename of source base_source_name = os.path.basename(src) + # Attach source basename to destination new_dst_path = os.path.join(dest, base_source_name) + # Copy source contents to new destination path subprocess.run( args=[ sys.executable, diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 288939d2b..30dbbeacb 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -374,7 +374,6 @@ def _as_run_request_args_and_policy( # this will need to be injected by the user or by us to have # the command execute next to any generated files. A similar # problem exists for the other settings. - # TODO: Find a way to inject this path path=path, env=env, # TODO: Not sure how this info is injected diff --git a/smartsim/entity/entity.py b/smartsim/entity/entity.py index 8f0ca73b8..8c4bd4e4f 100644 --- a/smartsim/entity/entity.py +++ b/smartsim/entity/entity.py @@ -106,7 +106,6 @@ def __init__(self, name: str, run_settings: "RunSettings") -> None: share these attributes. :param name: Name of the entity - :param path: path to output, error, and configuration files """ self.name = name self.run_settings = run_settings diff --git a/smartsim/experiment.py b/smartsim/experiment.py index f363dd1e2..d405e328f 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -182,8 +182,9 @@ def start(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: particular execution of the job. """ run_id = datetime.datetime.now().replace(microsecond=0).isoformat() + """Create the run id""" root = pathlib.Path(self.exp_path, run_id) - """Create the run id for Experiment.start""" + """Generate the root path""" return self._dispatch(Generator(root), dispatch.DEFAULT_DISPATCHER, *jobs) def _dispatch( @@ -248,16 +249,13 @@ def _generate( ) -> os.PathLike[str]: """Generate the directory and file structure for a ``Job`` - ``Experiment._generate`` calls the appropriate Generator - function to create a directory for the passed job. - If files or directories are attached to an ``application`` object associated with the Job using ``application.attach_generator_files()``, those files or directories will be symlinked, copied, or configured and written into the created job directory An instance of ``Generator`` and ``Job`` can be passed as an argument to - the protected _generate member. + the protected _generate member, as well as the Jobs index. :param generator: The generator is responsible for creating the job run and log directory. :param job: The job instance for which the output is generated. diff --git a/tests/test_generator.py b/tests/test_generator.py index c8dc95b04..e33a44b57 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -128,8 +128,8 @@ def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance) log_path = generator_instance.root / "log" log_path.mkdir(parents=True) generator_instance.generate_job(job, run_path, log_path) - expected_file = run_path / "to_copy_dir" - assert osp.isdir(expected_file) + expected_folder = run_path / "to_copy_dir" + assert osp.isdir(expected_folder) def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlink_dir): From 98cb30d83b4e59a5339bfc7785d419cd4fe574c8 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 12 Aug 2024 00:24:17 -0500 Subject: [PATCH 62/82] push changes --- smartsim/_core/generation/generator.py | 21 ++- smartsim/experiment.py | 9 +- smartsim/settings/arguments/launch/alps.py | 2 +- smartsim/settings/arguments/launch/local.py | 3 +- smartsim/settings/arguments/launch/lsf.py | 2 +- smartsim/settings/arguments/launch/mpi.py | 6 +- smartsim/settings/arguments/launch/pals.py | 2 +- smartsim/settings/arguments/launch/slurm.py | 2 +- smartsim/settings/dispatch.py | 32 ++-- .../test_settings/test_alpsLauncher.py | 9 +- .../test_settings/test_localLauncher.py | 9 +- .../test_settings/test_lsfLauncher.py | 11 +- .../test_settings/test_mpiLauncher.py | 7 +- .../test_settings/test_palsLauncher.py | 11 +- .../test_settings/test_slurmLauncher.py | 11 +- tests/test_generator.py | 15 +- tests/test_shell_launcher.py | 141 ++++++++++-------- 17 files changed, 192 insertions(+), 101 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 83b38be23..829cbf12a 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -71,6 +71,12 @@ def log_file(self, log_path: os.PathLike[str]) -> os.PathLike[str]: :returns: Path to file with parameter settings """ return pathlib.Path(log_path) / "smartsim_params.txt" + + def output_files(self, log_path: os.PathLike[str], job_name: str) -> None: + out_file_path = log_path / (job_name + ".out") + err_file_path = log_path / (job_name + ".err") + return out_file_path, err_file_path + def generate_job( self, job: Job, job_path: os.PathLike[str], log_path: os.PathLike[str] @@ -96,9 +102,23 @@ def generate_job( with open(self.log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") + + # Create output files + out_file, err_file = self.output_files(log_path, job.entity.name) + + # Open and write to .out file + with open(out_file, mode="w", encoding="utf-8") as log_file: + dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + log_file.write(f"Generation start date and time: {dt_string}\n") + # Open and write to .err file + with open(err_file, mode="w", encoding="utf-8") as log_file: + dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + log_file.write(f"Generation start date and time: {dt_string}\n") + # Perform file system operations on attached files self._build_operations(job, job_path) + return out_file, err_file def _build_operations(self, job: Job, job_path: os.PathLike[str]) -> None: """This method orchestrates file system ops for the attached SmartSim entity. @@ -124,7 +144,6 @@ def _copy_files(files: t.Union[EntityFiles, None], dest: os.PathLike[str]) -> No # Return if no files are attached if files is None: return - print(f"type is defined as: {type(files)}") for src in files.copy: if os.path.isdir(src): # Remove basename of source diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 18ddd2692..8bbc6430f 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -157,7 +157,6 @@ def __init__(self, name: str, exp_path: str | None = None): exp_path = osp.abspath(exp_path) else: exp_path = osp.join(getcwd(), name) - print("got here") self.exp_path = exp_path """The path under which the experiment operate""" @@ -236,8 +235,8 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: # pylint: disable-next=protected-access self._active_launchers.add(launch_config._adapted_launcher) # Generate the job directory and return the generated job path - job_execution_path = self._generate(generator, job, idx) - return launch_config.start(exe, job_execution_path, env) + job_execution_path, out, err = self._generate(generator, job, idx) + return launch_config.start(exe, job_execution_path, env, out, err) return execute_dispatch(generator, job, 0), *( execute_dispatch(generator, job, idx) for idx, job in enumerate(jobs, 1) @@ -268,8 +267,8 @@ def _generate( # Generate ../job_name/log directory log_path = self._generate_log_path(job, job_index, generator.root) try: - generator.generate_job(job, job_path, log_path) - return job_path + out, err = generator.generate_job(job, job_path, log_path) + return job_path, out, err except SmartSimError as e: logger.error(e) raise diff --git a/smartsim/settings/arguments/launch/alps.py b/smartsim/settings/arguments/launch/alps.py index 1879dd102..120293398 100644 --- a/smartsim/settings/arguments/launch/alps.py +++ b/smartsim/settings/arguments/launch/alps.py @@ -36,7 +36,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_aprun_command = make_shell_format_fn(run_command="aprun") +_as_aprun_command = make_shell_format_fn(run_command="aprun", out_flag="hold", err_flag="hold") @dispatch(with_format=_as_aprun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/local.py b/smartsim/settings/arguments/launch/local.py index 0bbba2584..44cbb6ade 100644 --- a/smartsim/settings/arguments/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations +from subprocess import PIPE import typing as t @@ -36,7 +37,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_local_command = make_shell_format_fn(run_command=None) +_as_local_command = make_shell_format_fn(run_command=None, out_flag="hold", err_flag="hold") @dispatch(with_format=_as_local_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index 80cd748f1..7659ef4ce 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -36,7 +36,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_jsrun_command = make_shell_format_fn(run_command="jsrun") +_as_jsrun_command = make_shell_format_fn(run_command="jsrun", out_flag="--stdio_stdout", err_flag="--stdio_stderr") @dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/mpi.py b/smartsim/settings/arguments/launch/mpi.py index 85fd38145..1cc03793b 100644 --- a/smartsim/settings/arguments/launch/mpi.py +++ b/smartsim/settings/arguments/launch/mpi.py @@ -36,9 +36,9 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_mpirun_command = make_shell_format_fn("mpirun") -_as_mpiexec_command = make_shell_format_fn("mpiexec") -_as_orterun_command = make_shell_format_fn("orterun") +_as_mpirun_command = make_shell_format_fn("mpirun", out_flag="hold", err_flag="hold") +_as_mpiexec_command = make_shell_format_fn("mpiexec", out_flag="hold", err_flag="hold") +_as_orterun_command = make_shell_format_fn("orterun", out_flag="hold", err_flag="hold") class _BaseMPILaunchArguments(LaunchArguments): diff --git a/smartsim/settings/arguments/launch/pals.py b/smartsim/settings/arguments/launch/pals.py index 3132f1b02..839fdd422 100644 --- a/smartsim/settings/arguments/launch/pals.py +++ b/smartsim/settings/arguments/launch/pals.py @@ -36,7 +36,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_pals_command = make_shell_format_fn(run_command="mpiexec") +_as_pals_command = make_shell_format_fn(run_command="mpiexec", out_flag="hold", err_flag="hold") @dispatch(with_format=_as_pals_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index ac485b7c8..accc27b94 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -38,7 +38,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_srun_command = make_shell_format_fn(run_command="srun") +_as_srun_command = make_shell_format_fn(run_command="srun", out_flag="--output", err_flag="--error") @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 41c55fc67..4fd91a35e 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -38,6 +38,9 @@ from smartsim._core.utils import helpers from smartsim.error import errors from smartsim.types import LaunchedJobID +from smartsim._core.commands import Command, CommandList +from ..settings.launchCommand import LauncherType +from subprocess import STDOUT if t.TYPE_CHECKING: from smartsim.experiment import Experiment @@ -63,14 +66,14 @@ a job """ _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType], + [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType, str, str], _LaunchableT, ] """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ _LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType]" + "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType, str, str]" ) """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings @@ -262,8 +265,10 @@ def format_( exe: ExecutableProtocol, path: str | os.PathLike[str], env: _EnvironMappingType, + out: str, + err: str, ) -> _LaunchableT: - return self.formatter(arguments, exe, path, env) + return self.formatter(arguments, exe, path, env, out, err) return _LauncherAdapter(launcher, format_) @@ -395,7 +400,7 @@ def create(cls, exp: Experiment, /) -> Self: ... def make_shell_format_fn( - run_command: str | None, + run_command: str | None, out_flag = str | None, err_flag = str | None, ) -> _FormatterType[LaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]]]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -425,14 +430,15 @@ def make_shell_format_fn( :returns: A function to format an arguments, an executable, and an environment as a shell launchable sequence for strings. """ - def impl( args: LaunchArguments, exe: ExecutableProtocol, path: str | os.PathLike[str], - _env: _EnvironMappingType, + env: _EnvironMappingType, + out_file: str, + err_file: str, ) -> t.Tuple[str | os.PathLike[str], t.Sequence[str]]: - return path, ( + command_tuple = ( ( run_command, *(args.format_launch_args() or ()), @@ -442,6 +448,10 @@ def impl( if run_command is not None else exe.as_program_arguments() ) + stdin = out_flag + "=" + str(out_file) + stdout = err_flag + "=" + str(err_file) + launchable_args = [env, path, stdin, stdout, command_tuple] + return launchable_args return impl @@ -456,14 +466,10 @@ def start( self, command: tuple[str | os.PathLike[str], t.Sequence[str]] ) -> LaunchedJobID: id_ = create_job_id() - path, args = command + env, path, stdin, stdout, args = command exe, *rest = args - print(f"first value: {(helpers.expand_exe_path(exe), *rest)}") - print(f"second value: {path}") # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path, env={}, stdin=None, stdout=None) - print(f"did this work: {self._launched[id_]}") - print("you got here") + self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), stdin, *rest), cwd=path, env=env) # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 370b67db7..8e78da7f9 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -211,8 +211,13 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_aprun_command( - AprunLaunchArguments(args), mock_echo_executable, test_dir, {} + outfile = "out.txt" + errfile = "err.txt" + env, path, stdin, stdout, cmd = _as_aprun_command( + AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) assert tuple(cmd) == expected assert path == test_dir + assert env == {} + assert stdin == f"hold={outfile}" + assert stdout == f"hold={errfile}" diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 48de0e7b5..777b205b9 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -143,8 +143,13 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): - path, cmd = _as_local_command( - LocalLaunchArguments({}), mock_echo_executable, test_dir, {} + outfile = "out.txt" + errfile = "err.txt" + env, path, stdin, stdout, cmd = _as_local_command( + LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, outfile, errfile ) assert tuple(cmd) == ("echo", "hello", "world") assert path == test_dir + assert env == {} + assert stdin == f"hold={outfile}" + assert stdout == f"hold={errfile}" diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index eec915860..55801862e 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -120,8 +120,13 @@ def test_launch_args(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_jsrun_command( - JsrunLaunchArguments(args), mock_echo_executable, test_dir, {} + outfile = "out.txt" + errfile = "err.txt" + env, path, stdin, stdout, args = _as_jsrun_command( + JsrunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) - assert tuple(cmd) == expected + assert tuple(args) == expected assert path == test_dir + assert env == {} + assert stdin == f"--stdio_stdout={outfile}" + assert stdout == f"--stdio_stderr={errfile}" diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index ff5200eca..254065d50 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -286,6 +286,11 @@ def test_invalid_hostlist_format(launcher): def test_formatting_launch_args( mock_echo_executable, cls, fmt, cmd, args, expected, test_dir ): - path, fmt_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}) + outfile = "out.txt" + errfile = "err.txt" + env, path, stdin, stdout, fmt_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, outfile, errfile) assert tuple(fmt_cmd) == (cmd,) + expected assert path == test_dir + assert env == {} + assert stdin == f"hold={outfile}" + assert stdout == f"hold={errfile}" diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 64b9dc7f1..690922007 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -132,8 +132,13 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_pals_command( - PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {} + outfile = "out.txt" + errfile = "err.txt" + env, path, stdin, stdout, args = _as_pals_command( + PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) - assert tuple(cmd) == expected + assert tuple(args) == expected assert path == test_dir + assert env == {} + assert stdin == f"hold={outfile}" + assert stdout == f"hold={errfile}" diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 1c21e3d01..bc7ad23a5 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -317,8 +317,13 @@ def test_set_het_groups(monkeypatch): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_srun_command( - SlurmLaunchArguments(args), mock_echo_executable, test_dir, {} + outfile = "out.txt" + errfile = "err.txt" + env, path, stdin, stdout, args = _as_srun_command( + SlurmLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) - assert tuple(cmd) == expected + assert tuple(args) == expected assert path == test_dir + assert env == {} + assert stdin == f"--output={outfile}" + assert stdout == f"--error={errfile}" diff --git a/tests/test_generator.py b/tests/test_generator.py index e33a44b57..9ec6d6358 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -49,6 +49,16 @@ def test_log_file_path(generator_instance): assert generator_instance.log_file(base_path) == pathlib.Path(expected_path) +def test_output_files(generator_instance): + """Test if the log_file function returns the correct log path.""" + log_path = pathlib.Path("/tmp") + expected_out_path = osp.join(log_path, "name.out") + expected_err_path = osp.join(log_path, "name.err") + out, err = generator_instance.output_files(log_path, "name") + assert out == pathlib.Path(expected_out_path) + assert err == pathlib.Path(expected_err_path) + + def test_generate_job_directory(wlmutils, generator_instance): """Test Generator.generate_job""" # Create Job @@ -75,7 +85,10 @@ def test_generate_job_directory(wlmutils, generator_instance): with open(log_path / "smartsim_params.txt", "r") as file: content = file.read() assert "Generation start date and time:" in content - + expected_out_path = osp.join(log_path, (job.entity.name + ".out")) + expected_err_path = osp.join(log_path, (job.entity.name + ".err")) + assert osp.isfile(expected_out_path) + assert osp.isfile(expected_err_path) def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): """Test that Job directory was created from Experiment._generate.""" diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index c28c2a46b..541f38938 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -28,7 +28,10 @@ import unittest.mock import pytest import time +import pathlib import weakref +import datetime +import subprocess from smartsim.entity import _mock, entity, Application from smartsim import Experiment from smartsim.settings import LaunchSettings @@ -36,7 +39,8 @@ SlurmLaunchArguments, _as_srun_command, ) -from smartsim.settings.dispatch import sp as dsp +from smartsim._core.utils import helpers +from smartsim.settings.dispatch import sp from smartsim.settings.dispatch import ShellLauncher from smartsim.settings.launchCommand import LauncherType from smartsim.launchable import Job @@ -56,6 +60,7 @@ def __init__(self): self._finalizer = weakref.finalize(self, path.cleanup) super().__init__("test-entity", _mock.Mock()) self.files = Files() + self.params = {} def __eq__(self, other): if type(self) is not type(other): @@ -63,30 +68,14 @@ def __eq__(self, other): return self.as_program_arguments() == other.as_program_arguments() def as_program_arguments(self): - return ("/usr/bin/echo", "Hello", "World!") - #return ("/usr/bin/sleep", "10") + return (helpers.expand_exe_path("echo"), "Hello", "World!") class Files(): + """Represents a collection of files with different attrs for Mock entity""" def __init__(self): self.copy = [] self.link = [] self.tagged = [] - - -# what is the success criteria -def test_shell_as_py(capsys): - # a unit test should init the obj bc testing that unit of code - launcher = ShellLauncher() # should be testing the method level - # avoid rep - expected_output = "hello" - launcher.start((["echo", expected_output], "/tmp")) # use time.sleep(0.1) -> we do not need sleep in other places - captured = capsys.readouterr() - output = captured.out - assert expected_output in captured.out - # do not need to build exact str, but can just have multiple assert - # verify echo hello - # make a separate test for stdout and stdin -> that test only verifies one component - # tests should do as little as possible, reduce number of constraints # popen returns a non 0 when an error occurs, so test invalid path @@ -104,44 +93,78 @@ def test_shell_launcher_init(): # Assert that private attribute is expected value assert shell_launcher._launched == {} -# test that the process leading up to the shell launcher was corrected, integration test -# my test is identifying the change in the code -def test_shell_launcher_calls_popen(test_dir: str, monkeypatch: pytest.MonkeyPatch): - # monkeypatch the popen - # create a Mock popen object - # def my_mock_popen(*args, **kwargs): - # print("foo") - # no longer care about the internals, only want to know that the process up to it was currect - mock_popen_obj = unittest.mock.MagicMock() - with monkeypatch.context() as ctx: - ctx.setattr(dsp, "Popen", mock_popen_obj) - - # mock2 = unittest.mock.MagicMock(return_value=0) # same as monkeypatch - implements getproperty or API that looks for a unknown prop on an obj - # mock3 = unittest.mock.MagicMock() - # # Avoid actual network request - # mock3.Popen = mock2 - # mock3.return_value = mock2 - env_vars = { - "LOGGING": "verbose", - } - slurm_settings = LaunchSettings(launcher=LauncherType.Slurm, env_vars=env_vars) - slurm_settings.launch_args.set_nodes(1) - job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=slurm_settings) + +def test_shell_launcher_calls_popen(test_dir: str): + """Test that the process leading up to the shell launcher popen call was corrected""" + job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) exp = Experiment(name="exp_name", exp_path=test_dir) - # can validate id here -> could build another mock that ensures that 22 is the pid - id = exp.start(job) - # mock2.assert_called_once_with( - # ('/usr/bin/srun', '--nodes=1', '--', '/usr/bin/echo', 'Hello', 'World!'), - # cwd=unittest.mock.ANY, - # env={}, - # stdin=None, - # stdout=None - # ) - # mock_popen_obj.assert_called() - #mock3.assert_called_with() # the process executed the correct launcher - # write something that makes sure the job has completed b4 the test exits - print(id) - #time.sleep(5) # TODO remove once blocking is added - # asyn = concurrent, not happening in another thread, not happening somewhere else - # focus on async io in python, make sure that anything that is io bound is async - \ No newline at end of file + # Setup mock for Popen class from the smartsim.settings.dispatch.sp module + # to temporarily replace the actual Popen class with a mock version + with unittest.mock.patch("smartsim.settings.dispatch.sp.Popen") as mock_open: + # Assign a mock value of 12345 to the process id attr of the mocked Popen object + mock_open.pid = unittest.mock.MagicMock(return_value=12345) + # Assign a mock return value of 0 to the returncode attr of the mocked Popen object + mock_open.returncode = unittest.mock.MagicMock(return_value=0) + # Execute Experiment.start + _ = exp.start(job) + # Assert that the mock_open object was called during the execution of the Experiment.start + mock_open.assert_called_once() + +def test_shell_launcher_calls_popen_with_value(test_dir: str): + """Test that popen was called with correct types""" + job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) + exp = Experiment(name="exp_name", exp_path=test_dir) + # Setup mock for Popen class from the smartsim.settings.dispatch.sp module + # to temporarily replace the actual Popen class with a mock version + with unittest.mock.patch("smartsim.settings.dispatch.sp.Popen") as mock_open: + # Assign a mock value of 12345 to the pid attr of the mocked Popen object + mock_open.pid = unittest.mock.MagicMock(return_value=12345) + # Assign a mock return value of 0 to the returncode attr of the mocked Popen object + mock_open.returncode = unittest.mock.MagicMock(return_value=0) + _ = exp.start(job) + # Assert that the mock_open object was called during the execution of the Experiment.start with value + mock_open.assert_called_once_with( + (helpers.expand_exe_path("srun"), '--', helpers.expand_exe_path("echo"), 'Hello', 'World!'), + cwd=unittest.mock.ANY, + env={}, + stdin=unittest.mock.ANY, + stdout=unittest.mock.ANY, + ) + +def test_this(test_dir: str): + """Test that popen was called with correct types""" + job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) + exp = Experiment(name="exp_name", exp_path=test_dir) + _ = exp.start(job) + + + + + + + + + + + + +# write something that makes sure the job has completed b4 the test exits +# print(id) +#time.sleep(5) # TODO remove once blocking is added +# asyn = concurrent, not happening in another thread, not happening somewhere else +# focus on async io in python, make sure that anything that is io bound is async + +# what is the success criteria +# def test_shell_as_py(capsys): +# # a unit test should init the obj bc testing that unit of code +# launcher = ShellLauncher() # should be testing the method level +# # avoid rep +# expected_output = "hello" +# launcher.start((["echo", expected_output], "/tmp")) # use time.sleep(0.1) -> we do not need sleep in other places +# captured = capsys.readouterr() +# output = captured.out +# assert expected_output in captured.out + # do not need to build exact str, but can just have multiple assert + # verify echo hello + # make a separate test for stdout and stdin -> that test only verifies one component + # tests should do as little as possible, reduce number of constraints \ No newline at end of file From dbe9ca77472d68f738148a9ce437710fe370e4a8 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 13 Aug 2024 12:28:00 -0500 Subject: [PATCH 63/82] pushing to switch branches - failing --- smartsim/_core/generation/generator.py | 3 ++- smartsim/experiment.py | 5 +++-- smartsim/settings/dispatch.py | 2 ++ tests/test_shell_launcher.py | 31 ++++++++++++++++++++++---- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 829cbf12a..4f7aaa99a 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -105,11 +105,12 @@ def generate_job( # Create output files out_file, err_file = self.output_files(log_path, job.entity.name) - + print(out_file) # Open and write to .out file with open(out_file, mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") + print(out_file.is_file()) # Open and write to .err file with open(err_file, mode="w", encoding="utf-8") as log_file: diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 8bbc6430f..eb80e7a89 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -235,7 +235,8 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: # pylint: disable-next=protected-access self._active_launchers.add(launch_config._adapted_launcher) # Generate the job directory and return the generated job path - job_execution_path, out, err = self._generate(generator, job, idx) + ret = self._generate(generator, job, idx) + job_execution_path, out, err = ret return launch_config.start(exe, job_execution_path, env, out, err) return execute_dispatch(generator, job, 0), *( @@ -268,7 +269,7 @@ def _generate( log_path = self._generate_log_path(job, job_index, generator.root) try: out, err = generator.generate_job(job, job_path, log_path) - return job_path, out, err + return (job_path, out, err) except SmartSimError as e: logger.error(e) raise diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 4fd91a35e..555174870 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -449,6 +449,7 @@ def impl( else exe.as_program_arguments() ) stdin = out_flag + "=" + str(out_file) + print(stdin) stdout = err_flag + "=" + str(err_file) launchable_args = [env, path, stdin, stdout, command_tuple] return launchable_args @@ -469,6 +470,7 @@ def start( env, path, stdin, stdout, args = command exe, *rest = args # pylint: disable-next=consider-using-with + print((helpers.expand_exe_path(exe), stdin, *rest)) self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), stdin, *rest), cwd=path, env=env) # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 541f38938..07b7c4ee4 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -29,6 +29,7 @@ import pytest import time import pathlib +import uuid import weakref import datetime import subprocess @@ -76,7 +77,25 @@ def __init__(self): self.copy = [] self.link = [] self.tagged = [] - + + +@pytest.fixture +def experiment(monkeypatch, test_dir): + """Fixture for creating an Experiment instance with a unique name and run directory + for testing. + """ + exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) + # Generate run directory + run_dir = pathlib.Path(test_dir) / "tmp" + run_dir.mkdir(exist_ok=True, parents=True) + # Generate out / err files + out_file = run_dir / "tmp.out" + err_file = run_dir / "tmp.err" + out_file.touch() + err_file.touch() + # MonkeyPatch Experiment._generate + monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: (run_dir, out_file, err_file)) + yield exp # popen returns a non 0 when an error occurs, so test invalid path # assert not id, might retry -> assert called 5 times, -> could verify that a warning was printed @@ -131,11 +150,15 @@ def test_shell_launcher_calls_popen_with_value(test_dir: str): stdout=unittest.mock.ANY, ) -def test_this(test_dir: str): +def test_this(experiment, test_dir): """Test that popen was called with correct types""" job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) - exp = Experiment(name="exp_name", exp_path=test_dir) - _ = exp.start(job) + _ = experiment.start(job) + run_dir = pathlib.Path(test_dir) / "tmp" / "tmp.out" + time.sleep(5) + with open(run_dir, 'r', encoding='utf-8') as file: + print(list({line.strip() for line in file.readlines()})) + From 9b152e08a5e223f46c090a67de799593b49269c5 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 13 Aug 2024 14:01:33 -0500 Subject: [PATCH 64/82] pushing changes --- smartsim/settings/arguments/launch/slurm.py | 3 +++ smartsim/settings/dispatch.py | 19 ++++++++++++------- tests/test_shell_launcher.py | 20 +++++++++++++++----- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index accc27b94..f43e1e894 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -38,8 +38,11 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) +# instead of the makeshell format -> do not here _as_srun_command = make_shell_format_fn(run_command="srun", out_flag="--output", err_flag="--error") +# def as_srun_cmd(): + @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) class SlurmLaunchArguments(LaunchArguments): diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 555174870..2eaed061e 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -448,10 +448,11 @@ def impl( if run_command is not None else exe.as_program_arguments() ) - stdin = out_flag + "=" + str(out_file) - print(stdin) - stdout = err_flag + "=" + str(err_file) - launchable_args = [env, path, stdin, stdout, command_tuple] + stdout = out_flag + "=" + str(out_file) + print(stdout) + stderr = err_flag + "=" + str(err_file) + print(command_tuple) + launchable_args = [env, path, stdout, stderr, command_tuple] return launchable_args return impl @@ -467,11 +468,15 @@ def start( self, command: tuple[str | os.PathLike[str], t.Sequence[str]] ) -> LaunchedJobID: id_ = create_job_id() - env, path, stdin, stdout, args = command + env, path, stdout, stderr, args = command + print(env) + print(path) + print(stdout) + print(stderr) + print(args) exe, *rest = args # pylint: disable-next=consider-using-with - print((helpers.expand_exe_path(exe), stdin, *rest)) - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), stdin, *rest), cwd=path, env=env) + self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path, env=env, stdout=stdout, stderr=stderr) # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 07b7c4ee4..a3843bede 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -153,11 +153,21 @@ def test_shell_launcher_calls_popen_with_value(test_dir: str): def test_this(experiment, test_dir): """Test that popen was called with correct types""" job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) - _ = experiment.start(job) - run_dir = pathlib.Path(test_dir) / "tmp" / "tmp.out" - time.sleep(5) - with open(run_dir, 'r', encoding='utf-8') as file: - print(list({line.strip() for line in file.readlines()})) + shell_launcher = ShellLauncher() + # Generate run directory + run_dir = pathlib.Path(test_dir) / "tmp" + run_dir.mkdir(exist_ok=True, parents=True) + # Generate out / err files + out_file = run_dir / "tmp.out" + err_file = run_dir / "tmp.err" + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + id = shell_launcher.start([{},run_dir,out,err,('srun', '--', '/usr/bin/echo', 'Hello', 'World!')]) + proc = shell_launcher._launched[id] + #ret_code = proc.wait() + assert proc.wait() == 0 + with open(out_file, "r", encoding="utf-8") as out: + assert out.read() == "Hello World!\n" + From 082f91742b9ae16ec06233b93f612048db2d1910 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 13 Aug 2024 16:58:37 -0500 Subject: [PATCH 65/82] pushing to switch branches, not passing --- smartsim/_core/commands/command.py | 15 +------- smartsim/settings/dispatch.py | 15 ++------ tests/test_shell_launcher.py | 59 ++++++++++++++++++++---------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index d89aa41ad..1b279d116 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -27,24 +27,14 @@ import typing as t from collections.abc import MutableSequence -from ...settings.launchCommand import LauncherType - class Command(MutableSequence[str]): """Basic container for command information""" - def __init__(self, launcher: LauncherType, command: t.List[str]) -> None: + def __init__(self, command: t.List[str]) -> None: """Command constructor""" - self._launcher = launcher self._command = command - @property - def launcher(self) -> LauncherType: - """Get the launcher type. - Return a reference to the LauncherType. - """ - return self._launcher - @property def command(self) -> t.List[str]: """Get the command list. @@ -73,6 +63,5 @@ def insert(self, idx: int, value: str) -> None: self._command.insert(idx, value) def __str__(self) -> str: # pragma: no cover - string = f"\nLauncher: {self.launcher.value}\n" - string += f"Command: {' '.join(str(cmd) for cmd in self.command)}" + string = f"\nCommand: {' '.join(str(cmd) for cmd in self.command)}" return string diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 2eaed061e..17dda1fc6 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -448,12 +448,10 @@ def impl( if run_command is not None else exe.as_program_arguments() ) - stdout = out_flag + "=" + str(out_file) - print(stdout) - stderr = err_flag + "=" + str(err_file) - print(command_tuple) - launchable_args = [env, path, stdout, stderr, command_tuple] - return launchable_args + stdout = "hold" + stderr = "hold" + cmd = Command([env, path, stdout, stderr, command_tuple]) + return cmd return impl @@ -469,11 +467,6 @@ def start( ) -> LaunchedJobID: id_ = create_job_id() env, path, stdout, stderr, args = command - print(env) - print(path) - print(stdout) - print(stderr) - print(args) exe, *rest = args # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path, env=env, stdout=stdout, stderr=stderr) diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index a3843bede..3f30381b0 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -27,12 +27,10 @@ import tempfile import unittest.mock import pytest -import time import pathlib +import os import uuid import weakref -import datetime -import subprocess from smartsim.entity import _mock, entity, Application from smartsim import Experiment from smartsim.settings import LaunchSettings @@ -40,6 +38,7 @@ SlurmLaunchArguments, _as_srun_command, ) +from smartsim._core.commands import Command from smartsim._core.utils import helpers from smartsim.settings.dispatch import sp from smartsim.settings.dispatch import ShellLauncher @@ -113,10 +112,12 @@ def test_shell_launcher_init(): assert shell_launcher._launched == {} -def test_shell_launcher_calls_popen(test_dir: str): +def test_shell_launcher_calls_popen(): """Test that the process leading up to the shell launcher popen call was corrected""" - job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) - exp = Experiment(name="exp_name", exp_path=test_dir) + # Init ShellLauncher + shell_launcher = ShellLauncher() + # Mock command passed to ShellLauncher.start + cmd = Command(["env_vars", "run_dir", "out_file_path", "err_file_path", EchoHelloWorldEntity().as_program_arguments()]) # Setup mock for Popen class from the smartsim.settings.dispatch.sp module # to temporarily replace the actual Popen class with a mock version with unittest.mock.patch("smartsim.settings.dispatch.sp.Popen") as mock_open: @@ -125,11 +126,11 @@ def test_shell_launcher_calls_popen(test_dir: str): # Assign a mock return value of 0 to the returncode attr of the mocked Popen object mock_open.returncode = unittest.mock.MagicMock(return_value=0) # Execute Experiment.start - _ = exp.start(job) + _ = shell_launcher.start(cmd) # Assert that the mock_open object was called during the execution of the Experiment.start mock_open.assert_called_once() -def test_shell_launcher_calls_popen_with_value(test_dir: str): +def test_this(test_dir: str): """Test that popen was called with correct types""" job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) exp = Experiment(name="exp_name", exp_path=test_dir) @@ -150,22 +151,42 @@ def test_shell_launcher_calls_popen_with_value(test_dir: str): stdout=unittest.mock.ANY, ) -def test_this(experiment, test_dir): - """Test that popen was called with correct types""" - job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) +def create_directory(directory_path) -> pathlib.Path: + """Creates the execution directory for testing.""" + tmp_dir = pathlib.Path(directory_path) + tmp_dir.mkdir(exist_ok=True, parents=True) + return tmp_dir + +def generate_output_files(tmp_dir): + """Generates output and error files within the run directory for testing.""" + out_file = tmp_dir / "tmp.out" + err_file = tmp_dir / "tmp.err" + return out_file, err_file + +def generate_directory(test_dir): + """Generates a execution directory, output file, and error file for testing.""" + execution_dir = create_directory(os.path.join(test_dir, "/tmp")) + out_file, err_file = generate_output_files(execution_dir) + return execution_dir, out_file, err_file + +def test_popen_writes_to_out(test_dir): + """TODO""" + # Init ShellLauncher shell_launcher = ShellLauncher() - # Generate run directory - run_dir = pathlib.Path(test_dir) / "tmp" - run_dir.mkdir(exist_ok=True, parents=True) - # Generate out / err files - out_file = run_dir / "tmp.out" - err_file = run_dir / "tmp.err" + # Generate testing directory + run_dir, out_file, err_file = generate_directory(test_dir) with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - id = shell_launcher.start([{},run_dir,out,err,('srun', '--', '/usr/bin/echo', 'Hello', 'World!')]) + # Construct a command to execute + cmd = Command([{}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()]) + # Start the execution of the command using a ShellLauncher + id = shell_launcher.start(cmd) + # Retrieve the process associated with the launched command proc = shell_launcher._launched[id] - #ret_code = proc.wait() + # Check successful execution assert proc.wait() == 0 + # Reopen out_file in read mode with open(out_file, "r", encoding="utf-8") as out: + # Assert that the content of the output file is expected assert out.read() == "Hello World!\n" From 335758e1df4a63715d94a11060d5ea27d7ca3224 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 15 Aug 2024 14:55:20 -0500 Subject: [PATCH 66/82] update to test suite, does not pass, pushing to save --- smartsim/experiment.py | 4 +- smartsim/settings/arguments/launch/slurm.py | 16 +- smartsim/settings/dispatch.py | 27 +- .../test_core/test_commands/test_command.py | 13 +- .../test_commands/test_commandList.py | 6 +- .../test_commands/test_launchCommands.py | 6 +- .../test_settings/test_slurmLauncher.py | 16 +- tests/test_shell_launcher.py | 295 ++++++++++-------- 8 files changed, 219 insertions(+), 164 deletions(-) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index c74dcde00..f10f34c15 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -33,6 +33,7 @@ import os import os.path as osp import pathlib +import datetime import textwrap import typing as t from os import environ, getcwd @@ -230,7 +231,8 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: launch_config = dispatch.create_new_launcher_configuration( for_experiment=self, with_arguments=args ) - id_ = launch_config.start(exe, env) + job_execution_path = self._generate(generator, job, idx) + id_ = launch_config.start(exe, job_execution_path, env) # Save the underlying launcher instance and launched job id. That # way we do not need to spin up a launcher instance for each # individual job, and the experiment can monitor job statuses. diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index f43e1e894..9fbe6dcb5 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -32,16 +32,26 @@ from smartsim.log import get_logger from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn +from smartsim._core.commands import Command +from smartsim.settings.dispatch import ExecutableProtocol, _FormatterType, _EnvironMappingType from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArguments import LaunchArguments logger = get_logger(__name__) -# instead of the makeshell format -> do not here -_as_srun_command = make_shell_format_fn(run_command="srun", out_flag="--output", err_flag="--error") -# def as_srun_cmd(): +def _as_srun_command(args: LaunchArguments, + exe: ExecutableProtocol, + path: str | os.PathLike[str], + _env: _EnvironMappingType) -> _FormatterType[LaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]]]: + command_tuple = ( + "srun", + *(args.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + ) + return Command(["test"]) @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 79d9d9bb3..a2f2c4439 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -494,21 +494,30 @@ def impl( return impl +class ShellLauncherCommand(t.NamedTuple): + env: str + path: str + stdout: str + stderr: str + command_tuple: tuple + class ShellLauncher: """Mock launcher for launching/tracking simple shell commands""" + # add a def check def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - + # covariant, contravariant, + boliscoff substitution princ def start( - self, command: tuple[str | os.PathLike[str], t.Sequence[str]] + self, shell_command: ShellLauncherCommand # this should be a named tuple ) -> LaunchedJobID: id_ = create_job_id() - env, path, stdout, stderr, args = command - exe, *rest = args + # raise ValueError -> invalid stuff throw + exe, *rest = shell_command.command_tuple + expanded_exe = helpers.expand_exe_path(exe) # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path, env=env, stdout=stdout, stderr=stderr) + self._launched[id_] = sp.Popen((expanded_exe, *rest), cwd=shell_command.path, env=shell_command.env, stdout=shell_command.stdout, stderr=shell_command.stderr) # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ @@ -521,10 +530,12 @@ def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: if (proc := self._launched.get(id_)) is None: msg = f"Launcher `{self}` has not launched a job with id `{id_}`" raise errors.LauncherJobNotFound(msg) - ret_code = proc.poll() + ret_code = proc.poll() # add a test that mocks out poll and raise some exception - terminal -> import subprocess -> start something echo blah - then poll and see what a valid fake output is + print(ret_code) + # try/catch around here and then reaise a smartsim.error if ret_code is None: - status = psutil.Process(proc.pid).status() - return { + status = psutil.Process(proc.pid).status() # TODO can mock this, put this into a parameterized test, when you put that in the mock thing, the correct thing comes out + return {#1st arg, 2nd arg in the param tests, need to solve branching problems, do an assertion psutil.STATUS_RUNNING: JobStatus.RUNNING, psutil.STATUS_SLEEPING: JobStatus.RUNNING, psutil.STATUS_WAKING: JobStatus.RUNNING, diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 71b1b87ff..c7643335f 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -27,25 +27,23 @@ import pytest from smartsim._core.commands.command import Command -from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a def test_command_init(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] - assert cmd.launcher == LauncherType.Slurm def test_command_getitem(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) get_value = cmd[0] assert get_value == "salloc" def test_command_setitem(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) cmd[0] = "srun" cmd[1] = "-n" assert cmd.command == ["srun", "-n", "1"] @@ -53,7 +51,6 @@ def test_command_setitem(): def test_command_delitem(): cmd = Command( - launcher=LauncherType.Slurm, command=["salloc", "-N", "1", "--constraint", "P100"], ) del cmd.command[3] @@ -62,11 +59,11 @@ def test_command_delitem(): def test_command_len(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) assert len(cmd) is 3 def test_command_insert(): - cmd = Command(launcher=LauncherType.Slurm, command=["-N", "1"]) + cmd = Command(command=["-N", "1"]) cmd.insert(0, "salloc") assert cmd.command == ["salloc", "-N", "1"] diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 1a8c25179..a30107b99 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -32,9 +32,9 @@ pytestmark = pytest.mark.group_a -salloc_cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) -srun_cmd = Command(launcher=LauncherType.Slurm, command=["srun", "-n", "1"]) -sacct_cmd = Command(launcher=LauncherType.Slurm, command=["sacct", "--user"]) +salloc_cmd = Command(command=["salloc", "-N", "1"]) +srun_cmd = Command(command=["srun", "-n", "1"]) +sacct_cmd = Command(command=["sacct", "--user"]) def test_command_init(): diff --git a/tests/temp_tests/test_core/test_commands/test_launchCommands.py b/tests/temp_tests/test_core/test_commands/test_launchCommands.py index 913de208b..0c5e719cc 100644 --- a/tests/temp_tests/test_core/test_commands/test_launchCommands.py +++ b/tests/temp_tests/test_core/test_commands/test_launchCommands.py @@ -33,9 +33,9 @@ pytestmark = pytest.mark.group_a -pre_cmd = Command(launcher=LauncherType.Slurm, command=["pre", "cmd"]) -launch_cmd = Command(launcher=LauncherType.Slurm, command=["launch", "cmd"]) -post_cmd = Command(launcher=LauncherType.Slurm, command=["post", "cmd"]) +pre_cmd = Command(command=["pre", "cmd"]) +launch_cmd = Command(command=["launch", "cmd"]) +post_cmd = Command(command=["post", "cmd"]) pre_commands_list = CommandList(commands=[pre_cmd]) launch_command_list = CommandList(commands=[launch_cmd]) post_command_list = CommandList(commands=[post_cmd]) diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index bc7ad23a5..9b9cbd5cd 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -31,6 +31,7 @@ _as_srun_command, ) from smartsim.settings.launchCommand import LauncherType +from smartsim._core.commands import Command pytestmark = pytest.mark.group_a @@ -319,11 +320,10 @@ def test_set_het_groups(monkeypatch): def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): outfile = "out.txt" errfile = "err.txt" - env, path, stdin, stdout, args = _as_srun_command( - SlurmLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile - ) - assert tuple(args) == expected - assert path == test_dir - assert env == {} - assert stdin == f"--output={outfile}" - assert stdout == f"--error={errfile}" + test = _as_srun_command(args=SlurmLaunchArguments(args), exe=mock_echo_executable, path=test_dir, _env={}) + assert isinstance(test, Command) + # assert tuple(args) == expected + # assert path == test_dir + # assert env == {} + # assert stdin == f"--output={outfile}" + # assert stdout == f"--error={errfile}" diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 3f30381b0..132f3afee 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -28,6 +28,7 @@ import unittest.mock import pytest import pathlib +import difflib import os import uuid import weakref @@ -38,29 +39,27 @@ SlurmLaunchArguments, _as_srun_command, ) +from smartsim.status import JobStatus +from smartsim._core.utils.shell import * from smartsim._core.commands import Command from smartsim._core.utils import helpers -from smartsim.settings.dispatch import sp -from smartsim.settings.dispatch import ShellLauncher +from smartsim.settings.dispatch import sp, ShellLauncher, ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType from smartsim.launchable import Job from smartsim.types import LaunchedJobID -# always start with unit tests, first test shell launcher init -# make sure to test passing invalid values to shell launcher, and correct values -# verify the simple assumptions -# give each test a doc string, add comments to isolate inline +from smartsim.error.errors import LauncherJobNotFound + +# TODO tests bad vars in Popen call at beginning + # tests -> helper.exe : pass in None, empty str, path with a space at beginning, a non valid command + # -> write a test for the invalid num of items - test_shell_launcher_fails_on_any_invalid_len_input + # -> have border tests for 0,1,4,6 cmd vals -> work correctly without them -> raise ValueError + # do all of the failures as well as the sucess criteria -# how can I write a good test suite without being brittle - separating unit tests, group tests -# unit tests first, integration tests next, do not rely on external behav in tests class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" def __init__(self): - path = tempfile.TemporaryDirectory() - self._finalizer = weakref.finalize(self, path.cleanup) super().__init__("test-entity", _mock.Mock()) - self.files = Files() - self.params = {} def __eq__(self, other): if type(self) is not type(other): @@ -70,155 +69,191 @@ def __eq__(self, other): def as_program_arguments(self): return (helpers.expand_exe_path("echo"), "Hello", "World!") -class Files(): - """Represents a collection of files with different attrs for Mock entity""" - def __init__(self): - self.copy = [] - self.link = [] - self.tagged = [] +def create_directory(directory_path: str) -> pathlib.Path: + """Creates the execution directory for testing.""" + tmp_dir = pathlib.Path(directory_path) + tmp_dir.mkdir(exist_ok=True, parents=True) + return tmp_dir +def generate_output_files(tmp_dir: pathlib.Path): + """Generates output and error files within the run directory for testing.""" + out_file = tmp_dir / "tmp.out" + err_file = tmp_dir / "tmp.err" + return out_file, err_file -@pytest.fixture -def experiment(monkeypatch, test_dir): - """Fixture for creating an Experiment instance with a unique name and run directory - for testing. - """ - exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) - # Generate run directory - run_dir = pathlib.Path(test_dir) / "tmp" - run_dir.mkdir(exist_ok=True, parents=True) - # Generate out / err files - out_file = run_dir / "tmp.out" - err_file = run_dir / "tmp.err" - out_file.touch() - err_file.touch() - # MonkeyPatch Experiment._generate - monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: (run_dir, out_file, err_file)) - yield exp - -# popen returns a non 0 when an error occurs, so test invalid path -# assert not id, might retry -> assert called 5 times, -> could verify that a warning was printed - -# should test a success cond, a failure condition +def generate_directory(test_dir: str): + """Generates a execution directory, output file, and error file for testing.""" + execution_dir = create_directory(os.path.join(test_dir, "tmp")) + out_file, err_file = generate_output_files(execution_dir) + return execution_dir, out_file, err_file +@pytest.fixture +def shell_cmd(test_dir: str) -> ShellLauncherCommand: + """Fixture to create an instance of Generator.""" + run_dir, out_file, err_file = generate_directory(test_dir) + return ShellLauncherCommand({}, run_dir, out_file, err_file, EchoHelloWorldEntity().as_program_arguments()) # UNIT TESTS +def test_shell_launcher_command_init(shell_cmd: ShellLauncherCommand, test_dir: str): + """Test that ShellLauncherCommand initializes correctly""" + assert shell_cmd.env == {} + assert shell_cmd.path == pathlib.Path(test_dir) / "tmp" + assert shell_cmd.stdout == shell_cmd.path / "tmp.out" + assert shell_cmd.stderr == shell_cmd.path / "tmp.err" + assert shell_cmd.command_tuple == EchoHelloWorldEntity().as_program_arguments() + def test_shell_launcher_init(): - """A simple test to validate that ShellLauncher correctly initializes""" - # Init ShellLauncher + """Test that ShellLauncher initializes correctly""" shell_launcher = ShellLauncher() - # Assert that private attribute is expected value assert shell_launcher._launched == {} - -def test_shell_launcher_calls_popen(): - """Test that the process leading up to the shell launcher popen call was corrected""" - # Init ShellLauncher +def test_shell_launcher_start_calls_popen(shell_cmd: ShellLauncherCommand): + """Test that the process leading up to the shell launcher popen call was correct""" shell_launcher = ShellLauncher() - # Mock command passed to ShellLauncher.start - cmd = Command(["env_vars", "run_dir", "out_file_path", "err_file_path", EchoHelloWorldEntity().as_program_arguments()]) - # Setup mock for Popen class from the smartsim.settings.dispatch.sp module - # to temporarily replace the actual Popen class with a mock version with unittest.mock.patch("smartsim.settings.dispatch.sp.Popen") as mock_open: - # Assign a mock value of 12345 to the process id attr of the mocked Popen object - mock_open.pid = unittest.mock.MagicMock(return_value=12345) - # Assign a mock return value of 0 to the returncode attr of the mocked Popen object - mock_open.returncode = unittest.mock.MagicMock(return_value=0) - # Execute Experiment.start - _ = shell_launcher.start(cmd) - # Assert that the mock_open object was called during the execution of the Experiment.start + _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once() -def test_this(test_dir: str): - """Test that popen was called with correct types""" - job = Job(name="jobs", entity=EchoHelloWorldEntity(), launch_settings=LaunchSettings(launcher=LauncherType.Slurm)) - exp = Experiment(name="exp_name", exp_path=test_dir) - # Setup mock for Popen class from the smartsim.settings.dispatch.sp module - # to temporarily replace the actual Popen class with a mock version +def test_shell_launcher_start_calls_popen_with_value(shell_cmd: ShellLauncherCommand): + """Test that popen was called with correct values""" + shell_launcher = ShellLauncher() with unittest.mock.patch("smartsim.settings.dispatch.sp.Popen") as mock_open: - # Assign a mock value of 12345 to the pid attr of the mocked Popen object - mock_open.pid = unittest.mock.MagicMock(return_value=12345) - # Assign a mock return value of 0 to the returncode attr of the mocked Popen object - mock_open.returncode = unittest.mock.MagicMock(return_value=0) - _ = exp.start(job) - # Assert that the mock_open object was called during the execution of the Experiment.start with value + _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once_with( - (helpers.expand_exe_path("srun"), '--', helpers.expand_exe_path("echo"), 'Hello', 'World!'), - cwd=unittest.mock.ANY, - env={}, - stdin=unittest.mock.ANY, - stdout=unittest.mock.ANY, + shell_cmd.command_tuple, + cwd=shell_cmd.path, + env=shell_cmd.env, + stdout=shell_cmd.stdout, + stderr=shell_cmd.stderr, ) -def create_directory(directory_path) -> pathlib.Path: - """Creates the execution directory for testing.""" - tmp_dir = pathlib.Path(directory_path) - tmp_dir.mkdir(exist_ok=True, parents=True) - return tmp_dir - -def generate_output_files(tmp_dir): - """Generates output and error files within the run directory for testing.""" - out_file = tmp_dir / "tmp.out" - err_file = tmp_dir / "tmp.err" - return out_file, err_file +def test_popen_returns_popen_object(test_dir: str): + """Test that the popen call returns a popen object""" + shell_launcher = ShellLauncher() + run_dir, out_file, err_file = generate_directory(test_dir) + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + assert isinstance(proc, sp.Popen) -def generate_directory(test_dir): - """Generates a execution directory, output file, and error file for testing.""" - execution_dir = create_directory(os.path.join(test_dir, "/tmp")) - out_file, err_file = generate_output_files(execution_dir) - return execution_dir, out_file, err_file -def test_popen_writes_to_out(test_dir): - """TODO""" - # Init ShellLauncher +def test_popen_writes_to_output_file(test_dir: str): + """Test that popen writes to .out file upon successful process call""" shell_launcher = ShellLauncher() - # Generate testing directory run_dir, out_file, err_file = generate_directory(test_dir) with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - # Construct a command to execute - cmd = Command([{}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()]) - # Start the execution of the command using a ShellLauncher + cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) id = shell_launcher.start(cmd) - # Retrieve the process associated with the launched command proc = shell_launcher._launched[id] - # Check successful execution + # Wait for subprocess to finish assert proc.wait() == 0 - # Reopen out_file in read mode + assert proc.returncode == 0 with open(out_file, "r", encoding="utf-8") as out: - # Assert that the content of the output file is expected assert out.read() == "Hello World!\n" - - - - - - - - - + with open(err_file, "r", encoding="utf-8") as err: + assert err.read() == "" +def test_popen_fails_with_invalid_cmd(test_dir): + """Test that popen returns a non zero returncode after failure""" + shell_launcher = ShellLauncher() + run_dir, out_file, err_file = generate_directory(test_dir) + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + args = (helpers.expand_exe_path("srun"), "--flag_dne") + cmd = ShellLauncherCommand({}, run_dir, out, err, args) + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + proc.wait() + assert proc.returncode != 0 + with open(out_file, "r", encoding="utf-8") as out: + assert out.read() == "" + with open(err_file, "r", encoding="utf-8") as err: + content = err.read() + assert "unrecognized option" in content + + +def test_popen_issues_unique_ids(test_dir): + """Validate that all ids are unique within ShellLauncher._launched""" + shell_launcher = ShellLauncher() + run_dir, out_file, err_file = generate_directory(test_dir) + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + for _ in range(5): + _ = shell_launcher.start(cmd) + assert len(shell_launcher._launched) == 5 +def test_retrieve_status_dne(): + """Test tht ShellLauncher returns the status of completed Jobs""" + # Init ShellLauncher + shell_launcher = ShellLauncher() + with pytest.raises(LauncherJobNotFound): + _ = shell_launcher.get_status("dne") -# write something that makes sure the job has completed b4 the test exits -# print(id) -#time.sleep(5) # TODO remove once blocking is added -# asyn = concurrent, not happening in another thread, not happening somewhere else -# focus on async io in python, make sure that anything that is io bound is async -# what is the success criteria -# def test_shell_as_py(capsys): -# # a unit test should init the obj bc testing that unit of code -# launcher = ShellLauncher() # should be testing the method level -# # avoid rep -# expected_output = "hello" -# launcher.start((["echo", expected_output], "/tmp")) # use time.sleep(0.1) -> we do not need sleep in other places -# captured = capsys.readouterr() -# output = captured.out -# assert expected_output in captured.out - # do not need to build exact str, but can just have multiple assert - # verify echo hello - # make a separate test for stdout and stdin -> that test only verifies one component - # tests should do as little as possible, reduce number of constraints \ No newline at end of file +def test_shell_launcher_returns_complete_status(test_dir): + """Test tht ShellLauncher returns the status of completed Jobs""" + shell_launcher = ShellLauncher() + run_dir, out_file, err_file = generate_directory(test_dir) + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + for _ in range(5): + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + proc.wait() + code = shell_launcher.get_status(id) + val = list(code.keys())[0] + assert code[val] == JobStatus.COMPLETED + +def test_shell_launcher_returns_failed_status(test_dir): + """Test tht ShellLauncher returns the status of completed Jobs""" + # Init ShellLauncher + shell_launcher = ShellLauncher() + # Generate testing directory + run_dir, out_file, err_file = generate_directory(test_dir) + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + # Construct a invalid command to execute + args = (helpers.expand_exe_path("srun"), "--flag_dne") + cmd = Command([{}, run_dir, out, err, args]) + # Start the execution of the command using a ShellLauncher + for _ in range(5): + id = shell_launcher.start(cmd) + # Retrieve popen object + proc = shell_launcher._launched[id] + # Wait for subprocess to complete + proc.wait() + # Retrieve status of subprocess + code = shell_launcher.get_status(id) + val = list(code.keys())[0] + # Assert that subprocess has completed + assert code[val] == JobStatus.FAILED + + +def test_shell_launcher_returns_running_status(test_dir): + """Test tht ShellLauncher returns the status of completed Jobs""" + # Init ShellLauncher + shell_launcher = ShellLauncher() + # Generate testing directory + run_dir, out_file, err_file = generate_directory(test_dir) + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + # Construct a command to execute + cmd = Command([{}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5")]) + # Start the execution of the command using a ShellLauncher + for _ in range(5): + id = shell_launcher.start(cmd) + # Retrieve status of subprocess + code = shell_launcher.get_status(id) + val = list(code.keys())[0] + # Assert that subprocess has completed + assert code[val] == JobStatus.RUNNING + + +# TODO one test that verifies the mapping in status, verify every single one, do not execute, just mock + +def test_this(monkeypatch): + shell_launcher = ShellLauncher() + shell_launcher._launched = {"test":sp.Popen} + monkeypatch.setattr(sp.Popen, "poll", lambda: "/") + shell_launcher.get_status("test") \ No newline at end of file From 5b28902e9ade7c2f5f11dc7425fb5bff49d30c22 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 19 Aug 2024 18:41:14 -0500 Subject: [PATCH 67/82] failing because a change in types --- smartsim/_core/generation/generator.py | 90 ++++++++---- smartsim/experiment.py | 124 ++++------------ smartsim/settings/arguments/launch/alps.py | 2 +- smartsim/settings/arguments/launch/local.py | 2 +- smartsim/settings/arguments/launch/lsf.py | 2 +- smartsim/settings/arguments/launch/mpi.py | 6 +- smartsim/settings/arguments/launch/pals.py | 2 +- smartsim/settings/arguments/launch/slurm.py | 16 ++- smartsim/settings/dispatch.py | 52 +++---- .../test_settings/test_alpsLauncher.py | 9 +- .../test_settings/test_localLauncher.py | 14 +- .../test_settings/test_mpiLauncher.py | 14 +- .../test_settings/test_palsLauncher.py | 14 +- .../test_settings/test_slurmLauncher.py | 15 +- tests/test_experiment.py | 12 +- tests/test_generator.py | 135 ++++++++---------- tests/test_shell_launcher.py | 43 ++++-- 17 files changed, 272 insertions(+), 280 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 4f7aaa99a..6486e6ec5 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -51,7 +51,7 @@ class Generator: files into the Job directory. """ - def __init__(self, root: os.PathLike[str]) -> None: + def __init__(self, root: pathlib.Path) -> None: """Initialize a Generator object The class handles symlinking, copying, and configuration of files @@ -62,7 +62,43 @@ def __init__(self, root: os.PathLike[str]) -> None: self.root = root """The root path under which to generate files""" - def log_file(self, log_path: os.PathLike[str]) -> os.PathLike[str]: + def _generate_job_root(self, job: Job, job_index: int) -> pathlib.Path: + """Generates the root directory for a specific job instance. + + :param job: The Job instance for which the root directory is generated. + :param job_index: The index of the Job instance (used for naming). + :returns: The path to the root directory for the Job instance. + """ + job_type = f"{job.__class__.__name__.lower()}s" + job_path = self.root / f"{job_type}/{job.name}-{job_index}" + return pathlib.Path(job_path) + + def _generate_run_path(self, job: Job, job_index: int) -> pathlib.Path: + """Generates the path for the "run" directory within the root directory + of a specific Job instance. + + :param job (Job): The Job instance for which the path is generated. + :param job_index (int): The index of the Job instance (used for naming). + :returns: The path to the "run" directory for the Job instance. + """ + path = self._generate_job_root(job, job_index) / "run" + path.mkdir(exist_ok=False, parents=True) + return pathlib.Path(path) + + def _generate_log_path(self, job: Job, job_index: int) -> pathlib.Path: + """ + Generates the path for the "log" directory within the root directory of a specific Job instance. + + :param job: The Job instance for which the path is generated. + :param job_index: The index of the Job instance (used for naming). + :returns: The path to the "log" directory for the Job instance. + """ + path = self._generate_job_root(job, job_index) / "log" + path.mkdir(exist_ok=False, parents=True) + return pathlib.Path(path) + + @staticmethod + def _log_file(log_path: pathlib.Path) -> pathlib.Path: """Returns the location of the file summarizing the parameters used for the generation of the entity. @@ -71,16 +107,13 @@ def log_file(self, log_path: os.PathLike[str]) -> os.PathLike[str]: :returns: Path to file with parameter settings """ return pathlib.Path(log_path) / "smartsim_params.txt" - - def output_files(self, log_path: os.PathLike[str], job_name: str) -> None: + + def _output_files(self, log_path: pathlib.Path, job_name: str) -> t.Tuple[pathlib.Path, pathlib.Path]: out_file_path = log_path / (job_name + ".out") err_file_path = log_path / (job_name + ".err") return out_file_path, err_file_path - - def generate_job( - self, job: Job, job_path: os.PathLike[str], log_path: os.PathLike[str] - ) -> None: + def generate_job(self, job: Job, job_index: int) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: """Write and configure input files for a Job. To have files or directories present in the created Job @@ -90,38 +123,43 @@ def generate_job( Tagged application files are read, checked for input variables to configure, and written. Input variables to configure are specified with a tag within the input file itself. - The default tag is surronding an input value with semicolons. + The default tag is surrounding an input value with semicolons. e.g. ``THERMO=;90;`` :param job: The job instance to write and configure files for. - :param job_path: The path to the \"run\" directory for the job instance. - :param log_path: The path to the \"log\" directory for the job instance. + :param job_path: The path to the "run" directory for the job instance. + :param log_path: The path to the "log" directory for the job instance. """ + # Generate ../job_name/run directory + job_path = self._generate_run_path(job, job_index) + # Generate ../job_name/log directory + log_path = self._generate_log_path(job, job_index) + # Create and write to the parameter settings file - with open(self.log_file(log_path), mode="w", encoding="utf-8") as log_file: + with open(self._log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") # Create output files - out_file, err_file = self.output_files(log_path, job.entity.name) - print(out_file) + out_file, err_file = self._output_files(log_path, job.entity.name) # Open and write to .out file with open(out_file, mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - print(out_file.is_file()) # Open and write to .err file with open(err_file, mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - + # Perform file system operations on attached files self._build_operations(job, job_path) - return out_file, err_file - def _build_operations(self, job: Job, job_path: os.PathLike[str]) -> None: + return job_path, out_file, err_file + + @classmethod + def _build_operations(cls, job: Job, job_path: pathlib.Path) -> None: """This method orchestrates file system ops for the attached SmartSim entity. It processes three types of file system operations: to_copy, to_symlink, and to_configure. For each type, it calls the corresponding private methods that open a subprocess @@ -131,12 +169,12 @@ def _build_operations(self, job: Job, job_path: os.PathLike[str]) -> None: :param job_path: Path to the Jobs run directory """ app = t.cast(Application, job.entity) - self._copy_files(app.files, job_path) - self._symlink_files(app.files, job_path) - self._write_tagged_files(app.files, app.params, job_path) + cls._copy_files(app.files, job_path) + cls._symlink_files(app.files, job_path) + cls._write_tagged_files(app.files, app.params, job_path) @staticmethod - def _copy_files(files: t.Union[EntityFiles, None], dest: os.PathLike[str]) -> None: + def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: """Perform copy file sys operations on a list of files. :param app: The Application attached to the Job @@ -176,9 +214,7 @@ def _copy_files(files: t.Union[EntityFiles, None], dest: os.PathLike[str]) -> No ) @staticmethod - def _symlink_files( - files: t.Union[EntityFiles, None], dest: os.PathLike[str] - ) -> None: + def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: """Perform symlink file sys operations on a list of files. :param app: The Application attached to the Job @@ -209,7 +245,7 @@ def _symlink_files( def _write_tagged_files( files: t.Union[EntityFiles, None], params: t.Mapping[str, str], - dest: os.PathLike[str], + dest: pathlib.Path, ) -> None: """Read, configure and write the tagged input files for a Job instance. This function specifically deals with the tagged @@ -308,4 +344,4 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: # logger.log( # level=self.log_level, # msg=f"Configured application {entity.name} with no parameters", - # ) + # ) \ No newline at end of file diff --git a/smartsim/experiment.py b/smartsim/experiment.py index f10f34c15..5f82152e6 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -29,11 +29,11 @@ from __future__ import annotations import collections +import datetime import itertools import os import os.path as osp import pathlib -import datetime import textwrap import typing as t from os import environ, getcwd @@ -43,8 +43,8 @@ from smartsim._core.config import CONFIG from smartsim._core.control.launch_history import LaunchHistory as _LaunchHistory from smartsim.error import errors -from smartsim.settings import dispatch from smartsim.status import InvalidJobStatus, JobStatus +from smartsim.settings import dispatch from ._core import Controller, Generator, Manifest, previewrenderer from .database import FeatureStore @@ -59,8 +59,8 @@ from .log import ctx_exp_path, get_logger, method_contextualizer if t.TYPE_CHECKING: - from smartsim.launchable.job import Job from smartsim.settings.dispatch import ExecutableProtocol + from smartsim.launchable.job import Job from smartsim.types import LaunchedJobID logger = get_logger(__name__) @@ -160,6 +160,7 @@ def __init__(self, name: str, exp_path: str | None = None): exp_path = osp.abspath(exp_path) else: exp_path = osp.join(getcwd(), name) + self.exp_path = exp_path """The path under which the experiment operate""" @@ -182,10 +183,10 @@ def start(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: jobs that can be used to query or alter the status of that particular execution of the job. """ + # Create the run id run_id = datetime.datetime.now().replace(microsecond=0).isoformat() - """Create the run id""" + # Generate the root path root = pathlib.Path(self.exp_path, run_id) - """Generate the root path""" return self._dispatch(Generator(root), dispatch.DEFAULT_DISPATCHER, *jobs) def _dispatch( @@ -231,8 +232,12 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: launch_config = dispatch.create_new_launcher_configuration( for_experiment=self, with_arguments=args ) - job_execution_path = self._generate(generator, job, idx) - id_ = launch_config.start(exe, job_execution_path, env) + # Generate the job directory and return the generated job path + ret = self._generate(generator, job, idx) + print(f"the type: {type(ret)}") + print(f"the val: {ret}") + job_execution_path, out, err = ret + id_ = launch_config.start(exe, job_execution_path, env, out, err) # Save the underlying launcher instance and launched job id. That # way we do not need to spin up a launcher instance for each # individual job, and the experiment can monitor job statuses. @@ -275,110 +280,29 @@ def get_status( stats = (stats_map.get(i, InvalidJobStatus.NEVER_STARTED) for i in ids) return tuple(stats) - def get_status( - self, *ids: LaunchedJobID - ) -> tuple[JobStatus | InvalidJobStatus, ...]: - """Get the status of jobs launched through the `Experiment` from their - launched job id returned when calling `Experiment.start`. - - The `Experiment` will map the launched ID back to the launcher that - started the job and request a status update. The order of the returned - statuses exactly matches the order of the launched job ids. - - If the `Experiment` cannot find any launcher that started the job - associated with the launched job id, then a - `InvalidJobStatus.NEVER_STARTED` status is returned for that id. - - If the experiment maps the launched job id to multiple launchers, then - a `ValueError` is raised. This should only happen in the case when - launched job ids issued by user defined launcher are not sufficiently - unique. - - :param ids: A sequence of launched job ids issued by the experiment. - :returns: A tuple of statuses with order respective of the order of the - calling arguments. - """ - to_query = self._launch_history.group_by_launcher( - set(ids), unknown_ok=True - ).items() - stats_iter = (launcher.get_status(*ids).items() for launcher, ids in to_query) - stats_map = dict(itertools.chain.from_iterable(stats_iter)) - stats = (stats_map.get(i, InvalidJobStatus.NEVER_STARTED) for i in ids) - return tuple(stats) - @_contextualize - def _generate( - self, generator: Generator, job: Job, job_index: int - ) -> os.PathLike[str]: - """Generate the directory and file structure for a ``Job`` + def _generate(self, generator: Generator, job: Job, job_index: int) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: + """Generate the directory structure and files for a ``Job`` - If files or directories are attached to an ``application`` object - associated with the Job using ``application.attach_generator_files()``, + If files or directories are attached to an ``Application`` object + associated with the Job using ``Application.attach_generator_files()``, those files or directories will be symlinked, copied, or configured and - written into the created job directory - - An instance of ``Generator`` and ``Job`` can be passed as an argument to - the protected _generate member, as well as the Jobs index. + written into the created job directory. - :param generator: The generator is responsible for creating the job run and log directory. - :param job: The job instance for which the output is generated. - :param job_index: The index of the job instance (used for naming). - :returns: The path to the generated output for the job instance. + :param generator: The generator is responsible for creating the job + run and log directory. + :param job: The Job instance for which the output is generated. + :param job_index: The index of the Job instance (used for naming). + :returns: The path to the generated output for the Job instance. :raises: A SmartSimError if an error occurs during the generation process. """ - # Generate ../job_name/run directory - job_path = self._generate_job_path(job, job_index, generator.root) - # Generate ../job_name/log directory - log_path = self._generate_log_path(job, job_index, generator.root) try: - out, err = generator.generate_job(job, job_path, log_path) + job_path, out, err = generator.generate_job(job, job_index) return (job_path, out, err) except SmartSimError as e: logger.error(e) raise - def _generate_job_root( - self, job: Job, job_index: int, root: os.PathLike[str] - ) -> pathlib.Path: - """Generates the root directory for a specific job instance. - - :param job: The Job instance for which the root directory is generated. - :param job_index: The index of the Job instance (used for naming). - :returns: The path to the root directory for the Job instance. - """ - job_type = f"{job.__class__.__name__.lower()}s" - job_path = pathlib.Path(root) / f"{job_type}/{job.name}-{job_index}" - job_path.mkdir(exist_ok=True, parents=True) - return pathlib.Path(job_path) - - def _generate_job_path( - self, job: Job, job_index: int, root: os.PathLike[str] - ) -> os.PathLike[str]: - """Generates the path for the \"run\" directory within the root directory - of a specific job instance. - - :param job (Job): The job instance for which the path is generated. - :param job_index (int): The index of the job instance (used for naming). - :returns: The path to the \"run\" directory for the job instance. - """ - path = self._generate_job_root(job, job_index, root) / "run" - path.mkdir(exist_ok=False, parents=True) - return pathlib.Path(path) - - def _generate_log_path( - self, job: Job, job_index: int, root: os.PathLike[str] - ) -> os.PathLike[str]: - """ - Generates the path for the \"log\" directory within the root directory of a specific job instance. - - :param job: The job instance for which the path is generated. - :param job_index: The index of the job instance (used for naming). - :returns: The path to the \"log\" directory for the job instance. - """ - path = self._generate_job_root(job, job_index, root) / "log" - path.mkdir(exist_ok=False, parents=True) - return pathlib.Path(path) - def preview( self, *args: t.Any, @@ -468,4 +392,4 @@ def _append_to_fs_identifier_list(self, fs_identifier: str) -> None: "with the same identifier" ) # Otherwise, add - self._fs_identifiers.add(fs_identifier) + self._fs_identifiers.add(fs_identifier) \ No newline at end of file diff --git a/smartsim/settings/arguments/launch/alps.py b/smartsim/settings/arguments/launch/alps.py index 120293398..1879dd102 100644 --- a/smartsim/settings/arguments/launch/alps.py +++ b/smartsim/settings/arguments/launch/alps.py @@ -36,7 +36,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_aprun_command = make_shell_format_fn(run_command="aprun", out_flag="hold", err_flag="hold") +_as_aprun_command = make_shell_format_fn(run_command="aprun") @dispatch(with_format=_as_aprun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/local.py b/smartsim/settings/arguments/launch/local.py index 44cbb6ade..c638b6470 100644 --- a/smartsim/settings/arguments/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -37,7 +37,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_local_command = make_shell_format_fn(run_command=None, out_flag="hold", err_flag="hold") +_as_local_command = make_shell_format_fn(run_command=None) @dispatch(with_format=_as_local_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index 7659ef4ce..80cd748f1 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -36,7 +36,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_jsrun_command = make_shell_format_fn(run_command="jsrun", out_flag="--stdio_stdout", err_flag="--stdio_stderr") +_as_jsrun_command = make_shell_format_fn(run_command="jsrun") @dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/mpi.py b/smartsim/settings/arguments/launch/mpi.py index 1cc03793b..85fd38145 100644 --- a/smartsim/settings/arguments/launch/mpi.py +++ b/smartsim/settings/arguments/launch/mpi.py @@ -36,9 +36,9 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_mpirun_command = make_shell_format_fn("mpirun", out_flag="hold", err_flag="hold") -_as_mpiexec_command = make_shell_format_fn("mpiexec", out_flag="hold", err_flag="hold") -_as_orterun_command = make_shell_format_fn("orterun", out_flag="hold", err_flag="hold") +_as_mpirun_command = make_shell_format_fn("mpirun") +_as_mpiexec_command = make_shell_format_fn("mpiexec") +_as_orterun_command = make_shell_format_fn("orterun") class _BaseMPILaunchArguments(LaunchArguments): diff --git a/smartsim/settings/arguments/launch/pals.py b/smartsim/settings/arguments/launch/pals.py index 839fdd422..3132f1b02 100644 --- a/smartsim/settings/arguments/launch/pals.py +++ b/smartsim/settings/arguments/launch/pals.py @@ -36,7 +36,7 @@ from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_pals_command = make_shell_format_fn(run_command="mpiexec", out_flag="hold", err_flag="hold") +_as_pals_command = make_shell_format_fn(run_command="mpiexec") @dispatch(with_format=_as_pals_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index 9fbe6dcb5..a92ac1646 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -27,13 +27,15 @@ from __future__ import annotations import os +import pathlib +import subprocess import re import typing as t from smartsim.log import get_logger from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from smartsim._core.commands import Command -from smartsim.settings.dispatch import ExecutableProtocol, _FormatterType, _EnvironMappingType +from smartsim.settings.dispatch import ExecutableProtocol, _FormatterType, _EnvironMappingType, ShellLauncherCommand from ...common import set_check_input from ...launchCommand import LauncherType @@ -43,15 +45,21 @@ def _as_srun_command(args: LaunchArguments, exe: ExecutableProtocol, - path: str | os.PathLike[str], - _env: _EnvironMappingType) -> _FormatterType[LaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]]]: + path: pathlib.Path, + env: _EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path) -> ShellLauncherCommand: command_tuple = ( "srun", *(args.format_launch_args() or ()), "--", *exe.as_program_arguments(), + f"--output={stdout_path}", + f"--error={stderr_path}", + ) - return Command(["test"]) + # add output and err to CMD tuple -> add dev Null for stdout and stderr + return ShellLauncherCommand(env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple) @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index a2f2c4439..57ea2098b 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -31,6 +31,7 @@ import dataclasses import os import subprocess as sp +import pathlib import typing as t import uuid import os @@ -50,10 +51,18 @@ from smartsim.experiment import Experiment from smartsim.settings.arguments import LaunchArguments +class ShellLauncherCommand(t.NamedTuple): + env: _EnvironMappingType + path: pathlib.Path + stdout: pathlib.Path + stderr: pathlib.Path + command_tuple: tuple[str, tuple[str, ...]] | t.Sequence[str] + + _Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) -_WorkingDirectory: TypeAlias = t.Union[str, os.PathLike[str]] +_WorkingDirectory: TypeAlias = pathlib.Path """A working directory represented as a string or PathLike object""" _DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") @@ -70,14 +79,14 @@ a job """ _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType, str, str], + [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType, pathlib.Path, pathlib.Path], _LaunchableT, ] """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ _LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType, str, str]" + "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType, pathlib.Path, pathlib.Path]" ) """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings @@ -267,10 +276,10 @@ def create_adapter_from_launcher( def format_( exe: ExecutableProtocol, - path: str | os.PathLike[str], + path: pathlib.Path, env: _EnvironMappingType, - out: str, - err: str, + out: pathlib.Path, + err: pathlib.Path, ) -> _LaunchableT: return self.formatter(arguments, exe, path, env, out, err) @@ -439,8 +448,8 @@ def get_status( def make_shell_format_fn( - run_command: str | None, out_flag = str | None, err_flag = str | None, -) -> _FormatterType[LaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]]]: + run_command: str | None +) -> _FormatterType[LaunchArguments, ShellLauncherCommand]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -472,11 +481,11 @@ def make_shell_format_fn( def impl( args: LaunchArguments, exe: ExecutableProtocol, - path: str | os.PathLike[str], + path: _WorkingDirectory, env: _EnvironMappingType, - out_file: str, - err_file: str, - ) -> t.Tuple[str | os.PathLike[str], t.Sequence[str]]: + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, + ) -> ShellLauncherCommand: command_tuple = ( ( run_command, @@ -487,19 +496,9 @@ def impl( if run_command is not None else exe.as_program_arguments() ) - stdout = "hold" - stderr = "hold" - cmd = Command([env, path, stdout, stderr, command_tuple]) - return cmd + return ShellLauncherCommand(env, pathlib.Path(path), stdout_path, stderr_path, command_tuple) return impl - -class ShellLauncherCommand(t.NamedTuple): - env: str - path: str - stdout: str - stderr: str - command_tuple: tuple class ShellLauncher: @@ -517,7 +516,7 @@ def start( exe, *rest = shell_command.command_tuple expanded_exe = helpers.expand_exe_path(exe) # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((expanded_exe, *rest), cwd=shell_command.path, env=shell_command.env, stdout=shell_command.stdout, stderr=shell_command.stderr) + self._launched[id_] = sp.Popen((expanded_exe, *rest), cwd=shell_command.path, env={k:v for k,v in shell_command.env.items() if v is not None}, stdout=open(shell_command.stdout), stderr=open(shell_command.stderr)) # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ @@ -534,8 +533,9 @@ def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: print(ret_code) # try/catch around here and then reaise a smartsim.error if ret_code is None: - status = psutil.Process(proc.pid).status() # TODO can mock this, put this into a parameterized test, when you put that in the mock thing, the correct thing comes out - return {#1st arg, 2nd arg in the param tests, need to solve branching problems, do an assertion + status = psutil.Process(proc.pid).status() + print(status) + return { psutil.STATUS_RUNNING: JobStatus.RUNNING, psutil.STATUS_SLEEPING: JobStatus.RUNNING, psutil.STATUS_WAKING: JobStatus.RUNNING, diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 8e78da7f9..52ed76846 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -30,6 +30,7 @@ AprunLaunchArguments, _as_aprun_command, ) +from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -213,11 +214,13 @@ def test_invalid_exclude_hostlist_format(): def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): outfile = "out.txt" errfile = "err.txt" - env, path, stdin, stdout, cmd = _as_aprun_command( + shell_launch_cmd = _as_aprun_command( AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + env, path, stdin, stdout, cmd = shell_launch_cmd assert tuple(cmd) == expected assert path == test_dir assert env == {} - assert stdin == f"hold={outfile}" - assert stdout == f"hold={errfile}" + assert stdin == outfile + assert stdout == errfile diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 777b205b9..23de67938 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -31,6 +31,7 @@ _as_local_command, ) from smartsim.settings.launchCommand import LauncherType +from smartsim.settings.dispatch import ShellLauncherCommand pytestmark = pytest.mark.group_a @@ -145,11 +146,12 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): outfile = "out.txt" errfile = "err.txt" - env, path, stdin, stdout, cmd = _as_local_command( + shell_launch_cmd = _as_local_command( LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, outfile, errfile ) - assert tuple(cmd) == ("echo", "hello", "world") - assert path == test_dir - assert env == {} - assert stdin == f"hold={outfile}" - assert stdout == f"hold={errfile}" + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == ("echo", "hello", "world") + assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.env == {} + assert shell_launch_cmd.stdout == outfile + assert shell_launch_cmd.stderr == errfile diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 254065d50..16dbe965d 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -29,6 +29,7 @@ import pytest from smartsim.settings import LaunchSettings +from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.arguments.launch.mpi import ( MpiexecLaunchArguments, MpirunLaunchArguments, @@ -288,9 +289,10 @@ def test_formatting_launch_args( ): outfile = "out.txt" errfile = "err.txt" - env, path, stdin, stdout, fmt_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, outfile, errfile) - assert tuple(fmt_cmd) == (cmd,) + expected - assert path == test_dir - assert env == {} - assert stdin == f"hold={outfile}" - assert stdout == f"hold={errfile}" + shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, outfile, errfile) + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == (cmd,) + expected + assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.env == {} + assert shell_launch_cmd.stdout == outfile + assert shell_launch_cmd.stderr == errfile diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 690922007..3f197e376 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -32,6 +32,7 @@ _as_pals_command, ) from smartsim.settings.launchCommand import LauncherType +from smartsim.settings.dispatch import ShellLauncherCommand pytestmark = pytest.mark.group_a @@ -134,11 +135,12 @@ def test_invalid_hostlist_format(): def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): outfile = "out.txt" errfile = "err.txt" - env, path, stdin, stdout, args = _as_pals_command( + shell_launch_cmd = _as_pals_command( PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) - assert tuple(args) == expected - assert path == test_dir - assert env == {} - assert stdin == f"hold={outfile}" - assert stdout == f"hold={errfile}" + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.env == {} + assert shell_launch_cmd.stdout == outfile + assert shell_launch_cmd.stderr == errfile diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 9b9cbd5cd..b2a8178af 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -26,6 +26,7 @@ import pytest from smartsim.settings import LaunchSettings +from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, _as_srun_command, @@ -320,10 +321,10 @@ def test_set_het_groups(monkeypatch): def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): outfile = "out.txt" errfile = "err.txt" - test = _as_srun_command(args=SlurmLaunchArguments(args), exe=mock_echo_executable, path=test_dir, _env={}) - assert isinstance(test, Command) - # assert tuple(args) == expected - # assert path == test_dir - # assert env == {} - # assert stdin == f"--output={outfile}" - # assert stdout == f"--error={errfile}" + shell_launch_cmd = _as_srun_command(args=SlurmLaunchArguments(args), exe=mock_echo_executable, path=test_dir, env={}, stdout_path=outfile, stderr_path=errfile) + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.env == {} + assert shell_launch_cmd.stdout == f"--output={outfile}" + assert shell_launch_cmd.stderr == f"--error={errfile}" diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 49a127fb0..ad22a53c7 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -53,7 +53,7 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: "/tmp/job") + monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: ("/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt")) yield exp @@ -64,7 +64,7 @@ def dispatcher(): """ d = dispatch.Dispatcher() to_record: dispatch._FormatterType[MockLaunchArgs, LaunchRecord] = ( - lambda settings, exe, path, env: LaunchRecord(settings, exe, env, path) + lambda settings, exe, path, env, out, err: LaunchRecord(settings, exe, env, path, out, err) ) d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) yield d @@ -141,6 +141,8 @@ class LaunchRecord: entity: entity.SmartSimEntity env: t.Mapping[str, str | None] path: str + out: str + err: str @classmethod def from_job(cls, job: job.Job): @@ -156,7 +158,9 @@ def from_job(cls, job: job.Job): entity = job._entity env = job._launch_settings.env_vars path = "/tmp/job" - return cls(args, entity, env, path) + out = "/tmp/job/out.txt" + err = "/tmp/job/err.txt" + return cls(args, entity, env, path, out, err) class MockLaunchArgs(launchArguments.LaunchArguments): @@ -249,7 +253,7 @@ def test_start_can_launch_jobs( @pytest.mark.parametrize( "num_starts", - [pytest.param(i, id=f"{i} start(s)") for i in (1, 2,)], + [pytest.param(i, id=f"{i} start(s)") for i in (1, 2, 3, 5, 10, 100, 1_000)], ) def test_start_can_start_a_job_multiple_times_accross_multiple_calls( experiment: Experiment, job_maker: JobMakerType, num_starts: int diff --git a/tests/test_generator.py b/tests/test_generator.py index 9ec6d6358..1c82a4175 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -2,6 +2,7 @@ import itertools import os import pathlib +import random from glob import glob from os import listdir from os import path as osp @@ -10,16 +11,20 @@ from smartsim import Experiment from smartsim._core.generation.generator import Generator -from smartsim.entity import Application, Ensemble, SmartSimEntity, _mock +from smartsim.entity import Application, Ensemble from smartsim.entity.files import EntityFiles from smartsim.launchable import Job -from smartsim.settings import LaunchSettings, dispatch - +from smartsim.settings import LaunchSettings +from smartsim.settings import dispatch # TODO Add JobGroup tests when JobGroup becomes a Launchable pytestmark = pytest.mark.group_a +def random_id(): + return str(random.randint(1, 100)) + + @pytest.fixture def get_gen_copy_dir(fileutils): yield fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) @@ -46,20 +51,10 @@ def test_log_file_path(generator_instance): """Test if the log_file function returns the correct log path.""" base_path = "/tmp" expected_path = osp.join(base_path, "smartsim_params.txt") - assert generator_instance.log_file(base_path) == pathlib.Path(expected_path) - - -def test_output_files(generator_instance): - """Test if the log_file function returns the correct log path.""" - log_path = pathlib.Path("/tmp") - expected_out_path = osp.join(log_path, "name.out") - expected_err_path = osp.join(log_path, "name.err") - out, err = generator_instance.output_files(log_path, "name") - assert out == pathlib.Path(expected_out_path) - assert err == pathlib.Path(expected_err_path) + assert generator_instance._log_file(base_path) == pathlib.Path(expected_path) -def test_generate_job_directory(wlmutils, generator_instance): +def test_generate_job_directory(test_dir, wlmutils, generator_instance): """Test Generator.generate_job""" # Create Job launch_settings = LaunchSettings(wlmutils.get_test_launcher()) @@ -68,27 +63,34 @@ def test_generate_job_directory(wlmutils, generator_instance): ) # Mock RunSettings job = Job(app, launch_settings) # Mock id - run_id = "mock_run" - # Create run directory - run_path = generator_instance.root / "run" - run_path.mkdir(parents=True) - assert osp.isdir(run_path) - # Create log directory - log_path = generator_instance.root / "log" - log_path.mkdir(parents=True) - assert osp.isdir(log_path) + run_id = "temp_id" # Call Generator.generate_job - generator_instance.generate_job(job, run_path, log_path) + job_run_path, _, _ = generator_instance.generate_job(job, 0) + assert isinstance(job_run_path, pathlib.Path) + expected_run_path = ( + pathlib.Path(test_dir) + / run_id + / f"{job.__class__.__name__.lower()}s" + / f"{app.name}-{0}" + / "run" + ) + assert job_run_path == expected_run_path + expected_log_path = ( + pathlib.Path(test_dir) + / run_id + / f"{job.__class__.__name__.lower()}s" + / f"{app.name}-{0}" + / "log" + ) + assert osp.isdir(expected_run_path) + assert osp.isdir(expected_log_path) # Assert smartsim params file created - assert osp.isfile(osp.join(log_path, "smartsim_params.txt")) + assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) # Assert smartsim params correctly written to - with open(log_path / "smartsim_params.txt", "r") as file: + with open(expected_log_path / "smartsim_params.txt", "r") as file: content = file.read() assert "Generation start date and time:" in content - expected_out_path = osp.join(log_path, (job.entity.name + ".out")) - expected_err_path = osp.join(log_path, (job.entity.name + ".err")) - assert osp.isfile(expected_out_path) - assert osp.isfile(expected_err_path) + def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): """Test that Job directory was created from Experiment._generate.""" @@ -100,7 +102,7 @@ def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): job = Job(app, launch_settings) # Generate Job directory job_index = 1 - job_execution_path = exp._generate(generator_instance, job, job_index) + job_execution_path, _, _ = exp._generate(generator_instance, job, job_index) # Assert Job run directory exists assert osp.isdir(job_execution_path) # Assert Job log directory exists @@ -109,7 +111,7 @@ def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): assert osp.isdir(expected_log_path) -def test_generate_copy_file(fileutils, wlmutils, generator_instance): +def test_generate_copy_file(generator_instance, fileutils, wlmutils): """Test that attached copy files are copied into Job directory""" # Create the Job and attach copy generator file launch_settings = LaunchSettings(wlmutils.get_test_launcher()) @@ -118,13 +120,9 @@ def test_generate_copy_file(fileutils, wlmutils, generator_instance): app.attach_generator_files(to_copy=script) job = Job(app, launch_settings) - # Call Generator.generate_job - run_path = generator_instance.root / "run" - run_path.mkdir(parents=True) - log_path = generator_instance.root / "log" - log_path.mkdir(parents=True) - generator_instance.generate_job(job, run_path, log_path) - expected_file = run_path / "sleep.py" + # Create the experiment + path, _, _ = generator_instance.generate_job(job, 1) + expected_file = pathlib.Path(path) / "sleep.py" assert osp.isfile(expected_file) @@ -136,12 +134,8 @@ def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance) job = Job(app, launch_settings) # Call Generator.generate_job - run_path = generator_instance.root / "run" - run_path.mkdir(parents=True) - log_path = generator_instance.root / "log" - log_path.mkdir(parents=True) - generator_instance.generate_job(job, run_path, log_path) - expected_folder = run_path / "to_copy_dir" + path, _, _ = generator_instance.generate_job(job, 1) + expected_folder = path / "to_copy_dir" assert osp.isdir(expected_folder) @@ -155,12 +149,8 @@ def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlin job = Job(app, launch_settings) # Call Generator.generate_job - run_path = generator_instance.root / "run" - run_path.mkdir(parents=True) - log_path = generator_instance.root / "log" - log_path.mkdir(parents=True) - generator_instance.generate_job(job, run_path, log_path) - expected_folder = run_path / "to_symlink_dir" + path, _, _ = generator_instance.generate_job(job, 1) + expected_folder = path / "to_symlink_dir" assert osp.isdir(expected_folder) assert expected_folder.is_symlink() assert os.fspath(expected_folder.resolve()) == osp.realpath(get_gen_symlink_dir) @@ -186,12 +176,8 @@ def test_generate_symlink_file(get_gen_symlink_dir, wlmutils, generator_instance job = Job(app, launch_settings) # Call Generator.generate_job - run_path = generator_instance.root / "run" - run_path.mkdir(parents=True) - log_path = generator_instance.root / "log" - log_path.mkdir(parents=True) - generator_instance.generate_job(job, run_path, log_path) - expected_file = pathlib.Path(run_path) / "mock2.txt" + path, _, _ = generator_instance.generate_job(job, 1) + expected_file = path / "mock2.txt" assert osp.isfile(expected_file) assert expected_file.is_symlink() assert os.fspath(expected_file.resolve()) == osp.join( @@ -228,13 +214,9 @@ def test_generate_configure(fileutils, wlmutils, generator_instance): job = Job(app, launch_settings) # Call Generator.generate_job - run_path = generator_instance.root / "run" - run_path.mkdir(parents=True) - log_path = generator_instance.root / "log" - log_path.mkdir(parents=True) - generator_instance.generate_job(job, run_path, log_path) + path, _, _ = generator_instance.generate_job(job, 0) # Retrieve the list of configured files in the test directory - configured_files = sorted(glob(str(run_path) + "/*")) + configured_files = sorted(glob(str(path) + "/*")) # Use filecmp.cmp to check that the corresponding files are equal for written, correct in itertools.zip_longest(configured_files, correct_files): assert filecmp.cmp(written, correct) @@ -247,7 +229,7 @@ def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_inst job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) for i, job in enumerate(job_list): - job_run_path = exp._generate(generator_instance, job, i) + job_run_path, _, _ = exp._generate(generator_instance, job, i) head, _ = os.path.split(job_run_path) expected_log_path = pathlib.Path(head) / "log" assert osp.isdir(job_run_path) @@ -260,15 +242,16 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): job_list = ensemble.as_jobs(launch_settings) for i, job in enumerate(job_list): # Call Generator.generate_job - run_path = generator_instance.root / f"run-{i}" - run_path.mkdir(parents=True) - log_path = generator_instance.root / f"log-{i}" - log_path.mkdir(parents=True) - generator_instance.generate_job(job, run_path, log_path) + path, _, _ = generator_instance.generate_job(job, i) + # Assert run directory created + assert osp.isdir(path) # Assert smartsim params file created - assert osp.isfile(osp.join(log_path, "smartsim_params.txt")) + head, _ = os.path.split(path) + expected_log_path = pathlib.Path(head) / "log" + assert osp.isdir(expected_log_path) + assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) # Assert smartsim params correctly written to - with open(log_path / "smartsim_params.txt", "r") as file: + with open(expected_log_path / "smartsim_params.txt", "r") as file: content = file.read() assert "Generation start date and time:" in content @@ -276,7 +259,7 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): monkeypatch.setattr( "smartsim.settings.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: "exit", + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) @@ -296,7 +279,7 @@ def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): monkeypatch.setattr( "smartsim.settings.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: "exit", + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) @@ -318,7 +301,7 @@ def test_generate_ensemble_symlink( ): monkeypatch.setattr( "smartsim.settings.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: "exit", + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( "ensemble-name", @@ -345,7 +328,7 @@ def test_generate_ensemble_configure( ): monkeypatch.setattr( "smartsim.settings.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: "exit", + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} # Retrieve a list of files for configuration diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 132f3afee..b59093c27 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -28,6 +28,7 @@ import unittest.mock import pytest import pathlib +import psutil import difflib import os import uuid @@ -35,6 +36,7 @@ from smartsim.entity import _mock, entity, Application from smartsim import Experiment from smartsim.settings import LaunchSettings +from smartsim.settings.dispatch import ShellLauncher from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, _as_srun_command, @@ -146,6 +148,8 @@ def test_popen_writes_to_output_file(test_dir: str): with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) id = shell_launcher.start(cmd) + val = shell_launcher.get_status(id) + print(val) proc = shell_launcher._launched[id] # Wait for subprocess to finish assert proc.wait() == 0 @@ -154,6 +158,8 @@ def test_popen_writes_to_output_file(test_dir: str): assert out.read() == "Hello World!\n" with open(err_file, "r", encoding="utf-8") as err: assert err.read() == "" + val = shell_launcher.get_status(id) + print(val) def test_popen_fails_with_invalid_cmd(test_dir): @@ -216,7 +222,7 @@ def test_shell_launcher_returns_failed_status(test_dir): with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: # Construct a invalid command to execute args = (helpers.expand_exe_path("srun"), "--flag_dne") - cmd = Command([{}, run_dir, out, err, args]) + cmd = ShellLauncherCommand({}, run_dir, out, err, args) # Start the execution of the command using a ShellLauncher for _ in range(5): id = shell_launcher.start(cmd) @@ -239,7 +245,7 @@ def test_shell_launcher_returns_running_status(test_dir): run_dir, out_file, err_file = generate_directory(test_dir) with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: # Construct a command to execute - cmd = Command([{}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5")]) + cmd = ShellLauncherCommand({}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5")) # Start the execution of the command using a ShellLauncher for _ in range(5): id = shell_launcher.start(cmd) @@ -249,11 +255,32 @@ def test_shell_launcher_returns_running_status(test_dir): # Assert that subprocess has completed assert code[val] == JobStatus.RUNNING - -# TODO one test that verifies the mapping in status, verify every single one, do not execute, just mock -def test_this(monkeypatch): +@pytest.mark.parametrize( + "psutil_status,job_status", + [ + pytest.param(psutil.STATUS_RUNNING, JobStatus.RUNNING, id="merp"), + pytest.param(psutil.STATUS_SLEEPING, JobStatus.RUNNING, id="merp"), + pytest.param(psutil.STATUS_WAKING, JobStatus.RUNNING, id="merp"), + pytest.param(psutil.STATUS_DISK_SLEEP, JobStatus.RUNNING, id="merp"), + pytest.param(psutil.STATUS_DEAD, JobStatus.FAILED, id="merp"), + pytest.param(psutil.STATUS_TRACING_STOP, JobStatus.PAUSED, id="merp"), + pytest.param(psutil.STATUS_WAITING, JobStatus.PAUSED, id="merp"), + pytest.param(psutil.STATUS_STOPPED, JobStatus.PAUSED, id="merp"), + pytest.param(psutil.STATUS_LOCKED, JobStatus.PAUSED, id="merp"), + pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="merp"), + pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="merp"), + pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="merp"), + ], +) +def test_this(psutil_status,job_status,monkeypatch: pytest.MonkeyPatch, test_dir): shell_launcher = ShellLauncher() - shell_launcher._launched = {"test":sp.Popen} - monkeypatch.setattr(sp.Popen, "poll", lambda: "/") - shell_launcher.get_status("test") \ No newline at end of file + run_dir, out_file, err_file = generate_directory(test_dir) + with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + monkeypatch.setattr(proc, "poll", lambda: None) + monkeypatch.setattr(psutil.Process, "status", lambda self: psutil_status) + value = shell_launcher.get_status(id) + assert value.get(id) == job_status \ No newline at end of file From f370b839d5c511ebde0693b17777107a0df78b7d Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 19 Aug 2024 18:59:36 -0500 Subject: [PATCH 68/82] slurm tests passing but work to be done on other settings --- .../test_settings/test_alpsLauncher.py | 12 +++++----- .../test_settings/test_localLauncher.py | 3 ++- .../test_settings/test_lsfLauncher.py | 3 ++- .../test_settings/test_mpiLauncher.py | 3 ++- .../test_settings/test_palsLauncher.py | 3 ++- .../test_settings/test_slurmLauncher.py | 24 ++++++++++--------- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 52ed76846..bf770ea6a 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -24,6 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pytest +import pathlib from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.alps import ( @@ -218,9 +219,8 @@ def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) - env, path, stdin, stdout, cmd = shell_launch_cmd - assert tuple(cmd) == expected - assert path == test_dir - assert env == {} - assert stdin == outfile - assert stdout == errfile + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert shell_launch_cmd.stdout == outfile + assert shell_launch_cmd.stderr == errfile diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 23de67938..7e2605576 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -24,6 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pytest +import pathlib from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.local import ( @@ -151,7 +152,7 @@ def test_formatting_returns_original_exe(mock_echo_executable, test_dir): ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == ("echo", "hello", "world") - assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} assert shell_launch_cmd.stdout == outfile assert shell_launch_cmd.stderr == errfile diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 55801862e..6d7df7c7c 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -24,6 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pytest +import pathlib from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.lsf import ( @@ -126,7 +127,7 @@ def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): JsrunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) assert tuple(args) == expected - assert path == test_dir + assert path == pathlib.Path(test_dir) assert env == {} assert stdin == f"--stdio_stdout={outfile}" assert stdout == f"--stdio_stderr={errfile}" diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 16dbe965d..fa0d1cc7f 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -27,6 +27,7 @@ import itertools import pytest +import pathlib from smartsim.settings import LaunchSettings from smartsim.settings.dispatch import ShellLauncherCommand @@ -292,7 +293,7 @@ def test_formatting_launch_args( shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, outfile, errfile) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == (cmd,) + expected - assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} assert shell_launch_cmd.stdout == outfile assert shell_launch_cmd.stderr == errfile diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 3f197e376..a25034401 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pytest +import pathlib from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.pals import ( @@ -140,7 +141,7 @@ def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == expected - assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} assert shell_launch_cmd.stdout == outfile assert shell_launch_cmd.stderr == errfile diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index b2a8178af..59068fc4f 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -24,6 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pytest +import pathlib +import subprocess from smartsim.settings import LaunchSettings from smartsim.settings.dispatch import ShellLauncherCommand @@ -290,41 +292,41 @@ def test_set_het_groups(monkeypatch): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("srun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param({}, ("srun", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), id="Empty Args"), pytest.param( {"N": "1"}, - ("srun", "-N", "1", "--", "echo", "hello", "world"), + ("srun", "-N", "1", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), id="Short Arg", ), pytest.param( {"nodes": "1"}, - ("srun", "--nodes=1", "--", "echo", "hello", "world"), + ("srun", "--nodes=1", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), id="Long Arg", ), pytest.param( {"v": None}, - ("srun", "-v", "--", "echo", "hello", "world"), + ("srun", "-v", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("srun", "--verbose", "--", "echo", "hello", "world"), + ("srun", "--verbose", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), id="Long Arg (No Value)", ), pytest.param( {"nodes": "1", "n": "123"}, - ("srun", "--nodes=1", "-n", "123", "--", "echo", "hello", "world"), + ("srun", "--nodes=1", "-n", "123", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), id="Short and Long Args", ), ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - outfile = "out.txt" - errfile = "err.txt" - shell_launch_cmd = _as_srun_command(args=SlurmLaunchArguments(args), exe=mock_echo_executable, path=test_dir, env={}, stdout_path=outfile, stderr_path=errfile) + + shell_launch_cmd = _as_srun_command(args=SlurmLaunchArguments(args), exe=mock_echo_executable, path=test_dir, env={}, stdout_path="output.txt", stderr_path="error.txt") assert isinstance(shell_launch_cmd, ShellLauncherCommand) + print(shell_launch_cmd.command_tuple) assert shell_launch_cmd.command_tuple == expected assert shell_launch_cmd.path == test_dir assert shell_launch_cmd.env == {} - assert shell_launch_cmd.stdout == f"--output={outfile}" - assert shell_launch_cmd.stderr == f"--error={errfile}" + assert shell_launch_cmd.stdout == subprocess.DEVNULL + assert shell_launch_cmd.stderr == subprocess.DEVNULL From 18b145549304d0a70d0a10171be1cc3d1e893652 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 19 Aug 2024 21:44:41 -0500 Subject: [PATCH 69/82] all tests pass --- .../_core/launcher/dragon/dragonLauncher.py | 7 ++++-- smartsim/settings/arguments/launch/lsf.py | 23 ++++++++++++++++-- smartsim/settings/dispatch.py | 6 ++--- .../test_settings/test_dragonLauncher.py | 6 +++-- .../test_settings/test_lsfLauncher.py | 24 +++++++++---------- tests/test_shell_launcher.py | 3 ++- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 992707959..56bb10bff 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -28,6 +28,7 @@ import os import typing as t +import pathlib from smartsim._core.schemas.dragonRequests import DragonRunPolicy from smartsim.error import errors @@ -368,6 +369,8 @@ def _as_run_request_args_and_policy( exe: ExecutableProtocol, path: str | os.PathLike[str], env: t.Mapping[str, str | None], + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, ) -> tuple[DragonRunRequestView, DragonRunPolicy]: # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # FIXME: This type is 100% unacceptable, but I don't want to spend too much @@ -389,8 +392,8 @@ def _as_run_request_args_and_policy( env=env, # TODO: Not sure how this info is injected name=None, - output_file=None, - error_file=None, + output_file=stdout_path, + error_file=stderr_path, **run_args, ), policy, diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index 80cd748f1..b8b6f5d91 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -27,16 +27,35 @@ from __future__ import annotations import typing as t +import pathlib +import subprocess from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn +from smartsim.settings.dispatch import ShellLauncher, dispatch, ExecutableProtocol, _EnvironMappingType, ShellLauncherCommand from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArguments import LaunchArguments logger = get_logger(__name__) -_as_jsrun_command = make_shell_format_fn(run_command="jsrun") + +def _as_jsrun_command(args: LaunchArguments, + exe: ExecutableProtocol, + path: pathlib.Path, + env: _EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path) -> ShellLauncherCommand: + command_tuple = ( + "jsrun", + *(args.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + f"--stdio_stdout={stdout_path}", + f"--stdio_stderr={stderr_path}", + + ) + # add output and err to CMD tuple -> add dev Null for stdout and stderr + return ShellLauncherCommand(env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple) @dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 57ea2098b..a2af2bd4f 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -54,8 +54,8 @@ class ShellLauncherCommand(t.NamedTuple): env: _EnvironMappingType path: pathlib.Path - stdout: pathlib.Path - stderr: pathlib.Path + stdout: TextIOWrapper + stderr: TextIOWrapper command_tuple: tuple[str, tuple[str, ...]] | t.Sequence[str] @@ -516,7 +516,7 @@ def start( exe, *rest = shell_command.command_tuple expanded_exe = helpers.expand_exe_path(exe) # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((expanded_exe, *rest), cwd=shell_command.path, env={k:v for k,v in shell_command.env.items() if v is not None}, stdout=open(shell_command.stdout), stderr=open(shell_command.stderr)) + self._launched[id_] = sp.Popen((expanded_exe, *rest), cwd=shell_command.path, env={k:v for k,v in shell_command.env.items() if v is not None}, stdout=shell_command.stdout, stderr=shell_command.stderr) # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 38ee11486..3b2837a60 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -78,7 +78,7 @@ def test_formatting_launch_args_into_request( if gpu_affinity is not NOT_SET: launch_args.set_gpu_affinity(gpu_affinity) req, policy = _as_run_request_args_and_policy( - launch_args, mock_echo_executable, test_dir, {} + launch_args, mock_echo_executable, test_dir, {}, "output.txt", "error.txt" ) expected_args = { @@ -90,7 +90,7 @@ def test_formatting_launch_args_into_request( if v is not NOT_SET } expected_run_req = DragonRunRequestView( - exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, **expected_args + exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, output_file="output.txt", error_file="error.txt", **expected_args ) assert req.exe == expected_run_req.exe assert req.exe_args == expected_run_req.exe_args @@ -99,6 +99,8 @@ def test_formatting_launch_args_into_request( assert req.hostlist == expected_run_req.hostlist assert req.pmi_enabled == expected_run_req.pmi_enabled assert req.path == expected_run_req.path + assert req.output_file == expected_run_req.output_file + assert req.error_file == expected_run_req.error_file expected_run_policy_args = { k: v diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 6d7df7c7c..e3879a3bc 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -24,7 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pytest -import pathlib +import subprocess from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.lsf import ( @@ -92,42 +92,42 @@ def test_launch_args(): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("jsrun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param({}, ("jsrun", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), id="Empty Args"), pytest.param( {"n": "1"}, - ("jsrun", "-n", "1", "--", "echo", "hello", "world"), + ("jsrun", "-n", "1", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), id="Short Arg", ), pytest.param( {"nrs": "1"}, - ("jsrun", "--nrs=1", "--", "echo", "hello", "world"), + ("jsrun", "--nrs=1", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), id="Long Arg", ), pytest.param( {"v": None}, - ("jsrun", "-v", "--", "echo", "hello", "world"), + ("jsrun", "-v", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("jsrun", "--verbose", "--", "echo", "hello", "world"), + ("jsrun", "--verbose", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), id="Long Arg (No Value)", ), pytest.param( {"tasks_per_rs": "1", "n": "123"}, - ("jsrun", "--tasks_per_rs=1", "-n", "123", "--", "echo", "hello", "world"), + ("jsrun", "--tasks_per_rs=1", "-n", "123", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), id="Short and Long Args", ), ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - outfile = "out.txt" - errfile = "err.txt" + outfile = "output.txt" + errfile = "error.txt" env, path, stdin, stdout, args = _as_jsrun_command( JsrunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) assert tuple(args) == expected - assert path == pathlib.Path(test_dir) + assert path == test_dir assert env == {} - assert stdin == f"--stdio_stdout={outfile}" - assert stdout == f"--stdio_stderr={errfile}" + assert stdin == subprocess.DEVNULL + assert stdout == subprocess.DEVNULL diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index b59093c27..7d5641bcf 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -27,6 +27,7 @@ import tempfile import unittest.mock import pytest +import subprocess import pathlib import psutil import difflib @@ -135,7 +136,7 @@ def test_popen_returns_popen_object(test_dir: str): shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + cmd = ShellLauncherCommand({}, run_dir, subprocess.DEVNULL, subprocess.DEVNULL, EchoHelloWorldEntity().as_program_arguments()) id = shell_launcher.start(cmd) proc = shell_launcher._launched[id] assert isinstance(proc, sp.Popen) From 2648258ac5a2b26a717af84eb34d2a862aa9284b Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 20 Aug 2024 11:24:34 -0500 Subject: [PATCH 70/82] all tests passing, mypy green, styling done --- smartsim/_core/generation/generator.py | 12 +- .../_core/launcher/dragon/dragonLauncher.py | 2 +- smartsim/experiment.py | 10 +- smartsim/settings/arguments/launch/local.py | 2 +- smartsim/settings/arguments/launch/lsf.py | 30 ++-- smartsim/settings/arguments/launch/slurm.py | 37 +++-- smartsim/settings/dispatch.py | 50 +++++-- .../test_settings/test_alpsLauncher.py | 20 +-- .../test_settings/test_dragonLauncher.py | 8 +- .../test_settings/test_localLauncher.py | 22 +-- .../test_settings/test_lsfLauncher.py | 75 +++++++++- .../test_settings/test_mpiLauncher.py | 17 ++- .../test_settings/test_palsLauncher.py | 27 ++-- .../test_settings/test_slurmLauncher.py | 89 ++++++++++-- tests/test_experiment.py | 10 +- tests/test_generator.py | 4 +- tests/test_shell_launcher.py | 129 +++++++++++++----- 17 files changed, 404 insertions(+), 140 deletions(-) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 6486e6ec5..37f6ce419 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -108,12 +108,16 @@ def _log_file(log_path: pathlib.Path) -> pathlib.Path: """ return pathlib.Path(log_path) / "smartsim_params.txt" - def _output_files(self, log_path: pathlib.Path, job_name: str) -> t.Tuple[pathlib.Path, pathlib.Path]: + def _output_files( + self, log_path: pathlib.Path, job_name: str + ) -> t.Tuple[pathlib.Path, pathlib.Path]: out_file_path = log_path / (job_name + ".out") err_file_path = log_path / (job_name + ".err") return out_file_path, err_file_path - def generate_job(self, job: Job, job_index: int) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: + def generate_job( + self, job: Job, job_index: int + ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: """Write and configure input files for a Job. To have files or directories present in the created Job @@ -140,7 +144,7 @@ def generate_job(self, job: Job, job_index: int) -> t.Tuple[pathlib.Path, pathli with open(self._log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - + # Create output files out_file, err_file = self._output_files(log_path, job.entity.name) # Open and write to .out file @@ -344,4 +348,4 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: # logger.log( # level=self.log_level, # msg=f"Configured application {entity.name} with no parameters", - # ) \ No newline at end of file + # ) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 56bb10bff..e71e694ee 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -27,8 +27,8 @@ from __future__ import annotations import os -import typing as t import pathlib +import typing as t from smartsim._core.schemas.dragonRequests import DragonRunPolicy from smartsim.error import errors diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 5f82152e6..7137bed36 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -43,8 +43,8 @@ from smartsim._core.config import CONFIG from smartsim._core.control.launch_history import LaunchHistory as _LaunchHistory from smartsim.error import errors -from smartsim.status import InvalidJobStatus, JobStatus from smartsim.settings import dispatch +from smartsim.status import InvalidJobStatus, JobStatus from ._core import Controller, Generator, Manifest, previewrenderer from .database import FeatureStore @@ -59,8 +59,8 @@ from .log import ctx_exp_path, get_logger, method_contextualizer if t.TYPE_CHECKING: - from smartsim.settings.dispatch import ExecutableProtocol from smartsim.launchable.job import Job + from smartsim.settings.dispatch import ExecutableProtocol from smartsim.types import LaunchedJobID logger = get_logger(__name__) @@ -281,7 +281,9 @@ def get_status( return tuple(stats) @_contextualize - def _generate(self, generator: Generator, job: Job, job_index: int) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: + def _generate( + self, generator: Generator, job: Job, job_index: int + ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: """Generate the directory structure and files for a ``Job`` If files or directories are attached to an ``Application`` object @@ -392,4 +394,4 @@ def _append_to_fs_identifier_list(self, fs_identifier: str) -> None: "with the same identifier" ) # Otherwise, add - self._fs_identifiers.add(fs_identifier) \ No newline at end of file + self._fs_identifiers.add(fs_identifier) diff --git a/smartsim/settings/arguments/launch/local.py b/smartsim/settings/arguments/launch/local.py index c638b6470..a61068cd4 100644 --- a/smartsim/settings/arguments/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -25,9 +25,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations -from subprocess import PIPE import typing as t +from subprocess import PIPE from smartsim.log import get_logger from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index b8b6f5d91..e99c39af7 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -26,12 +26,18 @@ from __future__ import annotations -import typing as t import pathlib import subprocess +import typing as t from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, ExecutableProtocol, _EnvironMappingType, ShellLauncherCommand +from smartsim.settings.dispatch import ( + ExecutableProtocol, + ShellLauncher, + ShellLauncherCommand, + _EnvironMappingType, + dispatch, +) from ...common import set_check_input from ...launchCommand import LauncherType @@ -39,12 +45,15 @@ logger = get_logger(__name__) -def _as_jsrun_command(args: LaunchArguments, - exe: ExecutableProtocol, - path: pathlib.Path, - env: _EnvironMappingType, - stdout_path: pathlib.Path, - stderr_path: pathlib.Path) -> ShellLauncherCommand: + +def _as_jsrun_command( + args: LaunchArguments, + exe: ExecutableProtocol, + path: pathlib.Path, + env: _EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, +) -> ShellLauncherCommand: command_tuple = ( "jsrun", *(args.format_launch_args() or ()), @@ -52,10 +61,11 @@ def _as_jsrun_command(args: LaunchArguments, *exe.as_program_arguments(), f"--stdio_stdout={stdout_path}", f"--stdio_stderr={stderr_path}", - ) # add output and err to CMD tuple -> add dev Null for stdout and stderr - return ShellLauncherCommand(env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple) + return ShellLauncherCommand( + env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple + ) @dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index a92ac1646..141e9adcd 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -28,14 +28,21 @@ import os import pathlib -import subprocess import re +import subprocess import typing as t -from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from smartsim._core.commands import Command -from smartsim.settings.dispatch import ExecutableProtocol, _FormatterType, _EnvironMappingType, ShellLauncherCommand +from smartsim.log import get_logger +from smartsim.settings.dispatch import ( + ExecutableProtocol, + ShellLauncher, + ShellLauncherCommand, + _EnvironMappingType, + _FormatterType, + dispatch, + make_shell_format_fn, +) from ...common import set_check_input from ...launchCommand import LauncherType @@ -43,12 +50,15 @@ logger = get_logger(__name__) -def _as_srun_command(args: LaunchArguments, - exe: ExecutableProtocol, - path: pathlib.Path, - env: _EnvironMappingType, - stdout_path: pathlib.Path, - stderr_path: pathlib.Path) -> ShellLauncherCommand: + +def _as_srun_command( + args: LaunchArguments, + exe: ExecutableProtocol, + path: pathlib.Path, + env: _EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, +) -> ShellLauncherCommand: command_tuple = ( "srun", *(args.format_launch_args() or ()), @@ -56,11 +66,12 @@ def _as_srun_command(args: LaunchArguments, *exe.as_program_arguments(), f"--output={stdout_path}", f"--error={stderr_path}", - ) # add output and err to CMD tuple -> add dev Null for stdout and stderr - return ShellLauncherCommand(env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple) - + return ShellLauncherCommand( + env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple + ) + @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) class SlurmLaunchArguments(LaunchArguments): diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index a2af2bd4f..77ac6c74a 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -29,33 +29,36 @@ import abc import collections.abc import dataclasses +import io import os -import subprocess as sp import pathlib +import subprocess +import subprocess as sp import typing as t import uuid -import os +from subprocess import STDOUT import psutil from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack +from smartsim._core.commands import Command, CommandList from smartsim._core.utils import helpers from smartsim.error import errors from smartsim.status import JobStatus from smartsim.types import LaunchedJobID -from smartsim._core.commands import Command, CommandList + from ..settings.launchCommand import LauncherType -from subprocess import STDOUT if t.TYPE_CHECKING: from smartsim.experiment import Experiment from smartsim.settings.arguments import LaunchArguments + class ShellLauncherCommand(t.NamedTuple): env: _EnvironMappingType path: pathlib.Path - stdout: TextIOWrapper - stderr: TextIOWrapper + stdout: io.TextIOWrapper | int + stderr: io.TextIOWrapper | int command_tuple: tuple[str, tuple[str, ...]] | t.Sequence[str] @@ -79,7 +82,14 @@ class ShellLauncherCommand(t.NamedTuple): a job """ _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType, pathlib.Path, pathlib.Path], + [ + _DispatchableT, + "ExecutableProtocol", + _WorkingDirectory, + _EnvironMappingType, + pathlib.Path, + pathlib.Path, + ], _LaunchableT, ] """A callable that is capable of formatting the components of a job into a type @@ -448,7 +458,7 @@ def get_status( def make_shell_format_fn( - run_command: str | None + run_command: str | None, ) -> _FormatterType[LaunchArguments, ShellLauncherCommand]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -478,6 +488,7 @@ def make_shell_format_fn( :returns: A function to format an arguments, an executable, and an environment as a shell launchable sequence for strings. """ + def impl( args: LaunchArguments, exe: ExecutableProtocol, @@ -496,27 +507,36 @@ def impl( if run_command is not None else exe.as_program_arguments() ) - return ShellLauncherCommand(env, pathlib.Path(path), stdout_path, stderr_path, command_tuple) + return ShellLauncherCommand( + env, pathlib.Path(path), open(stdout_path), open(stderr_path), command_tuple + ) return impl - + class ShellLauncher: """Mock launcher for launching/tracking simple shell commands""" + # add a def check def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} + # covariant, contravariant, + boliscoff substitution princ def start( - self, shell_command: ShellLauncherCommand # this should be a named tuple + self, shell_command: ShellLauncherCommand # this should be a named tuple ) -> LaunchedJobID: id_ = create_job_id() # raise ValueError -> invalid stuff throw exe, *rest = shell_command.command_tuple expanded_exe = helpers.expand_exe_path(exe) - # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((expanded_exe, *rest), cwd=shell_command.path, env={k:v for k,v in shell_command.env.items() if v is not None}, stdout=shell_command.stdout, stderr=shell_command.stderr) + self._launched[id_] = sp.Popen( + (expanded_exe, *rest), + cwd=shell_command.path, + env={k: v for k, v in shell_command.env.items() if v is not None}, + stdout=shell_command.stdout, + stderr=shell_command.stderr, + ) # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ @@ -529,7 +549,9 @@ def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: if (proc := self._launched.get(id_)) is None: msg = f"Launcher `{self}` has not launched a job with id `{id_}`" raise errors.LauncherJobNotFound(msg) - ret_code = proc.poll() # add a test that mocks out poll and raise some exception - terminal -> import subprocess -> start something echo blah - then poll and see what a valid fake output is + ret_code = ( + proc.poll() + ) # add a test that mocks out poll and raise some exception - terminal -> import subprocess -> start something echo blah - then poll and see what a valid fake output is print(ret_code) # try/catch around here and then reaise a smartsim.error if ret_code is None: diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index bf770ea6a..ed18d5810 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -23,9 +23,12 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest +import io +import os import pathlib +import pytest + from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.alps import ( AprunLaunchArguments, @@ -213,14 +216,15 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - outfile = "out.txt" - errfile = "err.txt" - shell_launch_cmd = _as_aprun_command( - AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile - ) + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + with open(out, "w") as _, open(err, "w") as _: + shell_launch_cmd = _as_aprun_command( + AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, out, err + ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == expected assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} - assert shell_launch_cmd.stdout == outfile - assert shell_launch_cmd.stderr == errfile + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 3b2837a60..6e3722dde 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -90,7 +90,13 @@ def test_formatting_launch_args_into_request( if v is not NOT_SET } expected_run_req = DragonRunRequestView( - exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, output_file="output.txt", error_file="error.txt", **expected_args + exe="echo", + exe_args=["hello", "world"], + path=test_dir, + env={}, + output_file="output.txt", + error_file="error.txt", + **expected_args, ) assert req.exe == expected_run_req.exe assert req.exe_args == expected_run_req.exe_args diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 7e2605576..65bcb6bc9 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -23,16 +23,19 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest +import io +import os import pathlib +import pytest + from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.local import ( LocalLaunchArguments, _as_local_command, ) -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.dispatch import ShellLauncherCommand +from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -145,14 +148,15 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): - outfile = "out.txt" - errfile = "err.txt" - shell_launch_cmd = _as_local_command( - LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, outfile, errfile - ) + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + with open(out, "w") as _, open(err, "w") as _: + shell_launch_cmd = _as_local_command( + LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, out, err + ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == ("echo", "hello", "world") assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} - assert shell_launch_cmd.stdout == outfile - assert shell_launch_cmd.stderr == errfile + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index e3879a3bc..eef23f530 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -23,9 +23,10 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest import subprocess +import pytest + from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.lsf import ( JsrunLaunchArguments, @@ -92,30 +93,90 @@ def test_launch_args(): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("jsrun", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), id="Empty Args"), + pytest.param( + {}, + ( + "jsrun", + "--", + "echo", + "hello", + "world", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + ), + id="Empty Args", + ), pytest.param( {"n": "1"}, - ("jsrun", "-n", "1", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), + ( + "jsrun", + "-n", + "1", + "--", + "echo", + "hello", + "world", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + ), id="Short Arg", ), pytest.param( {"nrs": "1"}, - ("jsrun", "--nrs=1", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), + ( + "jsrun", + "--nrs=1", + "--", + "echo", + "hello", + "world", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + ), id="Long Arg", ), pytest.param( {"v": None}, - ("jsrun", "-v", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), + ( + "jsrun", + "-v", + "--", + "echo", + "hello", + "world", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + ), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("jsrun", "--verbose", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), + ( + "jsrun", + "--verbose", + "--", + "echo", + "hello", + "world", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + ), id="Long Arg (No Value)", ), pytest.param( {"tasks_per_rs": "1", "n": "123"}, - ("jsrun", "--tasks_per_rs=1", "-n", "123", "--", "echo", "hello", "world", '--stdio_stdout=output.txt', '--stdio_stderr=error.txt'), + ( + "jsrun", + "--tasks_per_rs=1", + "-n", + "123", + "--", + "echo", + "hello", + "world", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + ), id="Short and Long Args", ), ), diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index fa0d1cc7f..eff6b3ca2 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -24,13 +24,14 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io import itertools +import os +import pathlib import pytest -import pathlib from smartsim.settings import LaunchSettings -from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.arguments.launch.mpi import ( MpiexecLaunchArguments, MpirunLaunchArguments, @@ -39,6 +40,7 @@ _as_mpirun_command, _as_orterun_command, ) +from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -288,12 +290,13 @@ def test_invalid_hostlist_format(launcher): def test_formatting_launch_args( mock_echo_executable, cls, fmt, cmd, args, expected, test_dir ): - outfile = "out.txt" - errfile = "err.txt" - shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, outfile, errfile) + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + with open(out, "w") as _, open(err, "w") as _: + shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, out, err) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == (cmd,) + expected assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} - assert shell_launch_cmd.stdout == outfile - assert shell_launch_cmd.stderr == errfile + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index a25034401..261c04c17 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -24,16 +24,19 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest +import io +import os import pathlib +import pytest + from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.pals import ( PalsMpiexecLaunchArguments, _as_pals_command, ) -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.dispatch import ShellLauncherCommand +from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a @@ -134,14 +137,20 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - outfile = "out.txt" - errfile = "err.txt" - shell_launch_cmd = _as_pals_command( - PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile - ) + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + with open(out, "w") as _, open(err, "w") as _: + shell_launch_cmd = _as_pals_command( + PalsMpiexecLaunchArguments(args), + mock_echo_executable, + test_dir, + {}, + out, + err, + ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == expected assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} - assert shell_launch_cmd.stdout == outfile - assert shell_launch_cmd.stderr == errfile + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 59068fc4f..892978dfb 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -23,18 +23,19 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest import pathlib import subprocess +import pytest + +from smartsim._core.commands import Command from smartsim.settings import LaunchSettings -from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, _as_srun_command, ) +from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType -from smartsim._core.commands import Command pytestmark = pytest.mark.group_a @@ -292,37 +293,103 @@ def test_set_het_groups(monkeypatch): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("srun", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), id="Empty Args"), + pytest.param( + {}, + ( + "srun", + "--", + "echo", + "hello", + "world", + "--output=output.txt", + "--error=error.txt", + ), + id="Empty Args", + ), pytest.param( {"N": "1"}, - ("srun", "-N", "1", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), + ( + "srun", + "-N", + "1", + "--", + "echo", + "hello", + "world", + "--output=output.txt", + "--error=error.txt", + ), id="Short Arg", ), pytest.param( {"nodes": "1"}, - ("srun", "--nodes=1", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), + ( + "srun", + "--nodes=1", + "--", + "echo", + "hello", + "world", + "--output=output.txt", + "--error=error.txt", + ), id="Long Arg", ), pytest.param( {"v": None}, - ("srun", "-v", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), + ( + "srun", + "-v", + "--", + "echo", + "hello", + "world", + "--output=output.txt", + "--error=error.txt", + ), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("srun", "--verbose", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), + ( + "srun", + "--verbose", + "--", + "echo", + "hello", + "world", + "--output=output.txt", + "--error=error.txt", + ), id="Long Arg (No Value)", ), pytest.param( {"nodes": "1", "n": "123"}, - ("srun", "--nodes=1", "-n", "123", "--", "echo", "hello", "world", '--output=output.txt', '--error=error.txt'), + ( + "srun", + "--nodes=1", + "-n", + "123", + "--", + "echo", + "hello", + "world", + "--output=output.txt", + "--error=error.txt", + ), id="Short and Long Args", ), ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - - shell_launch_cmd = _as_srun_command(args=SlurmLaunchArguments(args), exe=mock_echo_executable, path=test_dir, env={}, stdout_path="output.txt", stderr_path="error.txt") + shell_launch_cmd = _as_srun_command( + args=SlurmLaunchArguments(args), + exe=mock_echo_executable, + path=test_dir, + env={}, + stdout_path="output.txt", + stderr_path="error.txt", + ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) print(shell_launch_cmd.command_tuple) assert shell_launch_cmd.command_tuple == expected diff --git a/tests/test_experiment.py b/tests/test_experiment.py index ad22a53c7..6e4fd8a7f 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -53,7 +53,11 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: ("/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt")) + monkeypatch.setattr( + exp, + "_generate", + lambda gen, job, idx: ("/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt"), + ) yield exp @@ -64,7 +68,9 @@ def dispatcher(): """ d = dispatch.Dispatcher() to_record: dispatch._FormatterType[MockLaunchArgs, LaunchRecord] = ( - lambda settings, exe, path, env, out, err: LaunchRecord(settings, exe, env, path, out, err) + lambda settings, exe, path, env, out, err: LaunchRecord( + settings, exe, env, path, out, err + ) ) d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) yield d diff --git a/tests/test_generator.py b/tests/test_generator.py index 1c82a4175..602f300ea 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -14,8 +14,8 @@ from smartsim.entity import Application, Ensemble from smartsim.entity.files import EntityFiles from smartsim.launchable import Job -from smartsim.settings import LaunchSettings -from smartsim.settings import dispatch +from smartsim.settings import LaunchSettings, dispatch + # TODO Add JobGroup tests when JobGroup becomes a Launchable pytestmark = pytest.mark.group_a diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 7d5641bcf..3eb6a7766 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -24,39 +24,41 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import tempfile -import unittest.mock -import pytest -import subprocess -import pathlib -import psutil import difflib import os +import pathlib +import subprocess +import tempfile +import unittest.mock import uuid import weakref -from smartsim.entity import _mock, entity, Application + +import psutil +import pytest + from smartsim import Experiment +from smartsim._core.commands import Command +from smartsim._core.utils import helpers +from smartsim._core.utils.shell import * +from smartsim.entity import Application, _mock, entity +from smartsim.error.errors import LauncherJobNotFound +from smartsim.launchable import Job from smartsim.settings import LaunchSettings -from smartsim.settings.dispatch import ShellLauncher from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, _as_srun_command, ) -from smartsim.status import JobStatus -from smartsim._core.utils.shell import * -from smartsim._core.commands import Command -from smartsim._core.utils import helpers -from smartsim.settings.dispatch import sp, ShellLauncher, ShellLauncherCommand +from smartsim.settings.dispatch import ShellLauncher, ShellLauncherCommand, sp from smartsim.settings.launchCommand import LauncherType -from smartsim.launchable import Job +from smartsim.status import JobStatus from smartsim.types import LaunchedJobID -from smartsim.error.errors import LauncherJobNotFound # TODO tests bad vars in Popen call at beginning - # tests -> helper.exe : pass in None, empty str, path with a space at beginning, a non valid command - # -> write a test for the invalid num of items - test_shell_launcher_fails_on_any_invalid_len_input - # -> have border tests for 0,1,4,6 cmd vals -> work correctly without them -> raise ValueError - # do all of the failures as well as the sucess criteria +# tests -> helper.exe : pass in None, empty str, path with a space at beginning, a non valid command +# -> write a test for the invalid num of items - test_shell_launcher_fails_on_any_invalid_len_input +# -> have border tests for 0,1,4,6 cmd vals -> work correctly without them -> raise ValueError +# do all of the failures as well as the sucess criteria + class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" @@ -72,32 +74,40 @@ def __eq__(self, other): def as_program_arguments(self): return (helpers.expand_exe_path("echo"), "Hello", "World!") + def create_directory(directory_path: str) -> pathlib.Path: """Creates the execution directory for testing.""" tmp_dir = pathlib.Path(directory_path) tmp_dir.mkdir(exist_ok=True, parents=True) return tmp_dir + def generate_output_files(tmp_dir: pathlib.Path): """Generates output and error files within the run directory for testing.""" out_file = tmp_dir / "tmp.out" err_file = tmp_dir / "tmp.err" return out_file, err_file + def generate_directory(test_dir: str): """Generates a execution directory, output file, and error file for testing.""" execution_dir = create_directory(os.path.join(test_dir, "tmp")) out_file, err_file = generate_output_files(execution_dir) return execution_dir, out_file, err_file + @pytest.fixture def shell_cmd(test_dir: str) -> ShellLauncherCommand: """Fixture to create an instance of Generator.""" run_dir, out_file, err_file = generate_directory(test_dir) - return ShellLauncherCommand({}, run_dir, out_file, err_file, EchoHelloWorldEntity().as_program_arguments()) + return ShellLauncherCommand( + {}, run_dir, out_file, err_file, EchoHelloWorldEntity().as_program_arguments() + ) + # UNIT TESTS + def test_shell_launcher_command_init(shell_cmd: ShellLauncherCommand, test_dir: str): """Test that ShellLauncherCommand initializes correctly""" assert shell_cmd.env == {} @@ -106,11 +116,13 @@ def test_shell_launcher_command_init(shell_cmd: ShellLauncherCommand, test_dir: assert shell_cmd.stderr == shell_cmd.path / "tmp.err" assert shell_cmd.command_tuple == EchoHelloWorldEntity().as_program_arguments() + def test_shell_launcher_init(): """Test that ShellLauncher initializes correctly""" shell_launcher = ShellLauncher() assert shell_launcher._launched == {} + def test_shell_launcher_start_calls_popen(shell_cmd: ShellLauncherCommand): """Test that the process leading up to the shell launcher popen call was correct""" shell_launcher = ShellLauncher() @@ -118,6 +130,7 @@ def test_shell_launcher_start_calls_popen(shell_cmd: ShellLauncherCommand): _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once() + def test_shell_launcher_start_calls_popen_with_value(shell_cmd: ShellLauncherCommand): """Test that popen was called with correct values""" shell_launcher = ShellLauncher() @@ -131,12 +144,22 @@ def test_shell_launcher_start_calls_popen_with_value(shell_cmd: ShellLauncherCom stderr=shell_cmd.stderr, ) + def test_popen_returns_popen_object(test_dir: str): """Test that the popen call returns a popen object""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - cmd = ShellLauncherCommand({}, run_dir, subprocess.DEVNULL, subprocess.DEVNULL, EchoHelloWorldEntity().as_program_arguments()) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, + run_dir, + subprocess.DEVNULL, + subprocess.DEVNULL, + EchoHelloWorldEntity().as_program_arguments(), + ) id = shell_launcher.start(cmd) proc = shell_launcher._launched[id] assert isinstance(proc, sp.Popen) @@ -146,8 +169,13 @@ def test_popen_writes_to_output_file(test_dir: str): """Test that popen writes to .out file upon successful process call""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) id = shell_launcher.start(cmd) val = shell_launcher.get_status(id) print(val) @@ -167,7 +195,10 @@ def test_popen_fails_with_invalid_cmd(test_dir): """Test that popen returns a non zero returncode after failure""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): args = (helpers.expand_exe_path("srun"), "--flag_dne") cmd = ShellLauncherCommand({}, run_dir, out, err, args) id = shell_launcher.start(cmd) @@ -185,8 +216,13 @@ def test_popen_issues_unique_ids(test_dir): """Validate that all ids are unique within ShellLauncher._launched""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) for _ in range(5): _ = shell_launcher.start(cmd) assert len(shell_launcher._launched) == 5 @@ -204,8 +240,13 @@ def test_shell_launcher_returns_complete_status(test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) for _ in range(5): id = shell_launcher.start(cmd) proc = shell_launcher._launched[id] @@ -214,13 +255,17 @@ def test_shell_launcher_returns_complete_status(test_dir): val = list(code.keys())[0] assert code[val] == JobStatus.COMPLETED + def test_shell_launcher_returns_failed_status(test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" # Init ShellLauncher shell_launcher = ShellLauncher() # Generate testing directory run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): # Construct a invalid command to execute args = (helpers.expand_exe_path("srun"), "--flag_dne") cmd = ShellLauncherCommand({}, run_dir, out, err, args) @@ -244,9 +289,14 @@ def test_shell_launcher_returns_running_status(test_dir): shell_launcher = ShellLauncher() # Generate testing directory run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): # Construct a command to execute - cmd = ShellLauncherCommand({}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5")) + cmd = ShellLauncherCommand( + {}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5") + ) # Start the execution of the command using a ShellLauncher for _ in range(5): id = shell_launcher.start(cmd) @@ -255,7 +305,7 @@ def test_shell_launcher_returns_running_status(test_dir): val = list(code.keys())[0] # Assert that subprocess has completed assert code[val] == JobStatus.RUNNING - + @pytest.mark.parametrize( "psutil_status,job_status", @@ -274,14 +324,19 @@ def test_shell_launcher_returns_running_status(test_dir): pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="merp"), ], ) -def test_this(psutil_status,job_status,monkeypatch: pytest.MonkeyPatch, test_dir): +def test_this(psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir): shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) - with open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err: - cmd = ShellLauncherCommand({}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments()) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) id = shell_launcher.start(cmd) proc = shell_launcher._launched[id] monkeypatch.setattr(proc, "poll", lambda: None) monkeypatch.setattr(psutil.Process, "status", lambda self: psutil_status) value = shell_launcher.get_status(id) - assert value.get(id) == job_status \ No newline at end of file + assert value.get(id) == job_status From 0c552234163d4417196716bbd853c8725df61083 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 20 Aug 2024 11:58:25 -0500 Subject: [PATCH 71/82] usused import deletions --- conftest.py | 34 ++++++------ smartsim/experiment.py | 2 - smartsim/launchable/baseJobGroup.py | 26 +++++++++ smartsim/launchable/colocatedJobGroup.py | 26 +++++++++ smartsim/launchable/job.py | 1 - smartsim/launchable/jobGroup.py | 26 +++++++++ smartsim/settings/arguments/launch/local.py | 1 - smartsim/settings/arguments/launch/slurm.py | 3 - smartsim/settings/dispatch.py | 18 +----- .../test_settings/test_slurmLauncher.py | 2 - tests/test_experiment.py | 1 - tests/test_generator.py | 2 +- tests/test_shell_launcher.py | 55 +++++-------------- 13 files changed, 112 insertions(+), 85 deletions(-) diff --git a/conftest.py b/conftest.py index 63a8c8081..3facd09a9 100644 --- a/conftest.py +++ b/conftest.py @@ -154,23 +154,23 @@ def pytest_sessionfinish( Called after whole test run finished, right before returning the exit status to the system. """ - # if exitstatus == 0: - # cleanup_attempts = 5 - # while cleanup_attempts > 0: - # try: - # shutil.rmtree(test_output_root) - # except OSError as e: - # cleanup_attempts -= 1 - # time.sleep(1) - # if not cleanup_attempts: - # raise - # else: - # break - # else: - # # kill all spawned processes - # if CONFIG.test_launcher == "dragon": - # time.sleep(5) - # kill_all_test_spawned_processes() + if exitstatus == 0: + cleanup_attempts = 5 + while cleanup_attempts > 0: + try: + shutil.rmtree(test_output_root) + except OSError as e: + cleanup_attempts -= 1 + time.sleep(1) + if not cleanup_attempts: + raise + else: + break + else: + # kill all spawned processes + if CONFIG.test_launcher == "dragon": + time.sleep(5) + kill_all_test_spawned_processes() def build_mpi_app() -> t.Optional[pathlib.Path]: diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 7137bed36..50b06cabd 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -234,8 +234,6 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: ) # Generate the job directory and return the generated job path ret = self._generate(generator, job, idx) - print(f"the type: {type(ret)}") - print(f"the val: {ret}") job_execution_path, out, err = ret id_ = launch_config.start(exe, job_execution_path, env, out, err) # Save the underlying launcher instance and launched job id. That diff --git a/smartsim/launchable/baseJobGroup.py b/smartsim/launchable/baseJobGroup.py index d662550f5..b7becba56 100644 --- a/smartsim/launchable/baseJobGroup.py +++ b/smartsim/launchable/baseJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/colocatedJobGroup.py b/smartsim/launchable/colocatedJobGroup.py index 97e7aa4a3..1c3b96fba 100644 --- a/smartsim/launchable/colocatedJobGroup.py +++ b/smartsim/launchable/colocatedJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index e680e5f14..4c29c3c9f 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -26,7 +26,6 @@ from __future__ import annotations -import os import typing as t from copy import deepcopy diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index fd288deb4..d2e64c454 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settings/arguments/launch/local.py b/smartsim/settings/arguments/launch/local.py index a61068cd4..0bbba2584 100644 --- a/smartsim/settings/arguments/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -27,7 +27,6 @@ from __future__ import annotations import typing as t -from subprocess import PIPE from smartsim.log import get_logger from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index 141e9adcd..f8e514ab2 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -32,16 +32,13 @@ import subprocess import typing as t -from smartsim._core.commands import Command from smartsim.log import get_logger from smartsim.settings.dispatch import ( ExecutableProtocol, ShellLauncher, ShellLauncherCommand, _EnvironMappingType, - _FormatterType, dispatch, - make_shell_format_fn, ) from ...common import set_check_input diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 77ac6c74a..3c66afda6 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -30,25 +30,19 @@ import collections.abc import dataclasses import io -import os import pathlib -import subprocess import subprocess as sp import typing as t import uuid -from subprocess import STDOUT import psutil from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack -from smartsim._core.commands import Command, CommandList from smartsim._core.utils import helpers from smartsim.error import errors from smartsim.status import JobStatus from smartsim.types import LaunchedJobID -from ..settings.launchCommand import LauncherType - if t.TYPE_CHECKING: from smartsim.experiment import Experiment from smartsim.settings.arguments import LaunchArguments @@ -522,10 +516,7 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - # covariant, contravariant, + boliscoff substitution princ - def start( - self, shell_command: ShellLauncherCommand # this should be a named tuple - ) -> LaunchedJobID: + def start(self, shell_command: ShellLauncherCommand) -> LaunchedJobID: id_ = create_job_id() # raise ValueError -> invalid stuff throw exe, *rest = shell_command.command_tuple @@ -537,7 +528,6 @@ def start( stdout=shell_command.stdout, stderr=shell_command.stderr, ) - # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ def get_status( @@ -549,14 +539,10 @@ def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: if (proc := self._launched.get(id_)) is None: msg = f"Launcher `{self}` has not launched a job with id `{id_}`" raise errors.LauncherJobNotFound(msg) - ret_code = ( - proc.poll() - ) # add a test that mocks out poll and raise some exception - terminal -> import subprocess -> start something echo blah - then poll and see what a valid fake output is - print(ret_code) + ret_code = proc.poll() # try/catch around here and then reaise a smartsim.error if ret_code is None: status = psutil.Process(proc.pid).status() - print(status) return { psutil.STATUS_RUNNING: JobStatus.RUNNING, psutil.STATUS_SLEEPING: JobStatus.RUNNING, diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 892978dfb..4160e60f1 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -23,12 +23,10 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pathlib import subprocess import pytest -from smartsim._core.commands import Command from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 6e4fd8a7f..3f1195e9c 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -29,7 +29,6 @@ import dataclasses import itertools import random -import tempfile import typing as t import uuid diff --git a/tests/test_generator.py b/tests/test_generator.py index 602f300ea..3b2c95cf2 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -14,7 +14,7 @@ from smartsim.entity import Application, Ensemble from smartsim.entity.files import EntityFiles from smartsim.launchable import Job -from smartsim.settings import LaunchSettings, dispatch +from smartsim.settings import LaunchSettings # TODO Add JobGroup tests when JobGroup becomes a Launchable diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 3eb6a7766..d9143c52a 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -24,34 +24,20 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import difflib import os import pathlib import subprocess -import tempfile import unittest.mock -import uuid -import weakref import psutil import pytest -from smartsim import Experiment -from smartsim._core.commands import Command from smartsim._core.utils import helpers from smartsim._core.utils.shell import * -from smartsim.entity import Application, _mock, entity +from smartsim.entity import _mock, entity from smartsim.error.errors import LauncherJobNotFound -from smartsim.launchable import Job -from smartsim.settings import LaunchSettings -from smartsim.settings.arguments.launch.slurm import ( - SlurmLaunchArguments, - _as_srun_command, -) from smartsim.settings.dispatch import ShellLauncher, ShellLauncherCommand, sp -from smartsim.settings.launchCommand import LauncherType from smartsim.status import JobStatus -from smartsim.types import LaunchedJobID # TODO tests bad vars in Popen call at beginning # tests -> helper.exe : pass in None, empty str, path with a space at beginning, a non valid command @@ -258,73 +244,60 @@ def test_shell_launcher_returns_complete_status(test_dir): def test_shell_launcher_returns_failed_status(test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" - # Init ShellLauncher shell_launcher = ShellLauncher() - # Generate testing directory run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err, ): - # Construct a invalid command to execute args = (helpers.expand_exe_path("srun"), "--flag_dne") cmd = ShellLauncherCommand({}, run_dir, out, err, args) - # Start the execution of the command using a ShellLauncher for _ in range(5): id = shell_launcher.start(cmd) - # Retrieve popen object proc = shell_launcher._launched[id] - # Wait for subprocess to complete proc.wait() - # Retrieve status of subprocess code = shell_launcher.get_status(id) val = list(code.keys())[0] - # Assert that subprocess has completed assert code[val] == JobStatus.FAILED def test_shell_launcher_returns_running_status(test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" - # Init ShellLauncher shell_launcher = ShellLauncher() - # Generate testing directory run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err, ): - # Construct a command to execute cmd = ShellLauncherCommand( {}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5") ) - # Start the execution of the command using a ShellLauncher for _ in range(5): id = shell_launcher.start(cmd) - # Retrieve status of subprocess code = shell_launcher.get_status(id) val = list(code.keys())[0] - # Assert that subprocess has completed assert code[val] == JobStatus.RUNNING @pytest.mark.parametrize( "psutil_status,job_status", [ - pytest.param(psutil.STATUS_RUNNING, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_SLEEPING, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_WAKING, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_DISK_SLEEP, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_DEAD, JobStatus.FAILED, id="merp"), - pytest.param(psutil.STATUS_TRACING_STOP, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_WAITING, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_STOPPED, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_LOCKED, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="merp"), + pytest.param(psutil.STATUS_RUNNING, JobStatus.RUNNING, id="running"), + pytest.param(psutil.STATUS_SLEEPING, JobStatus.RUNNING, id="sleeping"), + pytest.param(psutil.STATUS_WAKING, JobStatus.RUNNING, id="waking"), + pytest.param(psutil.STATUS_DISK_SLEEP, JobStatus.RUNNING, id="disk_sleep"), + pytest.param(psutil.STATUS_DEAD, JobStatus.FAILED, id="dead"), + pytest.param(psutil.STATUS_TRACING_STOP, JobStatus.PAUSED, id="tracing_stop"), + pytest.param(psutil.STATUS_WAITING, JobStatus.PAUSED, id="waiting"), + pytest.param(psutil.STATUS_STOPPED, JobStatus.PAUSED, id="stopped"), + pytest.param(psutil.STATUS_LOCKED, JobStatus.PAUSED, id="locked"), + pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="parked"), + pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="idle"), + pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="zombie"), ], ) def test_this(psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir): + """Test tht ShellLauncher.get_status returns correct mapping""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( From 2de69432cf287da1148827e05f033f1f9346e02f Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 20 Aug 2024 15:31:36 -0500 Subject: [PATCH 72/82] marked test --- tests/test_shell_launcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 5ce397e2a..f3f7618d3 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -45,6 +45,7 @@ # -> have border tests for 0,1,4,6 cmd vals -> work correctly without them -> raise ValueError # do all of the failures as well as the sucess criteria +pytestmark = pytest.mark.group_a class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" From d834cedf995857989e0c143ce48e190b372a0902 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 20 Aug 2024 15:38:35 -0500 Subject: [PATCH 73/82] make style --- tests/test_shell_launcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index f3f7618d3..8bf5a6aa0 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -47,6 +47,7 @@ pytestmark = pytest.mark.group_a + class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" From f4822d741c84b7327eca16909d133b6e01caf0b1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 20 Aug 2024 16:29:54 -0500 Subject: [PATCH 74/82] skip when slurm not avail --- tests/test_shell_launcher.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 8bf5a6aa0..8571bf759 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -26,6 +26,7 @@ import os import pathlib +import shutil import subprocess import unittest.mock @@ -47,6 +48,8 @@ pytestmark = pytest.mark.group_a +requires_slurm = pytest.mark.skipif(not shutil.which("srun"), reason="requires srun") + class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" @@ -179,6 +182,7 @@ def test_popen_writes_to_output_file(test_dir: str): print(val) +@requires_slurm def test_popen_fails_with_invalid_cmd(test_dir): """Test that popen returns a non zero returncode after failure""" shell_launcher = ShellLauncher() @@ -244,6 +248,7 @@ def test_shell_launcher_returns_complete_status(test_dir): assert code[val] == JobStatus.COMPLETED +@requires_slurm def test_shell_launcher_returns_failed_status(test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" shell_launcher = ShellLauncher() From ae913a18fad75b94d7efdeee7bba730a5ced2896 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Fri, 23 Aug 2024 16:40:45 -0500 Subject: [PATCH 75/82] address all of matts comments --- pyproject.toml | 1 - smartsim/_core/commands/command.py | 37 ++++++++-- smartsim/_core/commands/commandList.py | 39 ++++++++-- smartsim/_core/dispatch.py | 15 ++-- smartsim/_core/entrypoints/file_operations.py | 1 - smartsim/_core/generation/generator.py | 9 +-- smartsim/experiment.py | 3 +- smartsim/settings/arguments/launch/lsf.py | 4 +- smartsim/settings/arguments/launch/slurm.py | 4 +- .../test_core/test_commands/test_command.py | 24 +++++- .../test_commands/test_commandList.py | 37 +++++++++- .../test_settings/test_alpsLauncher.py | 8 +- .../test_settings/test_localLauncher.py | 8 +- .../test_settings/test_lsfLauncher.py | 24 +++--- .../test_settings/test_mpiLauncher.py | 4 +- .../test_settings/test_palsLauncher.py | 18 ++--- .../test_settings/test_slurmLauncher.py | 25 +++---- tests/test_shell_launcher.py | 73 +++++++++---------- 18 files changed, 215 insertions(+), 119 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5df64aa97..e11c252ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,7 +147,6 @@ module = [ # FIXME: DO NOT MERGE THIS INTO DEVELOP BRANCH UNLESS THESE ARE PASSING OR # REMOVED!! "smartsim._core._cli.*", - "smartsim._core.commands.*", "smartsim._core.control.controller", "smartsim._core.control.manifest", "smartsim._core.entrypoints.dragon_client", diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 1b279d116..bb335d521 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -26,6 +26,9 @@ import typing as t from collections.abc import MutableSequence +from copy import deepcopy + +from typing_extensions import Self class Command(MutableSequence[str]): @@ -42,15 +45,39 @@ def command(self) -> t.List[str]: """ return self._command - def __getitem__(self, idx: int) -> str: + @t.overload + def __getitem__(self, idx: int) -> str: ... + @t.overload + def __getitem__(self, idx: slice) -> Self: ... + def __getitem__(self, idx: int | slice) -> str | Self: """Get the command at the specified index.""" - return self._command[idx] + cmd = self._command[idx] + if isinstance(cmd, str): + return cmd + return type(self)(cmd) - def __setitem__(self, idx: int, value: str) -> None: + @t.overload + def __setitem__(self, idx: int, value: str) -> None: ... + @t.overload + def __setitem__(self, idx: slice, value: t.Iterable[str]) -> None: ... + def __setitem__(self, idx: int | slice, value: str | t.Iterable[str]) -> None: """Set the command at the specified index.""" - self._command[idx] = value + if isinstance(idx, int): + if not isinstance(value, str): + raise ValueError( + "Value must be of type `str` when assigning to an index" + ) + self._command[idx] = deepcopy(value) + elif isinstance(idx, slice): + if not isinstance(value, list) or not all( + isinstance(item, str) for item in value + ): + raise ValueError( + "Value must be a list of strings when assigning to a slice" + ) + self._command[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int) -> None: + def __delitem__(self, idx: int | slice) -> None: """Delete the command at the specified index.""" del self._command[idx] diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 08b95bbfd..681b0e819 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -26,6 +26,7 @@ import typing as t from collections.abc import MutableSequence +from copy import deepcopy from .command import Command @@ -46,15 +47,43 @@ def commands(self) -> t.List[Command]: """ return self._commands - def __getitem__(self, idx: int) -> Command: + @t.overload + def __getitem__(self, idx: int) -> Command: ... + @t.overload + def __getitem__(self, idx: slice) -> t.List[Command]: ... + def __getitem__(self, idx: slice | int) -> Command | t.List[Command]: """Get the Command at the specified index.""" return self._commands[idx] - def __setitem__(self, idx: int, value: Command) -> None: - """Set the Command at the specified index.""" - self._commands[idx] = value + @t.overload + def __setitem__(self, idx: int, value: Command) -> None: ... + @t.overload + def __setitem__(self, idx: slice, value: t.Iterable[Command]) -> None: ... + def __setitem__( + self, idx: int | slice, value: Command | t.Iterable[Command] + ) -> None: + """Set the Commands at the specified index.""" + if isinstance(idx, int): + if not isinstance(value, Command): + raise ValueError( + "Value must be of type `Command` when assigning to an index" + ) + self._commands[idx] = deepcopy(value) + elif isinstance(idx, slice): + if not isinstance(value, list): + raise ValueError( + "Value must be a list of Commands when assigning to a slice" + ) + for sublist in value: + if not isinstance(sublist.command, list) or not all( + isinstance(item, str) for item in sublist.command + ): + raise ValueError( + "Value sublists must be a list of Commands when assigning to a slice" + ) + self._commands[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int) -> None: + def __delitem__(self, idx: int | slice) -> None: """Delete the Command at the specified index.""" del self._commands[idx] diff --git a/smartsim/_core/dispatch.py b/smartsim/_core/dispatch.py index 3c66afda6..d940d54e7 100644 --- a/smartsim/_core/dispatch.py +++ b/smartsim/_core/dispatch.py @@ -89,9 +89,13 @@ class ShellLauncherCommand(t.NamedTuple): """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ -_LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType, pathlib.Path, pathlib.Path]" -) +_LaunchConfigType: TypeAlias = """_LauncherAdapter[ + ExecutableProtocol, + _WorkingDirectory, + _EnvironMappingType, + pathlib.Path, + pathlib.Path]""" + """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings """ @@ -511,16 +515,14 @@ def impl( class ShellLauncher: """Mock launcher for launching/tracking simple shell commands""" - # add a def check - def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} def start(self, shell_command: ShellLauncherCommand) -> LaunchedJobID: id_ = create_job_id() - # raise ValueError -> invalid stuff throw exe, *rest = shell_command.command_tuple expanded_exe = helpers.expand_exe_path(exe) + # pylint: disable-next=consider-using-with self._launched[id_] = sp.Popen( (expanded_exe, *rest), cwd=shell_command.path, @@ -540,7 +542,6 @@ def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: msg = f"Launcher `{self}` has not launched a job with id `{id_}`" raise errors.LauncherJobNotFound(msg) ret_code = proc.poll() - # try/catch around here and then reaise a smartsim.error if ret_code is None: status = psutil.Process(proc.pid).status() return { diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 4271c2a63..618d30571 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -34,7 +34,6 @@ import pickle import shutil import typing as t -from os import path as osp from typing import Callable from ...log import get_logger diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 37f6ce419..6f7ab6d84 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -148,14 +148,9 @@ def generate_job( # Create output files out_file, err_file = self._output_files(log_path, job.entity.name) # Open and write to .out file - with open(out_file, mode="w", encoding="utf-8") as log_file: - dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - log_file.write(f"Generation start date and time: {dt_string}\n") - + open(out_file, mode="w", encoding="utf-8") # Open and write to .err file - with open(err_file, mode="w", encoding="utf-8") as log_file: - dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - log_file.write(f"Generation start date and time: {dt_string}\n") + open(err_file, mode="w", encoding="utf-8") # Perform file system operations on attached files self._build_operations(job, job_path) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 00e25ba6d..2103505a6 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -233,8 +233,7 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: for_experiment=self, with_arguments=args ) # Generate the job directory and return the generated job path - ret = self._generate(generator, job, idx) - job_execution_path, out, err = ret + job_execution_path, out, err = self._generate(generator, job, idx) id_ = launch_config.start(exe, job_execution_path, env, out, err) # Save the underlying launcher instance and launched job id. That # way we do not need to spin up a launcher instance for each diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index 3696ac5bc..607089782 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -58,10 +58,10 @@ def _as_jsrun_command( command_tuple = ( "jsrun", *(args.format_launch_args() or ()), - "--", - *exe.as_program_arguments(), f"--stdio_stdout={stdout_path}", f"--stdio_stderr={stderr_path}", + "--", + *exe.as_program_arguments(), ) # add output and err to CMD tuple -> add dev Null for stdout and stderr return ShellLauncherCommand( diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index c32e4b6bc..ab33c8e81 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -60,10 +60,10 @@ def _as_srun_command( command_tuple = ( "srun", *(args.format_launch_args() or ()), - "--", - *exe.as_program_arguments(), f"--output={stdout_path}", f"--error={stderr_path}", + "--", + *exe.as_program_arguments(), ) # add output and err to CMD tuple -> add dev Null for stdout and stderr return ShellLauncherCommand( diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index c7643335f..2d1ddfbe8 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -36,19 +36,39 @@ def test_command_init(): assert cmd.command == ["salloc", "-N", "1"] -def test_command_getitem(): +def test_command_getitem_int(): cmd = Command(command=["salloc", "-N", "1"]) get_value = cmd[0] assert get_value == "salloc" -def test_command_setitem(): +def test_command_getitem_slice(): + cmd = Command(command=["salloc", "-N", "1"]) + get_value = cmd[0:2] + assert get_value.command == ["salloc", "-N"] + + +def test_command_setitem_int(): cmd = Command(command=["salloc", "-N", "1"]) cmd[0] = "srun" cmd[1] = "-n" assert cmd.command == ["srun", "-n", "1"] +def test_command_setitem_slice(): + cmd = Command(command=["salloc", "-N", "1"]) + cmd[0:2] = ["srun", "-n"] + assert cmd.command == ["srun", "-n", "1"] + + +def test_command_setitem_fail(): + cmd = Command(command=["salloc", "-N", "1"]) + with pytest.raises(ValueError): + cmd[0] = 1 + with pytest.raises(ValueError): + cmd[0:2] = [1, "-n"] + + def test_command_delitem(): cmd = Command( command=["salloc", "-N", "1", "--constraint", "P100"], diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index a30107b99..79d6f7e78 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -42,16 +42,47 @@ def test_command_init(): assert cmd_list.commands == [salloc_cmd, srun_cmd] -def test_command_getitem(): +def test_command_getitem_int(): cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) get_value = cmd_list[0] assert get_value == salloc_cmd -def test_command_setitem(): +def test_command_getitem_slice(): + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) + get_value = cmd_list[0:2] + assert get_value == [salloc_cmd, srun_cmd] + + +def test_command_setitem_idx(): cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) cmd_list[0] = sacct_cmd - assert cmd_list.commands == [sacct_cmd, srun_cmd] + for cmd in cmd_list.commands: + assert cmd.command in [sacct_cmd.command, srun_cmd.command] + + +def test_command_setitem_slice(): + cmd_list = CommandList(commands=[srun_cmd, srun_cmd]) + cmd_list[0:2] = [sacct_cmd, sacct_cmd] + for cmd in cmd_list.commands: + assert cmd.command == sacct_cmd.command + + +def test_command_setitem_fail(): + cmd_list = CommandList(commands=[srun_cmd, srun_cmd]) + with pytest.raises(ValueError): + cmd_list[0] = "fail" + with pytest.raises(ValueError): + cmd_list[0:1] = "fail" + with pytest.raises(ValueError): + cmd_list[0:1] = "fail" + cmd_1 = Command(command=["salloc", "-N", 1]) + cmd_2 = Command(command=["salloc", "-N", "1"]) + cmd_3 = Command(command=1) + with pytest.raises(ValueError): + cmd_list[0:1] = [cmd_1, cmd_2] + with pytest.raises(ValueError): + cmd_list[0:1] = [cmd_3, cmd_2] def test_command_delitem(): diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 3bec0eb30..6a15fa8e9 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -218,10 +218,10 @@ def test_invalid_exclude_hostlist_format(): def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): out = os.path.join(test_dir, "out.txt") err = os.path.join(test_dir, "err.txt") - with open(out, "w") as _, open(err, "w") as _: - shell_launch_cmd = _as_aprun_command( - AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, out, err - ) + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_aprun_command( + AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, out, err + ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == expected assert shell_launch_cmd.path == pathlib.Path(test_dir) diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 4b5970d19..df871c6a3 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -150,10 +150,10 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): out = os.path.join(test_dir, "out.txt") err = os.path.join(test_dir, "err.txt") - with open(out, "w") as _, open(err, "w") as _: - shell_launch_cmd = _as_local_command( - LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, out, err - ) + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_local_command( + LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, out, err + ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == ("echo", "hello", "world") assert shell_launch_cmd.path == pathlib.Path(test_dir) diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index eef23f530..9a9c68e67 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -97,12 +97,12 @@ def test_launch_args(): {}, ( "jsrun", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", "--", "echo", "hello", "world", - "--stdio_stdout=output.txt", - "--stdio_stderr=error.txt", ), id="Empty Args", ), @@ -112,12 +112,12 @@ def test_launch_args(): "jsrun", "-n", "1", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", "--", "echo", "hello", "world", - "--stdio_stdout=output.txt", - "--stdio_stderr=error.txt", ), id="Short Arg", ), @@ -126,12 +126,12 @@ def test_launch_args(): ( "jsrun", "--nrs=1", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", "--", "echo", "hello", "world", - "--stdio_stdout=output.txt", - "--stdio_stderr=error.txt", ), id="Long Arg", ), @@ -140,12 +140,12 @@ def test_launch_args(): ( "jsrun", "-v", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", "--", "echo", "hello", "world", - "--stdio_stdout=output.txt", - "--stdio_stderr=error.txt", ), id="Short Arg (No Value)", ), @@ -154,12 +154,12 @@ def test_launch_args(): ( "jsrun", "--verbose", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", "--", "echo", "hello", "world", - "--stdio_stdout=output.txt", - "--stdio_stderr=error.txt", ), id="Long Arg (No Value)", ), @@ -170,12 +170,12 @@ def test_launch_args(): "--tasks_per_rs=1", "-n", "123", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", "--", "echo", "hello", "world", - "--stdio_stdout=output.txt", - "--stdio_stderr=error.txt", ), id="Short and Long Args", ), diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index c43d1e264..31893ed37 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -292,8 +292,8 @@ def test_formatting_launch_args( ): out = os.path.join(test_dir, "out.txt") err = os.path.join(test_dir, "err.txt") - with open(out, "w") as _, open(err, "w") as _: - shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, out, err) + open(out, "w"), open(err, "w") + shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, out, err) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == (cmd,) + expected assert shell_launch_cmd.path == pathlib.Path(test_dir) diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index acdf6c6a1..b4e860ec8 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -139,15 +139,15 @@ def test_invalid_hostlist_format(): def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): out = os.path.join(test_dir, "out.txt") err = os.path.join(test_dir, "err.txt") - with open(out, "w") as _, open(err, "w") as _: - shell_launch_cmd = _as_pals_command( - PalsMpiexecLaunchArguments(args), - mock_echo_executable, - test_dir, - {}, - out, - err, - ) + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_pals_command( + PalsMpiexecLaunchArguments(args), + mock_echo_executable, + test_dir, + {}, + out, + err, + ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) assert shell_launch_cmd.command_tuple == expected assert shell_launch_cmd.path == pathlib.Path(test_dir) diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 206d4965a..fcaca01cf 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -295,12 +295,12 @@ def test_set_het_groups(monkeypatch): {}, ( "srun", + "--output=output.txt", + "--error=error.txt", "--", "echo", "hello", "world", - "--output=output.txt", - "--error=error.txt", ), id="Empty Args", ), @@ -310,12 +310,12 @@ def test_set_het_groups(monkeypatch): "srun", "-N", "1", + "--output=output.txt", + "--error=error.txt", "--", "echo", "hello", "world", - "--output=output.txt", - "--error=error.txt", ), id="Short Arg", ), @@ -324,12 +324,12 @@ def test_set_het_groups(monkeypatch): ( "srun", "--nodes=1", + "--output=output.txt", + "--error=error.txt", "--", "echo", "hello", "world", - "--output=output.txt", - "--error=error.txt", ), id="Long Arg", ), @@ -338,12 +338,12 @@ def test_set_het_groups(monkeypatch): ( "srun", "-v", + "--output=output.txt", + "--error=error.txt", "--", "echo", "hello", "world", - "--output=output.txt", - "--error=error.txt", ), id="Short Arg (No Value)", ), @@ -352,12 +352,12 @@ def test_set_het_groups(monkeypatch): ( "srun", "--verbose", + "--output=output.txt", + "--error=error.txt", "--", "echo", "hello", "world", - "--output=output.txt", - "--error=error.txt", ), id="Long Arg (No Value)", ), @@ -368,12 +368,12 @@ def test_set_het_groups(monkeypatch): "--nodes=1", "-n", "123", + "--output=output.txt", + "--error=error.txt", "--", "echo", "hello", "world", - "--output=output.txt", - "--error=error.txt", ), id="Short and Long Args", ), @@ -389,7 +389,6 @@ def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): stderr_path="error.txt", ) assert isinstance(shell_launch_cmd, ShellLauncherCommand) - print(shell_launch_cmd.command_tuple) assert shell_launch_cmd.command_tuple == expected assert shell_launch_cmd.path == test_dir assert shell_launch_cmd.env == {} diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 8571bf759..7ea7868f9 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -87,6 +87,14 @@ def generate_directory(test_dir: str): return execution_dir, out_file, err_file +@pytest.fixture +def shell_launcher(): + launcher = ShellLauncher() + yield launcher + if any(proc.poll() is None for proc in launcher._launched.values()): + raise ("Test leaked processes") + + @pytest.fixture def shell_cmd(test_dir: str) -> ShellLauncherCommand: """Fixture to create an instance of Generator.""" @@ -108,23 +116,24 @@ def test_shell_launcher_command_init(shell_cmd: ShellLauncherCommand, test_dir: assert shell_cmd.command_tuple == EchoHelloWorldEntity().as_program_arguments() -def test_shell_launcher_init(): +def test_shell_launcher_init(shell_launcher): """Test that ShellLauncher initializes correctly""" - shell_launcher = ShellLauncher() assert shell_launcher._launched == {} -def test_shell_launcher_start_calls_popen(shell_cmd: ShellLauncherCommand): +def test_shell_launcher_start_calls_popen( + shell_launcher, shell_cmd: ShellLauncherCommand +): """Test that the process leading up to the shell launcher popen call was correct""" - shell_launcher = ShellLauncher() with unittest.mock.patch("smartsim._core.dispatch.sp.Popen") as mock_open: _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once() -def test_shell_launcher_start_calls_popen_with_value(shell_cmd: ShellLauncherCommand): +def test_shell_launcher_start_calls_popen_with_value( + shell_launcher, shell_cmd: ShellLauncherCommand +): """Test that popen was called with correct values""" - shell_launcher = ShellLauncher() with unittest.mock.patch("smartsim._core.dispatch.sp.Popen") as mock_open: _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once_with( @@ -136,9 +145,8 @@ def test_shell_launcher_start_calls_popen_with_value(shell_cmd: ShellLauncherCom ) -def test_popen_returns_popen_object(test_dir: str): +def test_popen_returns_popen_object(shell_launcher, test_dir: str): """Test that the popen call returns a popen object""" - shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, @@ -152,13 +160,12 @@ def test_popen_returns_popen_object(test_dir: str): EchoHelloWorldEntity().as_program_arguments(), ) id = shell_launcher.start(cmd) - proc = shell_launcher._launched[id] - assert isinstance(proc, sp.Popen) + with shell_launcher._launched[id] as proc: + assert isinstance(proc, sp.Popen) -def test_popen_writes_to_output_file(test_dir: str): +def test_popen_writes_to_output_file(shell_launcher, test_dir: str): """Test that popen writes to .out file upon successful process call""" - shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, @@ -168,24 +175,19 @@ def test_popen_writes_to_output_file(test_dir: str): {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() ) id = shell_launcher.start(cmd) - val = shell_launcher.get_status(id) - print(val) + out.close(), err.close() proc = shell_launcher._launched[id] - # Wait for subprocess to finish assert proc.wait() == 0 assert proc.returncode == 0 with open(out_file, "r", encoding="utf-8") as out: assert out.read() == "Hello World!\n" with open(err_file, "r", encoding="utf-8") as err: assert err.read() == "" - val = shell_launcher.get_status(id) - print(val) @requires_slurm -def test_popen_fails_with_invalid_cmd(test_dir): +def test_popen_fails_with_invalid_cmd(shell_launcher, test_dir): """Test that popen returns a non zero returncode after failure""" - shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, @@ -204,9 +206,8 @@ def test_popen_fails_with_invalid_cmd(test_dir): assert "unrecognized option" in content -def test_popen_issues_unique_ids(test_dir): +def test_popen_issues_unique_ids(shell_launcher, test_dir): """Validate that all ids are unique within ShellLauncher._launched""" - shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, @@ -218,19 +219,18 @@ def test_popen_issues_unique_ids(test_dir): for _ in range(5): _ = shell_launcher.start(cmd) assert len(shell_launcher._launched) == 5 + assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) -def test_retrieve_status_dne(): +def test_retrieve_status_dne(shell_launcher): """Test tht ShellLauncher returns the status of completed Jobs""" # Init ShellLauncher - shell_launcher = ShellLauncher() with pytest.raises(LauncherJobNotFound): _ = shell_launcher.get_status("dne") -def test_shell_launcher_returns_complete_status(test_dir): +def test_shell_launcher_returns_complete_status(shell_launcher, test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" - shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, @@ -243,15 +243,13 @@ def test_shell_launcher_returns_complete_status(test_dir): id = shell_launcher.start(cmd) proc = shell_launcher._launched[id] proc.wait() - code = shell_launcher.get_status(id) - val = list(code.keys())[0] - assert code[val] == JobStatus.COMPLETED + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.COMPLETED @requires_slurm -def test_shell_launcher_returns_failed_status(test_dir): +def test_shell_launcher_returns_failed_status(shell_launcher, test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" - shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, @@ -263,14 +261,12 @@ def test_shell_launcher_returns_failed_status(test_dir): id = shell_launcher.start(cmd) proc = shell_launcher._launched[id] proc.wait() - code = shell_launcher.get_status(id) - val = list(code.keys())[0] - assert code[val] == JobStatus.FAILED + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.FAILED -def test_shell_launcher_returns_running_status(test_dir): +def test_shell_launcher_returns_running_status(shell_launcher, test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" - shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, @@ -281,9 +277,9 @@ def test_shell_launcher_returns_running_status(test_dir): ) for _ in range(5): id = shell_launcher.start(cmd) - code = shell_launcher.get_status(id) - val = list(code.keys())[0] - assert code[val] == JobStatus.RUNNING + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.RUNNING + assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) @pytest.mark.parametrize( @@ -320,3 +316,4 @@ def test_this(psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_d monkeypatch.setattr(psutil.Process, "status", lambda self: psutil_status) value = shell_launcher.get_status(id) assert value.get(id) == job_status + assert proc.wait() == 0 From 20844d00cb826e530ab7c73c185610821089d367 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 27 Aug 2024 00:26:55 -0500 Subject: [PATCH 76/82] mypy updates, resolving | issues --- smartsim/_core/commands/command.py | 6 +++--- smartsim/_core/commands/commandList.py | 6 +++--- smartsim/_core/shell/shellLauncher.py | 5 +---- smartsim/settings/arguments/launch/lsf.py | 3 +-- smartsim/settings/arguments/launch/slurm.py | 3 +-- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index bb335d521..0769d342e 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -49,7 +49,7 @@ def command(self) -> t.List[str]: def __getitem__(self, idx: int) -> str: ... @t.overload def __getitem__(self, idx: slice) -> Self: ... - def __getitem__(self, idx: int | slice) -> str | Self: + def __getitem__(self, idx: t.Union[int, slice]) -> t.Union[str, Self]: """Get the command at the specified index.""" cmd = self._command[idx] if isinstance(cmd, str): @@ -60,7 +60,7 @@ def __getitem__(self, idx: int | slice) -> str | Self: def __setitem__(self, idx: int, value: str) -> None: ... @t.overload def __setitem__(self, idx: slice, value: t.Iterable[str]) -> None: ... - def __setitem__(self, idx: int | slice, value: str | t.Iterable[str]) -> None: + def __setitem__(self, idx: t.Union[int, slice], value: t.Union[str, t.Iterable[str]]) -> None: """Set the command at the specified index.""" if isinstance(idx, int): if not isinstance(value, str): @@ -77,7 +77,7 @@ def __setitem__(self, idx: int | slice, value: str | t.Iterable[str]) -> None: ) self._command[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int | slice) -> None: + def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the command at the specified index.""" del self._command[idx] diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 681b0e819..92d4dd8df 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -51,7 +51,7 @@ def commands(self) -> t.List[Command]: def __getitem__(self, idx: int) -> Command: ... @t.overload def __getitem__(self, idx: slice) -> t.List[Command]: ... - def __getitem__(self, idx: slice | int) -> Command | t.List[Command]: + def __getitem__(self, idx: t.Union[slice, int]) -> t.Union[Command, t.List[Command]]: """Get the Command at the specified index.""" return self._commands[idx] @@ -60,7 +60,7 @@ def __setitem__(self, idx: int, value: Command) -> None: ... @t.overload def __setitem__(self, idx: slice, value: t.Iterable[Command]) -> None: ... def __setitem__( - self, idx: int | slice, value: Command | t.Iterable[Command] + self, idx: t.Union[int, slice], value: t.Union[Command, t.Iterable[Command]] ) -> None: """Set the Commands at the specified index.""" if isinstance(idx, int): @@ -83,7 +83,7 @@ def __setitem__( ) self._commands[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int | slice) -> None: + def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the Command at the specified index.""" del self._commands[idx] diff --git a/smartsim/_core/shell/shellLauncher.py b/smartsim/_core/shell/shellLauncher.py index 6e32d11e6..a3270fc12 100644 --- a/smartsim/_core/shell/shellLauncher.py +++ b/smartsim/_core/shell/shellLauncher.py @@ -27,7 +27,6 @@ from __future__ import annotations -import os import subprocess as sp import typing as t import io @@ -62,9 +61,7 @@ class ShellLauncherCommand(t.NamedTuple): def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[ - ShellLaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]] -]: +) -> _FormatterType[ShellLaunchArguments, ShellLauncherCommand]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index ce7833149..52b4035b5 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -46,7 +46,7 @@ def _as_jsrun_command( - args: LaunchArguments, + args: ShellLaunchArguments, exe: ExecutableProtocol, path: pathlib.Path, env: _EnvironMappingType, @@ -61,7 +61,6 @@ def _as_jsrun_command( "--", *exe.as_program_arguments(), ) - # add output and err to CMD tuple -> add dev Null for stdout and stderr return ShellLauncherCommand( env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple ) diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index c0539d6b3..6e8c2d130 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -50,7 +50,7 @@ def _as_srun_command( - args: LaunchArguments, + args: ShellLaunchArguments, exe: ExecutableProtocol, path: pathlib.Path, env: _EnvironMappingType, @@ -65,7 +65,6 @@ def _as_srun_command( "--", *exe.as_program_arguments(), ) - # add output and err to CMD tuple -> add dev Null for stdout and stderr return ShellLauncherCommand( env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple ) From e9058742fa139cf1bb50c9dd2ebf5db6357ed06c Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 27 Aug 2024 01:05:17 -0500 Subject: [PATCH 77/82] styling --- smartsim/_core/commands/command.py | 4 +++- smartsim/_core/commands/commandList.py | 4 +++- smartsim/_core/dispatch.py | 1 - smartsim/_core/shell/shellLauncher.py | 11 ++++++++--- smartsim/settings/arguments/launch/lsf.py | 7 ++----- smartsim/settings/arguments/launch/slurm.py | 9 ++------- tests/test_shell_launcher.py | 8 ++++++-- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 0769d342e..3e47a0c40 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -60,7 +60,9 @@ def __getitem__(self, idx: t.Union[int, slice]) -> t.Union[str, Self]: def __setitem__(self, idx: int, value: str) -> None: ... @t.overload def __setitem__(self, idx: slice, value: t.Iterable[str]) -> None: ... - def __setitem__(self, idx: t.Union[int, slice], value: t.Union[str, t.Iterable[str]]) -> None: + def __setitem__( + self, idx: t.Union[int, slice], value: t.Union[str, t.Iterable[str]] + ) -> None: """Set the command at the specified index.""" if isinstance(idx, int): if not isinstance(value, str): diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 92d4dd8df..a6018aa24 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -51,7 +51,9 @@ def commands(self) -> t.List[Command]: def __getitem__(self, idx: int) -> Command: ... @t.overload def __getitem__(self, idx: slice) -> t.List[Command]: ... - def __getitem__(self, idx: t.Union[slice, int]) -> t.Union[Command, t.List[Command]]: + def __getitem__( + self, idx: t.Union[slice, int] + ) -> t.Union[Command, t.List[Command]]: """Get the Command at the specified index.""" return self._commands[idx] diff --git a/smartsim/_core/dispatch.py b/smartsim/_core/dispatch.py index 029d22506..f16d3f294 100644 --- a/smartsim/_core/dispatch.py +++ b/smartsim/_core/dispatch.py @@ -29,7 +29,6 @@ import dataclasses import os import pathlib - import typing as t from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack diff --git a/smartsim/_core/shell/shellLauncher.py b/smartsim/_core/shell/shellLauncher.py index a3270fc12..aaa864aaa 100644 --- a/smartsim/_core/shell/shellLauncher.py +++ b/smartsim/_core/shell/shellLauncher.py @@ -27,16 +27,19 @@ from __future__ import annotations -import subprocess as sp -import typing as t import io import pathlib import subprocess as sp +import typing as t import psutil -from smartsim._core.dispatch import _EnvironMappingType, _FormatterType, _WorkingDirectory from smartsim._core.arguments.shell import ShellLaunchArguments +from smartsim._core.dispatch import ( + _EnvironMappingType, + _FormatterType, + _WorkingDirectory, +) from smartsim._core.utils import helpers from smartsim._core.utils.launcher import ExecutableProtocol, create_job_id from smartsim.error import errors @@ -52,6 +55,7 @@ logger = get_logger(__name__) + class ShellLauncherCommand(t.NamedTuple): env: _EnvironMappingType path: pathlib.Path @@ -59,6 +63,7 @@ class ShellLauncherCommand(t.NamedTuple): stderr: io.TextIOWrapper | int command_tuple: tuple[str, tuple[str, ...]] | t.Sequence[str] + def make_shell_format_fn( run_command: str | None, ) -> _FormatterType[ShellLaunchArguments, ShellLauncherCommand]: diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index 52b4035b5..bc8571dcf 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -30,13 +30,10 @@ import subprocess import typing as t -from smartsim._core.dispatch import ( - _EnvironMappingType, - dispatch, -) +from smartsim._core.arguments.shell import ShellLaunchArguments +from smartsim._core.dispatch import _EnvironMappingType, dispatch from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand from smartsim._core.utils.launcher import ExecutableProtocol -from smartsim._core.arguments.shell import ShellLaunchArguments from smartsim.log import get_logger from ...common import set_check_input diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index 6e8c2d130..b842a647c 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -32,15 +32,10 @@ import subprocess import typing as t -from smartsim._core.dispatch import ( - _EnvironMappingType, - dispatch, -) +from smartsim._core.arguments.shell import ShellLaunchArguments +from smartsim._core.dispatch import _EnvironMappingType, dispatch from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand from smartsim._core.utils.launcher import ExecutableProtocol -from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import dispatch - from smartsim.log import get_logger from ...common import set_check_input diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index dbee64aa8..c1c447da7 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -125,7 +125,9 @@ def test_shell_launcher_start_calls_popen( shell_launcher, shell_cmd: ShellLauncherCommand ): """Test that the process leading up to the shell launcher popen call was correct""" - with unittest.mock.patch("smartsim._core.shell.shellLauncher.sp.Popen") as mock_open: + with unittest.mock.patch( + "smartsim._core.shell.shellLauncher.sp.Popen" + ) as mock_open: _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once() @@ -134,7 +136,9 @@ def test_shell_launcher_start_calls_popen_with_value( shell_launcher, shell_cmd: ShellLauncherCommand ): """Test that popen was called with correct values""" - with unittest.mock.patch("smartsim._core.shell.shellLauncher.sp.Popen") as mock_open: + with unittest.mock.patch( + "smartsim._core.shell.shellLauncher.sp.Popen" + ) as mock_open: _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once_with( shell_cmd.command_tuple, From c1145699316bbdb6741ce4dae58e3334542401cf Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 27 Aug 2024 11:20:47 -0500 Subject: [PATCH 78/82] working on invalid char in file path when loading artifact --- .github/workflows/run_tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 3230d8da8..e71d9f65d 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -158,6 +158,11 @@ jobs: echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests + - name: try zip the test result folder + if: failure() + run: | + zip -r testResult.zip tests/test_output + # Upload artifacts on failure, ignoring binary files - name: Upload Artifact if: failure() @@ -165,7 +170,7 @@ jobs: with: name: test_artifact path: | - tests/test_output + testResult.zip !**/*.so !**/*.pb !**/*.pt From 1a555aa8513880293d7972160f54b9dfdc71e8cc Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 27 Aug 2024 17:22:27 -0500 Subject: [PATCH 79/82] addressing matts comments + styling --- smartsim/_core/commands/command.py | 16 +-- smartsim/_core/commands/commandList.py | 22 ++-- smartsim/_core/dispatch.py | 24 ++-- smartsim/_core/generation/generator.py | 11 +- smartsim/_core/shell/shellLauncher.py | 20 +-- smartsim/settings/arguments/launch/lsf.py | 4 +- smartsim/settings/arguments/launch/slurm.py | 4 +- .../test_settings/test_alpsLauncher.py | 2 + .../test_settings/test_localLauncher.py | 2 + .../test_settings/test_mpiLauncher.py | 2 + .../test_settings/test_palsLauncher.py | 2 + tests/test_experiment.py | 2 +- tests/test_shell_launcher.py | 118 ++++++++++-------- 13 files changed, 123 insertions(+), 106 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 3e47a0c40..3f41f32fe 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -70,14 +70,14 @@ def __setitem__( "Value must be of type `str` when assigning to an index" ) self._command[idx] = deepcopy(value) - elif isinstance(idx, slice): - if not isinstance(value, list) or not all( - isinstance(item, str) for item in value - ): - raise ValueError( - "Value must be a list of strings when assigning to a slice" - ) - self._command[idx] = (deepcopy(val) for val in value) + return + if not isinstance(value, list) or not all( + isinstance(item, str) for item in value + ): + raise ValueError( + "Value must be a list of strings when assigning to a slice" + ) + self._command[idx] = (deepcopy(val) for val in value) def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the command at the specified index.""" diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index a6018aa24..34743063e 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -71,19 +71,19 @@ def __setitem__( "Value must be of type `Command` when assigning to an index" ) self._commands[idx] = deepcopy(value) - elif isinstance(idx, slice): - if not isinstance(value, list): + return + if not isinstance(value, list): + raise ValueError( + "Value must be a list of Commands when assigning to a slice" + ) + for sublist in value: + if not isinstance(sublist.command, list) or not all( + isinstance(item, str) for item in sublist.command + ): raise ValueError( - "Value must be a list of Commands when assigning to a slice" + "Value sublists must be a list of Commands when assigning to a slice" ) - for sublist in value: - if not isinstance(sublist.command, list) or not all( - isinstance(item, str) for item in sublist.command - ): - raise ValueError( - "Value sublists must be a list of Commands when assigning to a slice" - ) - self._commands[idx] = (deepcopy(val) for val in value) + self._commands[idx] = (deepcopy(val) for val in value) def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the Command at the specified index.""" diff --git a/smartsim/_core/dispatch.py b/smartsim/_core/dispatch.py index f16d3f294..551c27d18 100644 --- a/smartsim/_core/dispatch.py +++ b/smartsim/_core/dispatch.py @@ -47,7 +47,7 @@ _Ts = TypeVarTuple("_Ts") -_WorkingDirectory: TypeAlias = pathlib.Path +WorkingDirectory: TypeAlias = pathlib.Path """A working directory represented as a string or PathLike object""" _DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") @@ -59,16 +59,16 @@ to the to the `LauncherProtocol.start` method """ -_EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] +EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] """A mapping of user provided mapping of environment variables in which to run a job """ -_FormatterType: TypeAlias = t.Callable[ +FormatterType: TypeAlias = t.Callable[ [ _DispatchableT, "ExecutableProtocol", - _WorkingDirectory, - _EnvironMappingType, + WorkingDirectory, + EnvironMappingType, pathlib.Path, pathlib.Path, ], @@ -79,8 +79,8 @@ """ _LaunchConfigType: TypeAlias = """_LauncherAdapter[ ExecutableProtocol, - _WorkingDirectory, - _EnvironMappingType, + WorkingDirectory, + EnvironMappingType, pathlib.Path, pathlib.Path]""" @@ -145,7 +145,7 @@ def dispatch( # Signature when used as a decorator self, args: None = ..., *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]]: ... @@ -154,7 +154,7 @@ def dispatch( # Signature when used as a method self, args: type[_DispatchableT], *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> None: ... @@ -162,7 +162,7 @@ def dispatch( # Actual implementation self, args: type[_DispatchableT] | None = None, *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = False, ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]] | None: @@ -228,7 +228,7 @@ class _DispatchRegistration(t.Generic[_DispatchableT, _LaunchableT]): to be launched by the afore mentioned launcher. """ - formatter: _FormatterType[_DispatchableT, _LaunchableT] + formatter: FormatterType[_DispatchableT, _LaunchableT] launcher_type: type[LauncherProtocol[_LaunchableT]] def _is_compatible_launcher(self, launcher: LauncherProtocol[t.Any]) -> bool: @@ -273,7 +273,7 @@ def create_adapter_from_launcher( def format_( exe: ExecutableProtocol, path: pathlib.Path, - env: _EnvironMappingType, + env: EnvironMappingType, out: pathlib.Path, err: pathlib.Path, ) -> _LaunchableT: diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 6f7ab6d84..e4018ccc3 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -108,11 +108,12 @@ def _log_file(log_path: pathlib.Path) -> pathlib.Path: """ return pathlib.Path(log_path) / "smartsim_params.txt" + @staticmethod def _output_files( - self, log_path: pathlib.Path, job_name: str + log_path: pathlib.Path, job_name: str ) -> t.Tuple[pathlib.Path, pathlib.Path]: - out_file_path = log_path / (job_name + ".out") - err_file_path = log_path / (job_name + ".err") + out_file_path = log_path / f"{job_name}.out" + err_file_path = log_path / f"{job_name}.err" return out_file_path, err_file_path def generate_job( @@ -147,10 +148,6 @@ def generate_job( # Create output files out_file, err_file = self._output_files(log_path, job.entity.name) - # Open and write to .out file - open(out_file, mode="w", encoding="utf-8") - # Open and write to .err file - open(err_file, mode="w", encoding="utf-8") # Perform file system operations on attached files self._build_operations(job, job_path) diff --git a/smartsim/_core/shell/shellLauncher.py b/smartsim/_core/shell/shellLauncher.py index aaa864aaa..8fd1d5306 100644 --- a/smartsim/_core/shell/shellLauncher.py +++ b/smartsim/_core/shell/shellLauncher.py @@ -35,11 +35,7 @@ import psutil from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import ( - _EnvironMappingType, - _FormatterType, - _WorkingDirectory, -) +from smartsim._core.dispatch import EnvironMappingType, FormatterType, WorkingDirectory from smartsim._core.utils import helpers from smartsim._core.utils.launcher import ExecutableProtocol, create_job_id from smartsim.error import errors @@ -57,7 +53,7 @@ class ShellLauncherCommand(t.NamedTuple): - env: _EnvironMappingType + env: EnvironMappingType path: pathlib.Path stdout: io.TextIOWrapper | int stderr: io.TextIOWrapper | int @@ -66,7 +62,7 @@ class ShellLauncherCommand(t.NamedTuple): def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[ShellLaunchArguments, ShellLauncherCommand]: +) -> FormatterType[ShellLaunchArguments, ShellLauncherCommand]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -99,8 +95,8 @@ def make_shell_format_fn( def impl( args: ShellLaunchArguments, exe: ExecutableProtocol, - path: _WorkingDirectory, - env: _EnvironMappingType, + path: WorkingDirectory, + env: EnvironMappingType, stdout_path: pathlib.Path, stderr_path: pathlib.Path, ) -> ShellLauncherCommand: @@ -114,6 +110,7 @@ def impl( if run_command is not None else exe.as_program_arguments() ) + # pylint: disable-next=consider-using-with return ShellLauncherCommand( env, pathlib.Path(path), open(stdout_path), open(stderr_path), command_tuple ) @@ -127,7 +124,12 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} + def check_popen_inputs(self, shell_command: ShellLauncherCommand) -> None: + if not shell_command.path.exists(): + raise ValueError("Please provide a valid shell command path.") + def start(self, shell_command: ShellLauncherCommand) -> LaunchedJobID: + self.check_popen_inputs(shell_command) id_ = create_job_id() exe, *rest = shell_command.command_tuple expanded_exe = helpers.expand_exe_path(exe) diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index bc8571dcf..6177cb6b6 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -31,7 +31,7 @@ import typing as t from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import _EnvironMappingType, dispatch +from smartsim._core.dispatch import EnvironMappingType, dispatch from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand from smartsim._core.utils.launcher import ExecutableProtocol from smartsim.log import get_logger @@ -46,7 +46,7 @@ def _as_jsrun_command( args: ShellLaunchArguments, exe: ExecutableProtocol, path: pathlib.Path, - env: _EnvironMappingType, + env: EnvironMappingType, stdout_path: pathlib.Path, stderr_path: pathlib.Path, ) -> ShellLauncherCommand: diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index b842a647c..adbbfab93 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -33,7 +33,7 @@ import typing as t from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import _EnvironMappingType, dispatch +from smartsim._core.dispatch import EnvironMappingType, dispatch from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand from smartsim._core.utils.launcher import ExecutableProtocol from smartsim.log import get_logger @@ -48,7 +48,7 @@ def _as_srun_command( args: ShellLaunchArguments, exe: ExecutableProtocol, path: pathlib.Path, - env: _EnvironMappingType, + env: EnvironMappingType, stdout_path: pathlib.Path, stderr_path: pathlib.Path, ) -> ShellLauncherCommand: diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index d627d6eb4..3628bc351 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -227,4 +227,6 @@ def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index a637db1e2..251659c6f 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -164,4 +164,6 @@ def test_formatting_returns_original_exe(mock_echo_executable, test_dir): assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 72047275b..f2513a2f7 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -299,4 +299,6 @@ def test_formatting_launch_args( assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index db9402428..857b3799a 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -153,4 +153,6 @@ def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): assert shell_launch_cmd.path == pathlib.Path(test_dir) assert shell_launch_cmd.env == {} assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 41ebe65cc..855068619 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -68,7 +68,7 @@ def dispatcher(): dispatches any jobs with `MockLaunchArgs` to a `NoOpRecordLauncher` """ d = dispatch.Dispatcher() - to_record: dispatch._FormatterType[MockLaunchArgs, LaunchRecord] = ( + to_record: dispatch.FormatterType[MockLaunchArgs, LaunchRecord] = ( lambda settings, exe, path, env, out, err: LaunchRecord( settings, exe, env, path, out, err ) diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index c1c447da7..4d4776103 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -40,16 +40,8 @@ from smartsim.error.errors import LauncherJobNotFound from smartsim.status import JobStatus -# TODO tests bad vars in Popen call at beginning -# tests -> helper.exe : pass in None, empty str, path with a space at beginning, a non valid command -# -> write a test for the invalid num of items - test_shell_launcher_fails_on_any_invalid_len_input -# -> have border tests for 0,1,4,6 cmd vals -> work correctly without them -> raise ValueError -# do all of the failures as well as the sucess criteria - pytestmark = pytest.mark.group_a -requires_slurm = pytest.mark.skipif(not shutil.which("srun"), reason="requires srun") - class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" @@ -99,9 +91,13 @@ def shell_launcher(): def shell_cmd(test_dir: str) -> ShellLauncherCommand: """Fixture to create an instance of Generator.""" run_dir, out_file, err_file = generate_directory(test_dir) - return ShellLauncherCommand( - {}, run_dir, out_file, err_file, EchoHelloWorldEntity().as_program_arguments() - ) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + yield ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) # UNIT TESTS @@ -111,18 +107,31 @@ def test_shell_launcher_command_init(shell_cmd: ShellLauncherCommand, test_dir: """Test that ShellLauncherCommand initializes correctly""" assert shell_cmd.env == {} assert shell_cmd.path == pathlib.Path(test_dir) / "tmp" - assert shell_cmd.stdout == shell_cmd.path / "tmp.out" - assert shell_cmd.stderr == shell_cmd.path / "tmp.err" + assert shell_cmd.stdout.name == os.path.join(test_dir, "tmp", "tmp.out") + assert shell_cmd.stderr.name == os.path.join(test_dir, "tmp", "tmp.err") assert shell_cmd.command_tuple == EchoHelloWorldEntity().as_program_arguments() -def test_shell_launcher_init(shell_launcher): +def test_shell_launcher_init(shell_launcher: ShellLauncher): """Test that ShellLauncher initializes correctly""" assert shell_launcher._launched == {} +def test_check_popen_inputs(shell_launcher: ShellLauncher, test_dir: str): + """Test that ShellLauncher.check_popen_inputs throws correctly""" + cmd = ShellLauncherCommand( + {}, + pathlib.Path(test_dir) / "directory_dne", + subprocess.DEVNULL, + subprocess.DEVNULL, + EchoHelloWorldEntity().as_program_arguments(), + ) + with pytest.raises(ValueError): + _ = shell_launcher.start(cmd) + + def test_shell_launcher_start_calls_popen( - shell_launcher, shell_cmd: ShellLauncherCommand + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand ): """Test that the process leading up to the shell launcher popen call was correct""" with unittest.mock.patch( @@ -133,7 +142,7 @@ def test_shell_launcher_start_calls_popen( def test_shell_launcher_start_calls_popen_with_value( - shell_launcher, shell_cmd: ShellLauncherCommand + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand ): """Test that popen was called with correct values""" with unittest.mock.patch( @@ -149,37 +158,27 @@ def test_shell_launcher_start_calls_popen_with_value( ) -def test_popen_returns_popen_object(shell_launcher, test_dir: str): +def test_popen_returns_popen_object(shell_launcher: ShellLauncher, test_dir: str): """Test that the popen call returns a popen object""" - run_dir, out_file, err_file = generate_directory(test_dir) - with ( - open(out_file, "w", encoding="utf-8") as out, - open(err_file, "w", encoding="utf-8") as err, - ): - cmd = ShellLauncherCommand( - {}, - run_dir, - subprocess.DEVNULL, - subprocess.DEVNULL, - EchoHelloWorldEntity().as_program_arguments(), - ) - id = shell_launcher.start(cmd) + run_dir, _, _ = generate_directory(test_dir) + cmd = ShellLauncherCommand( + {}, + run_dir, + subprocess.DEVNULL, + subprocess.DEVNULL, + EchoHelloWorldEntity().as_program_arguments(), + ) + id = shell_launcher.start(cmd) with shell_launcher._launched[id] as proc: assert isinstance(proc, sp.Popen) -def test_popen_writes_to_output_file(shell_launcher, test_dir: str): +def test_popen_writes_to_output_file( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): """Test that popen writes to .out file upon successful process call""" - run_dir, out_file, err_file = generate_directory(test_dir) - with ( - open(out_file, "w", encoding="utf-8") as out, - open(err_file, "w", encoding="utf-8") as err, - ): - cmd = ShellLauncherCommand( - {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() - ) - id = shell_launcher.start(cmd) - out.close(), err.close() + _, out_file, err_file = generate_directory(test_dir) + id = shell_launcher.start(shell_cmd) proc = shell_launcher._launched[id] assert proc.wait() == 0 assert proc.returncode == 0 @@ -189,15 +188,14 @@ def test_popen_writes_to_output_file(shell_launcher, test_dir: str): assert err.read() == "" -@requires_slurm -def test_popen_fails_with_invalid_cmd(shell_launcher, test_dir): +def test_popen_fails_with_invalid_cmd(shell_launcher: ShellLauncher, test_dir: str): """Test that popen returns a non zero returncode after failure""" run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err, ): - args = (helpers.expand_exe_path("srun"), "--flag_dne") + args = (helpers.expand_exe_path("ls"), "--flag_dne") cmd = ShellLauncherCommand({}, run_dir, out, err, args) id = shell_launcher.start(cmd) proc = shell_launcher._launched[id] @@ -210,7 +208,7 @@ def test_popen_fails_with_invalid_cmd(shell_launcher, test_dir): assert "unrecognized option" in content -def test_popen_issues_unique_ids(shell_launcher, test_dir): +def test_popen_issues_unique_ids(shell_launcher: ShellLauncher, test_dir: str): """Validate that all ids are unique within ShellLauncher._launched""" run_dir, out_file, err_file = generate_directory(test_dir) with ( @@ -220,20 +218,24 @@ def test_popen_issues_unique_ids(shell_launcher, test_dir): cmd = ShellLauncherCommand( {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() ) + seen = set() for _ in range(5): - _ = shell_launcher.start(cmd) + id = shell_launcher.start(cmd) + assert id not in seen, "Duplicate ID issued" + seen.add(id) assert len(shell_launcher._launched) == 5 assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) -def test_retrieve_status_dne(shell_launcher): +def test_retrieve_status_dne(shell_launcher: ShellLauncher): """Test tht ShellLauncher returns the status of completed Jobs""" - # Init ShellLauncher with pytest.raises(LauncherJobNotFound): _ = shell_launcher.get_status("dne") -def test_shell_launcher_returns_complete_status(shell_launcher, test_dir): +def test_shell_launcher_returns_complete_status( + shell_launcher: ShellLauncher, test_dir: str +): """Test tht ShellLauncher returns the status of completed Jobs""" run_dir, out_file, err_file = generate_directory(test_dir) with ( @@ -251,15 +253,16 @@ def test_shell_launcher_returns_complete_status(shell_launcher, test_dir): assert code == JobStatus.COMPLETED -@requires_slurm -def test_shell_launcher_returns_failed_status(shell_launcher, test_dir): +def test_shell_launcher_returns_failed_status( + shell_launcher: ShellLauncher, test_dir: str +): """Test tht ShellLauncher returns the status of completed Jobs""" run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err, ): - args = (helpers.expand_exe_path("srun"), "--flag_dne") + args = (helpers.expand_exe_path("ls"), "--flag_dne") cmd = ShellLauncherCommand({}, run_dir, out, err, args) for _ in range(5): id = shell_launcher.start(cmd) @@ -269,7 +272,9 @@ def test_shell_launcher_returns_failed_status(shell_launcher, test_dir): assert code == JobStatus.FAILED -def test_shell_launcher_returns_running_status(shell_launcher, test_dir): +def test_shell_launcher_returns_running_status( + shell_launcher: ShellLauncher, test_dir: str +): """Test tht ShellLauncher returns the status of completed Jobs""" run_dir, out_file, err_file = generate_directory(test_dir) with ( @@ -301,9 +306,14 @@ def test_shell_launcher_returns_running_status(shell_launcher, test_dir): pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="parked"), pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="idle"), pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="zombie"), + pytest.param( + "some-brand-new-unknown-status-str", JobStatus.UNKNOWN, id="unknown" + ), ], ) -def test_this(psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir): +def test_this( + psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir: str +): """Test tht ShellLauncher.get_status returns correct mapping""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) From 4a44e3333814f8cad91acee7af9c29b6569d5576 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 27 Aug 2024 17:40:01 -0500 Subject: [PATCH 80/82] updating shell tests --- smartsim/_core/shell/shellLauncher.py | 2 +- tests/test_shell_launcher.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/smartsim/_core/shell/shellLauncher.py b/smartsim/_core/shell/shellLauncher.py index 8fd1d5306..1197e2569 100644 --- a/smartsim/_core/shell/shellLauncher.py +++ b/smartsim/_core/shell/shellLauncher.py @@ -126,7 +126,7 @@ def __init__(self) -> None: def check_popen_inputs(self, shell_command: ShellLauncherCommand) -> None: if not shell_command.path.exists(): - raise ValueError("Please provide a valid shell command path.") + raise ValueError("Please provide a valid path to ShellLauncherCommand.") def start(self, shell_command: ShellLauncherCommand) -> LaunchedJobID: self.check_popen_inputs(shell_command) diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 4d4776103..a8e320fdd 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -26,7 +26,6 @@ import os import pathlib -import shutil import subprocess import unittest.mock From 262084e280aacf5d41b496179047c14980e708c6 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 27 Aug 2024 18:19:14 -0500 Subject: [PATCH 81/82] final comments, and fix duplicate num --- tests/test_generator.py | 15 +++++++-- tests/test_shell_launcher.py | 64 +++++++++++++----------------------- 2 files changed, 35 insertions(+), 44 deletions(-) diff --git a/tests/test_generator.py b/tests/test_generator.py index 695c84d2e..e44022779 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -20,9 +20,15 @@ pytestmark = pytest.mark.group_a +ids = set() + def random_id(): - return str(random.randint(1, 100)) + while True: + num = str(random.randint(1, 100)) + if num not in ids: + ids.add(num) + return num @pytest.fixture @@ -274,6 +280,7 @@ def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): log_path = os.path.join(jobs_dir, ensemble_dir, "log") assert osp.isdir(run_path) assert osp.isdir(log_path) + ids.clear() def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): @@ -294,6 +301,7 @@ def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_di for ensemble_dir in job_dir: copy_folder_path = os.path.join(jobs_dir, ensemble_dir, "run", "to_copy_dir") assert osp.isdir(copy_folder_path) + ids.clear() def test_generate_ensemble_symlink( @@ -321,6 +329,7 @@ def test_generate_ensemble_symlink( assert osp.isdir(sym_file_path) assert sym_file_path.is_symlink() assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) + ids.clear() def test_generate_ensemble_configure( @@ -343,7 +352,8 @@ def test_generate_ensemble_configure( launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) + id = exp.start(*job_list) + print(id) run_dir = listdir(test_dir) jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") @@ -364,3 +374,4 @@ def _check_generated(param_0, param_1, dir): _check_generated(1, 2, os.path.join(jobs_dir, "ensemble-name-2-2", "run")) _check_generated(1, 3, os.path.join(jobs_dir, "ensemble-name-3-3", "run")) _check_generated(0, 2, os.path.join(jobs_dir, "ensemble-name-0-0", "run")) + ids.clear() diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index a8e320fdd..6b03f8501 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -157,17 +157,11 @@ def test_shell_launcher_start_calls_popen_with_value( ) -def test_popen_returns_popen_object(shell_launcher: ShellLauncher, test_dir: str): +def test_popen_returns_popen_object( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): """Test that the popen call returns a popen object""" - run_dir, _, _ = generate_directory(test_dir) - cmd = ShellLauncherCommand( - {}, - run_dir, - subprocess.DEVNULL, - subprocess.DEVNULL, - EchoHelloWorldEntity().as_program_arguments(), - ) - id = shell_launcher.start(cmd) + id = shell_launcher.start(shell_cmd) with shell_launcher._launched[id] as proc: assert isinstance(proc, sp.Popen) @@ -207,23 +201,17 @@ def test_popen_fails_with_invalid_cmd(shell_launcher: ShellLauncher, test_dir: s assert "unrecognized option" in content -def test_popen_issues_unique_ids(shell_launcher: ShellLauncher, test_dir: str): +def test_popen_issues_unique_ids( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): """Validate that all ids are unique within ShellLauncher._launched""" - run_dir, out_file, err_file = generate_directory(test_dir) - with ( - open(out_file, "w", encoding="utf-8") as out, - open(err_file, "w", encoding="utf-8") as err, - ): - cmd = ShellLauncherCommand( - {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() - ) - seen = set() - for _ in range(5): - id = shell_launcher.start(cmd) - assert id not in seen, "Duplicate ID issued" - seen.add(id) - assert len(shell_launcher._launched) == 5 - assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) + seen = set() + for _ in range(5): + id = shell_launcher.start(shell_cmd) + assert id not in seen, "Duplicate ID issued" + seen.add(id) + assert len(shell_launcher._launched) == 5 + assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) def test_retrieve_status_dne(shell_launcher: ShellLauncher): @@ -233,23 +221,15 @@ def test_retrieve_status_dne(shell_launcher: ShellLauncher): def test_shell_launcher_returns_complete_status( - shell_launcher: ShellLauncher, test_dir: str + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str ): """Test tht ShellLauncher returns the status of completed Jobs""" - run_dir, out_file, err_file = generate_directory(test_dir) - with ( - open(out_file, "w", encoding="utf-8") as out, - open(err_file, "w", encoding="utf-8") as err, - ): - cmd = ShellLauncherCommand( - {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() - ) - for _ in range(5): - id = shell_launcher.start(cmd) - proc = shell_launcher._launched[id] - proc.wait() - code = shell_launcher.get_status(id)[id] - assert code == JobStatus.COMPLETED + for _ in range(5): + id = shell_launcher.start(shell_cmd) + proc = shell_launcher._launched[id] + proc.wait() + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.COMPLETED def test_shell_launcher_returns_failed_status( @@ -310,7 +290,7 @@ def test_shell_launcher_returns_running_status( ), ], ) -def test_this( +def test_get_status_maps_correctly( psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir: str ): """Test tht ShellLauncher.get_status returns correct mapping""" From aa00b09b9ceb1a298c577c98cecb4270a310f082 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 27 Aug 2024 18:36:57 -0500 Subject: [PATCH 82/82] remove the zip --- .github/workflows/run_tests.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index e71d9f65d..50a05fbad 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -157,11 +157,6 @@ jobs: run: | echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests - - - name: try zip the test result folder - if: failure() - run: | - zip -r testResult.zip tests/test_output # Upload artifacts on failure, ignoring binary files - name: Upload Artifact @@ -170,7 +165,7 @@ jobs: with: name: test_artifact path: | - testResult.zip + tests/test_output !**/*.so !**/*.pb !**/*.pt