diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index c57192ea8..618d30571 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -133,7 +133,7 @@ def copy(parsed_args: argparse.Namespace) -> None: dirs_exist_ok=parsed_args.dirs_exist_ok, ) else: - shutil.copyfile(parsed_args.source, parsed_args.dest) + shutil.copy(parsed_args.source, parsed_args.dest) def symlink(parsed_args: argparse.Namespace) -> None: diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index b1d241416..9c58cceaa 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -24,219 +24,223 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import base64 +import os import pathlib +import pickle import shutil +import subprocess +import sys import typing as t from datetime import datetime -from distutils import dir_util # pylint: disable=deprecated-module -from logging import DEBUG, INFO -from os import mkdir, path, symlink -from os.path import join, relpath +from os import mkdir, path +from os.path import join -from tabulate import tabulate - -from ...database import FeatureStore -from ...entity import Application, Ensemble, TaggedFilesHierarchy +from ...entity import Application, TaggedFilesHierarchy +from ...entity.files import EntityFiles +from ...launchable import Job from ...log import get_logger -from ..control import Manifest -from .modelwriter import ApplicationWriter logger = get_logger(__name__) logger.propagate = False class Generator: - """The primary job of the generator is to create the file structure - for a SmartSim experiment. The Generator is responsible for reading - and writing into configuration files as well. + """The primary job of the Generator is to create the directory and file structure + for a SmartSim Job. The Generator is also responsible for writing and configuring + files into the Job directory. """ - def __init__( - self, gen_path: str, overwrite: bool = False, verbose: bool = True - ) -> None: - """Initialize a generator object - - if overwrite is true, replace any existing - configured applications within an ensemble if there - is a name collision. Also replace any and all directories - for the experiment with fresh copies. Otherwise, if overwrite - is false, raises EntityExistsError when there is a name - collision between entities. - - :param gen_path: Path in which files need to be generated - :param overwrite: toggle entity replacement - :param verbose: Whether generation information should be logged to std out - """ - self._writer = ApplicationWriter() - self.gen_path = gen_path - self.overwrite = overwrite - self.log_level = DEBUG if not verbose else INFO + def __init__(self, root: pathlib.Path) -> None: + """Initialize a Generator object - @property - def log_file(self) -> str: - """Returns the location of the file - summarizing the parameters used for the last generation - of all generated entities. - - :returns: path to file with parameter settings + The class handles symlinking, copying, and configuration of files + associated with a Jobs entity. Additionally, it writes entity parameters + used for the specific run into the "smartsim_params.txt" settings file within + the Jobs log folder. """ - return join(self.gen_path, "smartsim_params.txt") + self.root = root + """The root path under which to generate files""" - def generate_experiment(self, *args: t.Any) -> None: - """Run ensemble and experiment file structure generation + def _generate_job_root(self, job: Job, job_index: int) -> pathlib.Path: + """Generates the root directory for a specific job instance. - Generate the file structure for a SmartSim experiment. This - includes the writing and configuring of input files for a - application. + :param job: The Job instance for which the root directory is generated. + :param job_index: The index of the Job instance (used for naming). + :returns: The path to the root directory for the Job instance. + """ + job_type = f"{job.__class__.__name__.lower()}s" + job_path = self.root / f"{job_type}/{job.name}-{job_index}" + return pathlib.Path(job_path) - To have files or directories present in the created entity - directories, such as datasets or input files, call - ``entity.attach_generator_files`` prior to generation. See - ``entity.attach_generator_files`` for more information on - what types of files can be included. + def _generate_run_path(self, job: Job, job_index: int) -> pathlib.Path: + """Generates the path for the "run" directory within the root directory + of a specific Job instance. - Tagged application files are read, checked for input variables to - configure, and written. Input variables to configure are - specified with a tag within the input file itself. - The default tag is surronding an input value with semicolons. - e.g. ``THERMO=;90;`` + :param job (Job): The Job instance for which the path is generated. + :param job_index (int): The index of the Job instance (used for naming). + :returns: The path to the "run" directory for the Job instance. + """ + path = self._generate_job_root(job, job_index) / "run" + path.mkdir(exist_ok=False, parents=True) + return pathlib.Path(path) + def _generate_log_path(self, job: Job, job_index: int) -> pathlib.Path: """ - generator_manifest = Manifest(*args) + Generates the path for the "log" directory within the root directory of a specific Job instance. - self._gen_exp_dir() - self._gen_feature_store_dir(generator_manifest.fss) - self._gen_entity_list_dir(generator_manifest.ensembles) - self._gen_entity_dirs(generator_manifest.applications) + :param job: The Job instance for which the path is generated. + :param job_index: The index of the Job instance (used for naming). + :returns: The path to the "log" directory for the Job instance. + """ + path = self._generate_job_root(job, job_index) / "log" + path.mkdir(exist_ok=False, parents=True) + return pathlib.Path(path) - def set_tag(self, tag: str, regex: t.Optional[str] = None) -> None: - """Set the tag used for tagging input files + @staticmethod + def _log_file(log_path: pathlib.Path) -> pathlib.Path: + """Returns the location of the file + summarizing the parameters used for the generation + of the entity. - Set a tag or a regular expression for the - generator to look for when configuring new applications. + :param log_path: Path to log directory + :returns: Path to file with parameter settings + """ + return pathlib.Path(log_path) / "smartsim_params.txt" - For example, a tag might be ``;`` where the - expression being replaced in the application configuration - file would look like ``;expression;`` + def generate_job(self, job: Job, job_index: int) -> pathlib.Path: + """Write and configure input files for a Job. - A full regular expression might tag specific - application configurations such that the configuration - files don't need to be tagged manually. + To have files or directories present in the created Job + directory, such as datasets or input files, call + ``entity.attach_generator_files`` prior to generation. - :param tag: A string of characters that signify - the string to be changed. Defaults to ``;`` - :param regex: full regex for the applicationwriter to search for - """ - self._writer.set_tag(tag, regex) + Tagged application files are read, checked for input variables to + configure, and written. Input variables to configure are + specified with a tag within the input file itself. + The default tag is surrounding an input value with semicolons. + e.g. ``THERMO=;90;`` - def _gen_exp_dir(self) -> None: - """Create the directory for an experiment if it does not - already exist. + :param job: The job instance to write and configure files for. + :param job_path: The path to the "run" directory for the job instance. + :param log_path: The path to the "log" directory for the job instance. """ - if path.isfile(self.gen_path): - raise FileExistsError( - f"Experiment directory could not be created. {self.gen_path} exists" - ) - if not path.isdir(self.gen_path): - # keep exists ok for race conditions on NFS - pathlib.Path(self.gen_path).mkdir(exist_ok=True, parents=True) - else: - logger.log( - level=self.log_level, msg="Working in previously created experiment" - ) + # Generate ../job_name/run directory + job_path = self._generate_run_path(job, job_index) + # Generate ../job_name/log directory + log_path = self._generate_log_path(job, job_index) - # The log_file only keeps track of the last generation - # this is to avoid gigantic files in case the user repeats - # generation several times. The information is anyhow - # redundant, as it is also written in each entity's dir - with open(self.log_file, mode="w", encoding="utf-8") as log_file: + # Create and write to the parameter settings file + with open(self._log_file(log_path), mode="w", encoding="utf-8") as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - def _gen_feature_store_dir(self, feature_store_list: t.List[FeatureStore]) -> None: - """Create the directory that will hold the error, output and - configuration files for the feature store. + # Perform file system operations on attached files + self._build_operations(job, job_path) - :param featurestore: FeatureStore instance - """ - # Loop through feature stores - for featurestore in feature_store_list: - feature_store_path = path.join(self.gen_path, featurestore.name) - - featurestore.set_path(feature_store_path) - # Always remove featurestore files if present. - if path.isdir(feature_store_path): - shutil.rmtree(feature_store_path, ignore_errors=True) - pathlib.Path(feature_store_path).mkdir( - exist_ok=self.overwrite, parents=True - ) + return job_path - def _gen_entity_list_dir(self, entity_lists: t.List[Ensemble]) -> None: - """Generate directories for Ensemble instances + @classmethod + def _build_operations(cls, job: Job, job_path: pathlib.Path) -> None: + """This method orchestrates file system ops for the attached SmartSim entity. + It processes three types of file system operations: to_copy, to_symlink, and to_configure. + For each type, it calls the corresponding private methods that open a subprocess + to complete each task. - :param entity_lists: list of Ensemble instances + :param job: The Job to perform file ops on attached entity files + :param job_path: Path to the Jobs run directory """ + app = t.cast(Application, job.entity) + cls._copy_files(app.files, job_path) + cls._symlink_files(app.files, job_path) + cls._write_tagged_files(app.files, app.params, job_path) - if not entity_lists: - return + @staticmethod + def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: + """Perform copy file sys operations on a list of files. - for elist in entity_lists: - elist_dir = path.join(self.gen_path, elist.name) - if path.isdir(elist_dir): - if self.overwrite: - shutil.rmtree(elist_dir) - mkdir(elist_dir) + :param app: The Application attached to the Job + :param dest: Path to the Jobs run directory + """ + # Return if no files are attached + if files is None: + return + for src in files.copy: + if os.path.isdir(src): + # Remove basename of source + base_source_name = os.path.basename(src) + # Attach source basename to destination + new_dst_path = os.path.join(dest, base_source_name) + # Copy source contents to new destination path + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "copy", + src, + new_dst_path, + "--dirs_exist_ok", + ] + ) else: - mkdir(elist_dir) - elist.path = elist_dir + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "copy", + src, + dest, + ] + ) - def _gen_entity_dirs( - self, - entities: t.List[Application], - entity_list: t.Optional[Ensemble] = None, - ) -> None: - """Generate directories for Entity instances + @staticmethod + def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: + """Perform symlink file sys operations on a list of files. - :param entities: list of Application instances - :param entity_list: Ensemble instance - :raises EntityExistsError: if a directory already exists for an - entity by that name + :param app: The Application attached to the Job + :param dest: Path to the Jobs run directory """ - if not entities: + # Return if no files are attached + if files is None: return + for src in files.link: + # Normalize the path to remove trailing slashes + normalized_path = os.path.normpath(src) + # Get the parent directory (last folder) + parent_dir = os.path.basename(normalized_path) + # Create destination + new_dest = os.path.join(str(dest), parent_dir) + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "symlink", + src, + new_dest, + ] + ) - for entity in entities: - if entity_list: - dst = path.join(self.gen_path, entity_list.name, entity.name) - else: - dst = path.join(self.gen_path, entity.name) - - if path.isdir(dst): - if self.overwrite: - shutil.rmtree(dst) - else: - error = ( - f"Directory for entity {entity.name} " - f"already exists in path {dst}" - ) - raise FileExistsError(error) - pathlib.Path(dst).mkdir(exist_ok=True) - entity.path = dst - - self._copy_entity_files(entity) - self._link_entity_files(entity) - self._write_tagged_entity_files(entity) - - def _write_tagged_entity_files(self, entity: Application) -> None: + @staticmethod + def _write_tagged_files( + files: t.Union[EntityFiles, None], + params: t.Mapping[str, str], + dest: pathlib.Path, + ) -> None: """Read, configure and write the tagged input files for - a Application instance within an ensemble. This function - specifically deals with the tagged files attached to - an Ensemble. + a Job instance. This function specifically deals with the tagged + files attached to an entity. - :param entity: a Application instance + :param app: The Application attached to the Job + :param dest: Path to the Jobs run directory """ - if entity.files: + # Return if no files are attached + if files is None: + return + if files.tagged: to_write = [] def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: @@ -247,92 +251,80 @@ def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: directory structure """ for file in tagged.files: - dst_path = path.join(entity.path, tagged.base, path.basename(file)) + dst_path = path.join(dest, tagged.base, path.basename(file)) shutil.copyfile(file, dst_path) to_write.append(dst_path) for tagged_dir in tagged.dirs: - mkdir( - path.join( - entity.path, tagged.base, path.basename(tagged_dir.base) - ) - ) + mkdir(path.join(dest, tagged.base, path.basename(tagged_dir.base))) _build_tagged_files(tagged_dir) - if entity.files.tagged_hierarchy: - _build_tagged_files(entity.files.tagged_hierarchy) - - # write in changes to configurations - if isinstance(entity, Application): - files_to_params = self._writer.configure_tagged_application_files( - to_write, entity.params + if files.tagged_hierarchy: + _build_tagged_files(files.tagged_hierarchy) + + # Pickle the dictionary + pickled_dict = pickle.dumps(params) + # Default tag delimiter + tag = ";" + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + for dest_path in to_write: + subprocess.run( + args=[ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "configure", + dest_path, + dest_path, + tag, + encoded_dict, + ] ) - self._log_params(entity, files_to_params) - - def _log_params( - self, entity: Application, files_to_params: t.Dict[str, t.Dict[str, str]] - ) -> None: - """Log which files were modified during generation - - and what values were set to the parameters - :param entity: the application being generated - :param files_to_params: a dict connecting each file to its parameter settings - """ - used_params: t.Dict[str, str] = {} - file_to_tables: t.Dict[str, str] = {} - for file, params in files_to_params.items(): - used_params.update(params) - table = tabulate(params.items(), headers=["Name", "Value"]) - file_to_tables[relpath(file, self.gen_path)] = table - - if used_params: - used_params_str = ", ".join( - [f"{name}={value}" for name, value in used_params.items()] - ) - logger.log( - level=self.log_level, - msg=f"Configured application {entity.name} with params {used_params_str}", - ) - file_table = tabulate( - file_to_tables.items(), - headers=["File name", "Parameters"], - ) - log_entry = f"Application name: {entity.name}\n{file_table}\n\n" - with open(self.log_file, mode="a", encoding="utf-8") as logfile: - logfile.write(log_entry) - with open( - join(entity.path, "smartsim_params.txt"), mode="w", encoding="utf-8" - ) as local_logfile: - local_logfile.write(log_entry) - - else: - logger.log( - level=self.log_level, - msg=f"Configured application {entity.name} with no parameters", - ) - - @staticmethod - def _copy_entity_files(entity: Application) -> None: - """Copy the entity files and directories attached to this entity. - - :param entity: Application - """ - if entity.files: - for to_copy in entity.files.copy: - dst_path = path.join(entity.path, path.basename(to_copy)) - if path.isdir(to_copy): - dir_util.copy_tree(to_copy, entity.path) - else: - shutil.copyfile(to_copy, dst_path) - - @staticmethod - def _link_entity_files(entity: Application) -> None: - """Symlink the entity files attached to this entity. - - :param entity: Application - """ - if entity.files: - for to_link in entity.files.link: - dst_path = path.join(entity.path, path.basename(to_link)) - symlink(to_link, dst_path) + # TODO address in ticket 723 + # self._log_params(entity, files_to_params) + + # TODO to be refactored in ticket 723 + # def _log_params( + # self, entity: Application, files_to_params: t.Dict[str, t.Dict[str, str]] + # ) -> None: + # """Log which files were modified during generation + + # and what values were set to the parameters + + # :param entity: the application being generated + # :param files_to_params: a dict connecting each file to its parameter settings + # """ + # used_params: t.Dict[str, str] = {} + # file_to_tables: t.Dict[str, str] = {} + # for file, params in files_to_params.items(): + # used_params.update(params) + # table = tabulate(params.items(), headers=["Name", "Value"]) + # file_to_tables[relpath(file, self.gen_path)] = table + + # if used_params: + # used_params_str = ", ".join( + # [f"{name}={value}" for name, value in used_params.items()] + # ) + # logger.log( + # level=self.log_level, + # msg=f"Configured application {entity.name} with params {used_params_str}", + # ) + # file_table = tabulate( + # file_to_tables.items(), + # headers=["File name", "Parameters"], + # ) + # log_entry = f"Application name: {entity.name}\n{file_table}\n\n" + # with open(self.log_file, mode="a", encoding="utf-8") as logfile: + # logfile.write(log_entry) + # with open( + # join(entity.path, "smartsim_params.txt"), mode="w", encoding="utf-8" + # ) as local_logfile: + # local_logfile.write(log_entry) + + # else: + # logger.log( + # level=self.log_level, + # msg=f"Configured application {entity.name} with no parameters", + # ) diff --git a/smartsim/_core/generation/modelwriter.py b/smartsim/_core/generation/modelwriter.py deleted file mode 100644 index a22bc029a..000000000 --- a/smartsim/_core/generation/modelwriter.py +++ /dev/null @@ -1,158 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2024, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import collections -import re -import typing as t - -from smartsim.error.errors import SmartSimError - -from ...error import ParameterWriterError -from ...log import get_logger - -logger = get_logger(__name__) - - -class ApplicationWriter: - def __init__(self) -> None: - self.tag = ";" - self.regex = "(;[^;]+;)" - self.lines: t.List[str] = [] - - def set_tag(self, tag: str, regex: t.Optional[str] = None) -> None: - """Set the tag for the applicationwriter to search for within - tagged files attached to an entity. - - :param tag: tag for the applicationwriter to search for, - defaults to semi-colon e.g. ";" - :param regex: full regex for the applicationwriter to search for, - defaults to "(;.+;)" - """ - if regex: - self.regex = regex - else: - self.tag = tag - self.regex = "".join(("(", tag, ".+", tag, ")")) - - def configure_tagged_application_files( - self, - tagged_files: t.List[str], - params: t.Dict[str, str], - make_missing_tags_fatal: bool = False, - ) -> t.Dict[str, t.Dict[str, str]]: - """Read, write and configure tagged files attached to a Application - instance. - - :param tagged_files: list of paths to tagged files - :param params: application parameters - :param make_missing_tags_fatal: raise an error if a tag is missing - :returns: A dict connecting each file to its parameter settings - """ - files_to_tags: t.Dict[str, t.Dict[str, str]] = {} - for tagged_file in tagged_files: - self._set_lines(tagged_file) - used_tags = self._replace_tags(params, make_missing_tags_fatal) - self._write_changes(tagged_file) - files_to_tags[tagged_file] = used_tags - - return files_to_tags - - def _set_lines(self, file_path: str) -> None: - """Set the lines for the applicationwriter to iterate over - - :param file_path: path to the newly created and tagged file - :raises ParameterWriterError: if the newly created file cannot be read - """ - try: - with open(file_path, "r+", encoding="utf-8") as file_stream: - self.lines = file_stream.readlines() - except (IOError, OSError) as e: - raise ParameterWriterError(file_path) from e - - def _write_changes(self, file_path: str) -> None: - """Write the ensemble-specific changes - - :raises ParameterWriterError: if the newly created file cannot be read - """ - try: - with open(file_path, "w+", encoding="utf-8") as file_stream: - for line in self.lines: - file_stream.write(line) - except (IOError, OSError) as e: - raise ParameterWriterError(file_path, read=False) from e - - def _replace_tags( - self, params: t.Dict[str, str], make_fatal: bool = False - ) -> t.Dict[str, str]: - """Replace the tagged parameters within the file attached to this - application. The tag defaults to ";" - - :param application: The application instance - :param make_fatal: (Optional) Set to True to force a fatal error - if a tag is not matched - :returns: A dict of parameter names and values set for the file - """ - edited = [] - unused_tags: t.DefaultDict[str, t.List[int]] = collections.defaultdict(list) - used_params: t.Dict[str, str] = {} - for i, line in enumerate(self.lines, 1): - while search := re.search(self.regex, line): - tagged_line = search.group(0) - previous_value = self._get_prev_value(tagged_line) - if self._is_ensemble_spec(tagged_line, params): - new_val = str(params[previous_value]) - line = re.sub(self.regex, new_val, line, 1) - used_params[previous_value] = new_val - - # if a tag is found but is not in this application's configurations - # put in placeholder value - else: - tag = tagged_line.split(self.tag)[1] - unused_tags[tag].append(i) - line = re.sub(self.regex, previous_value, line) - break - edited.append(line) - - for tag, value in unused_tags.items(): - missing_tag_message = f"Unused tag {tag} on line(s): {str(value)}" - if make_fatal: - raise SmartSimError(missing_tag_message) - logger.warning(missing_tag_message) - self.lines = edited - return used_params - - def _is_ensemble_spec( - self, tagged_line: str, application_params: t.Dict[str, str] - ) -> bool: - split_tag = tagged_line.split(self.tag) - prev_val = split_tag[1] - if prev_val in application_params.keys(): - return True - return False - - def _get_prev_value(self, tagged_line: str) -> str: - split_tag = tagged_line.split(self.tag) - return split_tag[1] diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index c4c7d8365..992707959 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -366,6 +366,7 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: def _as_run_request_args_and_policy( run_req_args: DragonLaunchArguments, exe: ExecutableProtocol, + path: str | os.PathLike[str], env: t.Mapping[str, str | None], ) -> tuple[DragonRunRequestView, DragonRunPolicy]: # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -384,8 +385,7 @@ def _as_run_request_args_and_policy( # this will need to be injected by the user or by us to have # the command execute next to any generated files. A similar # problem exists for the other settings. - # TODO: Find a way to inject this path - path=os.getcwd(), + path=path, env=env, # TODO: Not sure how this info is injected name=None, diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index 1321c5b7e..62d176259 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -52,6 +52,18 @@ _TSignalHandlerFn = t.Callable[[int, t.Optional["FrameType"]], object] +def check_name(name: str) -> None: + """ + Checks if the input name is valid. + + :param name: The name to be checked. + + :raises ValueError: If the name contains the path separator (os.path.sep). + """ + if os.path.sep in name: + raise ValueError("Invalid input: String contains the path separator.") + + def unpack_fs_identifier(fs_id: str, token: str) -> t.Tuple[str, str]: """Unpack the unformatted feature store identifier and format for env variable suffix using the token diff --git a/smartsim/entity/dbnode.py b/smartsim/entity/dbnode.py index 16fd9863f..54ec68e1a 100644 --- a/smartsim/entity/dbnode.py +++ b/smartsim/entity/dbnode.py @@ -64,7 +64,7 @@ def __init__( fs_identifier: str = "", ) -> None: """Initialize a feature store node within an feature store.""" - super().__init__(name, path, run_settings) + super().__init__(name, run_settings) self.exe = [exe] if run_settings.container else [expand_exe_path(exe)] self.exe_args = exe_args or [] self.ports = ports diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index 517d33161..07ebe25de 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -53,7 +53,6 @@ def __init__( exe: str | os.PathLike[str], exe_args: t.Sequence[str] | None = None, exe_arg_parameters: t.Mapping[str, t.Sequence[t.Sequence[str]]] | None = None, - path: str | os.PathLike[str] | None = None, files: EntityFiles | None = None, file_parameters: t.Mapping[str, t.Sequence[str]] | None = None, permutation_strategy: str | strategies.PermutationStrategyType = "all_perm", @@ -66,11 +65,6 @@ def __init__( self.exe_arg_parameters = ( copy.deepcopy(exe_arg_parameters) if exe_arg_parameters else {} ) - self.path = os.fspath(path) if path is not None else os.getcwd() - # ^^^^^^^^^^^ - # TODO: Copied from the original implementation, but I'm not sure that - # I like this default. Shouldn't it be something under an - # experiment directory? If so, how it injected?? self.files = copy.deepcopy(files) if files else EntityFiles() self.file_parameters = dict(file_parameters) if file_parameters else {} self.permutation_strategy = permutation_strategy @@ -97,7 +91,6 @@ def _create_applications(self) -> tuple[Application, ...]: # ^^^^^^^^^^^^^^^^^^^^^^^ # FIXME: remove this constructor arg! It should not exist!! exe_args=self.exe_args, - path=os.path.join(self.path, self.name), files=self.files, params=permutation.params, params_as_args=permutation.exe_args, # type: ignore[arg-type] @@ -111,4 +104,4 @@ def as_jobs(self, settings: LaunchSettings) -> tuple[Job, ...]: apps = self._create_applications() if not apps: raise ValueError("There are no members as part of this ensemble") - return tuple(Job(app, settings) for app in apps) + return tuple(Job(app, settings, app.name) for app in apps) diff --git a/smartsim/entity/entity.py b/smartsim/entity/entity.py index 6416a8b2b..8c4bd4e4f 100644 --- a/smartsim/entity/entity.py +++ b/smartsim/entity/entity.py @@ -98,7 +98,7 @@ def _on_disable(self) -> None: class SmartSimEntity: - def __init__(self, name: str, path: str, run_settings: "RunSettings") -> None: + def __init__(self, name: str, run_settings: "RunSettings") -> None: """Initialize a SmartSim entity. Each entity must have a name, path, and @@ -106,11 +106,9 @@ def __init__(self, name: str, path: str, run_settings: "RunSettings") -> None: share these attributes. :param name: Name of the entity - :param path: path to output, error, and configuration files """ self.name = name self.run_settings = run_settings - self.path = path @property def type(self) -> str: diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index 4304ee95b..a1186cedd 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -64,7 +64,6 @@ def __init__( run_settings: "RunSettings", params: t.Optional[t.Dict[str, str]] = None, exe_args: t.Optional[t.List[str]] = None, - path: t.Optional[str] = getcwd(), params_as_args: t.Optional[t.List[str]] = None, batch_settings: t.Optional["BatchSettings"] = None, files: t.Optional[EntityFiles] = None, @@ -76,7 +75,6 @@ def __init__( :param exe_args: executable arguments :param params: application parameters for writing into configuration files or to be passed as command line arguments to executable. - :param path: path to output, error, and configuration files :param run_settings: launcher settings specified in the experiment :param params_as_args: list of parameters which have to be interpreted as command line arguments to @@ -85,7 +83,7 @@ def __init__( application as a batch job :param files: Files to have available to the application """ - super().__init__(name, str(path), run_settings) + super().__init__(name, run_settings) self.exe = [expand_exe_path(exe)] # self.exe = [exe] if run_settings.container else [expand_exe_path(exe)] self.exe_args = exe_args or [] @@ -228,7 +226,6 @@ def attach_generator_files( "`smartsim_params.txt` is a file automatically " + "generated by SmartSim and cannot be ovewritten." ) - self.files = EntityFiles(to_configure, to_copy, to_symlink) @property diff --git a/smartsim/experiment.py b/smartsim/experiment.py index ed62d9479..03e3012ee 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -29,9 +29,11 @@ from __future__ import annotations import collections +import datetime import itertools import os import os.path as osp +import pathlib import textwrap import typing as t from os import environ, getcwd @@ -181,13 +183,23 @@ def start(self, *jobs: Job) -> tuple[LaunchedJobID, ...]: jobs that can be used to query or alter the status of that particular execution of the job. """ - return self._dispatch(dispatch.DEFAULT_DISPATCHER, *jobs) + # Create the run id + run_id = datetime.datetime.now().replace(microsecond=0).isoformat() + # Generate the root path + root = pathlib.Path(self.exp_path, run_id) + return self._dispatch(Generator(root), dispatch.DEFAULT_DISPATCHER, *jobs) def _dispatch( - self, dispatcher: dispatch.Dispatcher, job: Job, *jobs: Job + self, + generator: Generator, + dispatcher: dispatch.Dispatcher, + job: Job, + *jobs: Job, ) -> tuple[LaunchedJobID, ...]: """Dispatch a series of jobs with a particular dispatcher + :param generator: The generator is responsible for creating the + job run and log directory. :param dispatcher: The dispatcher that should be used to determine how to start a job based on its launch settings. :param job: The first job instance to dispatch @@ -197,7 +209,7 @@ def _dispatch( particular dispatch of the job. """ - def execute_dispatch(job: Job) -> LaunchedJobID: + def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: args = job.launch_settings.launch_args env = job.launch_settings.env_vars # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @@ -220,7 +232,8 @@ def execute_dispatch(job: Job) -> LaunchedJobID: launch_config = dispatch.create_new_launcher_configuration( for_experiment=self, with_arguments=args ) - id_ = launch_config.start(exe, env) + job_execution_path = self._generate(generator, job, idx) + id_ = launch_config.start(exe, job_execution_path, env) # Save the underlying launcher instance and launched job id. That # way we do not need to spin up a launcher instance for each # individual job, and the experiment can monitor job statuses. @@ -228,7 +241,9 @@ def execute_dispatch(job: Job) -> LaunchedJobID: self._launch_history.save_launch(launch_config._adapted_launcher, id_) return id_ - return execute_dispatch(job), *map(execute_dispatch, jobs) + return execute_dispatch(generator, job, 0), *( + execute_dispatch(generator, job, idx) for idx, job in enumerate(jobs, 1) + ) def get_status( self, *ids: LaunchedJobID @@ -262,35 +277,24 @@ def get_status( return tuple(stats) @_contextualize - def generate( - self, - *args: t.Union[SmartSimEntity, EntitySequence[SmartSimEntity]], - tag: t.Optional[str] = None, - overwrite: bool = False, - verbose: bool = False, - ) -> None: - """Generate the file structure for an ``Experiment`` - - ``Experiment.generate`` creates directories for each entity - passed to organize Experiments that launch many entities. - - If files or directories are attached to ``application`` objects - using ``application.attach_generator_files()``, those files or - directories will be symlinked, copied, or configured and - written into the created directory for that instance. - - Instances of ``application``, ``Ensemble`` and ``FeatureStore`` - can all be passed as arguments to the generate method. - - :param tag: tag used in `to_configure` generator files - :param overwrite: overwrite existing folders and contents - :param verbose: log parameter settings to std out + def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.Path: + """Generate the directory structure and files for a ``Job`` + + If files or directories are attached to an ``Application`` object + associated with the Job using ``Application.attach_generator_files()``, + those files or directories will be symlinked, copied, or configured and + written into the created job directory. + + :param generator: The generator is responsible for creating the job + run and log directory. + :param job: The Job instance for which the output is generated. + :param job_index: The index of the Job instance (used for naming). + :returns: The path to the generated output for the Job instance. + :raises: A SmartSimError if an error occurs during the generation process. """ try: - generator = Generator(self.exp_path, overwrite=overwrite, verbose=verbose) - if tag: - generator.set_tag(tag) - generator.generate_experiment(*args) + job_run_path = generator.generate_job(job, job_index) + return job_run_path except SmartSimError as e: logger.error(e) raise @@ -372,22 +376,6 @@ def telemetry(self) -> TelemetryConfiguration: """ return self._telemetry_cfg - def _create_entity_dir(self, start_manifest: Manifest) -> None: - def create_entity_dir( - entity: t.Union[FeatureStore, Application, Ensemble] - ) -> None: - if not osp.isdir(entity.path): - os.makedirs(entity.path) - - for application in start_manifest.applications: - create_entity_dir(application) - - for feature_store in start_manifest.fss: - create_entity_dir(feature_store) - - for ensemble in start_manifest.ensembles: - create_entity_dir(ensemble) - def __str__(self) -> str: return self.name diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index f440ead0b..a433319ac 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -26,17 +26,23 @@ from __future__ import annotations +import os import typing as t from copy import deepcopy from smartsim._core.commands.launchCommands import LaunchCommands +from smartsim._core.utils.helpers import check_name from smartsim.launchable.basejob import BaseJob +from smartsim.log import get_logger from smartsim.settings import LaunchSettings +logger = get_logger(__name__) + if t.TYPE_CHECKING: from smartsim.entity.entity import SmartSimEntity +@t.final class Job(BaseJob): """A Job holds a reference to a SmartSimEntity and associated LaunchSettings prior to launch. It is responsible for turning @@ -50,26 +56,44 @@ def __init__( self, entity: SmartSimEntity, launch_settings: LaunchSettings, + name: str | None = None, ): super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) - # TODO: self.warehouse_runner = JobWarehouseRunner + self._name = name if name else entity.name + check_name(self._name) + + @property + def name(self) -> str: + """Retrieves the name of the Job.""" + return self._name + + @name.setter + def name(self, name: str) -> None: + """Sets the name of the Job.""" + check_name(name) + logger.debug(f'Overwriting the Job name from "{self._name}" to "{name}"') + self._name = name @property def entity(self) -> SmartSimEntity: + """Retrieves the Job entity.""" return deepcopy(self._entity) @entity.setter def entity(self, value: SmartSimEntity) -> None: + """Sets the Job entity.""" self._entity = deepcopy(value) @property def launch_settings(self) -> LaunchSettings: + """Retrieves the Job LaunchSettings.""" return deepcopy(self._launch_settings) @launch_settings.setter def launch_settings(self, value: LaunchSettings) -> None: + """Sets the Job LaunchSettings.""" self._launch_settings = deepcopy(value) def get_launch_steps(self) -> LaunchCommands: diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index de7ed691b..65914cde4 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -3,13 +3,19 @@ import typing as t from copy import deepcopy +from smartsim.log import get_logger + +from .._core.utils.helpers import check_name from .basejob import BaseJob from .baseJobGroup import BaseJobGroup +logger = get_logger(__name__) + if t.TYPE_CHECKING: from typing_extensions import Self +@t.final class JobGroup(BaseJobGroup): """A job group holds references to multiple jobs that will be executed all at the same time when resources @@ -19,9 +25,24 @@ class JobGroup(BaseJobGroup): def __init__( self, jobs: t.List[BaseJob], + name: str = "job_group", ) -> None: super().__init__() self._jobs = deepcopy(jobs) + self._name = name + check_name(self._name) + + @property + def name(self) -> str: + """Retrieves the name of the JobGroup.""" + return self._name + + @name.setter + def name(self, name: str) -> None: + """Sets the name of the JobGroup.""" + check_name(name) + logger.debug(f'Overwriting Job name from "{self._name}" to "{name}"') + self._name = name @property def jobs(self) -> t.List[BaseJob]: diff --git a/smartsim/settings/arguments/launch/__init__.py b/smartsim/settings/arguments/launch/__init__.py index 30502394b..629d45f67 100644 --- a/smartsim/settings/arguments/launch/__init__.py +++ b/smartsim/settings/arguments/launch/__init__.py @@ -11,9 +11,9 @@ "DragonLaunchArguments", "LocalLaunchArguments", "JsrunLaunchArguments", - "MpiLaunchArguments", + "MpirunLaunchArguments", "MpiexecLaunchArguments", - "OrteLaunchArguments", + "OrterunLaunchArguments", "PalsMpiexecLaunchArguments", "SlurmLaunchArguments", ] diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index cc2dadd73..95e80b121 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -29,6 +29,7 @@ import abc import collections.abc import dataclasses +import os import subprocess as sp import typing as t import uuid @@ -48,6 +49,9 @@ _Ts = TypeVarTuple("_Ts") _T_contra = t.TypeVar("_T_contra", contravariant=True) +_WorkingDirectory: TypeAlias = t.Union[str, os.PathLike[str]] +"""A working directory represented as a string or PathLike object""" + _DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") """Any type of luanch arguments, typically used when the type bound by the type argument is a key a `Dispatcher` dispatch registry @@ -62,13 +66,14 @@ a job """ _FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _EnvironMappingType], _LaunchableT + [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType], + _LaunchableT, ] """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ _LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _EnvironMappingType]" + "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType]" ) """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings @@ -256,8 +261,12 @@ def create_adapter_from_launcher( f"exactly `{self.launcher_type}`" ) - def format_(exe: ExecutableProtocol, env: _EnvironMappingType) -> _LaunchableT: - return self.formatter(arguments, exe, env) + def format_( + exe: ExecutableProtocol, + path: str | os.PathLike[str], + env: _EnvironMappingType, + ) -> _LaunchableT: + return self.formatter(arguments, exe, path, env) return _LauncherAdapter(launcher, format_) @@ -425,7 +434,7 @@ def get_status( def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[LaunchArguments, t.Sequence[str]]: +) -> _FormatterType[LaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]]]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -456,9 +465,12 @@ def make_shell_format_fn( """ def impl( - args: LaunchArguments, exe: ExecutableProtocol, _env: _EnvironMappingType - ) -> t.Sequence[str]: - return ( + args: LaunchArguments, + exe: ExecutableProtocol, + path: str | os.PathLike[str], + _env: _EnvironMappingType, + ) -> t.Tuple[str | os.PathLike[str], t.Sequence[str]]: + return path, ( ( run_command, *(args.format_launch_args() or ()), @@ -478,11 +490,14 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - def start(self, command: t.Sequence[str]) -> LaunchedJobID: + def start( + self, command: tuple[str | os.PathLike[str], t.Sequence[str]] + ) -> LaunchedJobID: id_ = create_job_id() - exe, *rest = command + path, args = command + exe, *rest = args # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest)) + self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path) return id_ def get_status( diff --git a/tests/temp_tests/test_jobGroup.py b/tests/temp_tests/test_jobGroup.py index b129adb8d..20c25d36a 100644 --- a/tests/temp_tests/test_jobGroup.py +++ b/tests/temp_tests/test_jobGroup.py @@ -44,25 +44,40 @@ def get_launch_steps(self): raise NotImplementedError +def test_invalid_job_name(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2, wlmutils.get_test_launcher()) + with pytest.raises(ValueError): + _ = JobGroup([job_1, job_2], name="name/not/allowed") + + def test_create_JobGroup(): job_1 = MockJob() job_group = JobGroup([job_1]) assert len(job_group) == 1 -def test_getitem_JobGroup(): - job_1 = Job(app_1, LaunchSettings("slurm")) - job_2 = Job(app_2, LaunchSettings("slurm")) +def test_name_setter(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2, wlmutils.get_test_launcher()) + job_group = JobGroup([job_1, job_2]) + job_group.name = "new_name" + assert job_group.name == "new_name" + + +def test_getitem_JobGroup(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2, wlmutils.get_test_launcher()) job_group = JobGroup([job_1, job_2]) get_value = job_group[0].entity.name assert get_value == job_1.entity.name -def test_setitem_JobGroup(): - job_1 = Job(app_1, LaunchSettings("slurm")) - job_2 = Job(app_2, LaunchSettings("slurm")) +def test_setitem_JobGroup(wlmutils): + job_1 = Job(app_1, wlmutils.get_test_launcher()) + job_2 = Job(app_2, wlmutils.get_test_launcher()) job_group = JobGroup([job_1, job_2]) - job_3 = Job(app_3, LaunchSettings("slurm")) + job_3 = Job(app_3, wlmutils.get_test_launcher()) job_group[1] = job_3 assert len(job_group) == 2 get_value = job_group[1] diff --git a/tests/temp_tests/test_launchable.py b/tests/temp_tests/test_launchable.py index 02a2e073b..16fba6cff 100644 --- a/tests/temp_tests/test_launchable.py +++ b/tests/temp_tests/test_launchable.py @@ -50,6 +50,18 @@ def test_launchable_init(): assert isinstance(launchable, Launchable) +def test_invalid_job_name(wlmutils): + entity = Application( + "test_name", + run_settings="RunSettings", + exe="echo", + exe_args=["spam", "eggs"], + ) # Mock RunSettings + settings = LaunchSettings(wlmutils.get_test_launcher()) + with pytest.raises(ValueError): + _ = Job(entity, settings, name="path/to/name") + + def test_job_init(): entity = Application( "test_name", @@ -65,6 +77,18 @@ def test_job_init(): assert "eggs" in job.entity.exe_args +def test_name_setter(): + entity = Application( + "test_name", + run_settings=LaunchSettings("slurm"), + exe="echo", + exe_args=["spam", "eggs"], + ) + job = Job(entity, LaunchSettings("slurm")) + job.name = "new_name" + assert job.name == "new_name" + + def test_job_init_deepcopy(): entity = Application( "test_name", @@ -80,7 +104,7 @@ def test_job_init_deepcopy(): def test_add_mpmd_pair(): - entity = SmartSimEntity("test_name", "python", LaunchSettings("slurm")) + entity = SmartSimEntity("test_name", LaunchSettings("slurm")) mpmd_job = MPMDJob() mpmd_job.add_mpmd_pair(entity, LaunchSettings("slurm")) @@ -155,10 +179,10 @@ def test_add_mpmd_pair_check_launcher_error(): """Test that an error is raised when a pairs is added to an mpmd job using add_mpmd_pair that does not have the same launcher type""" mpmd_pairs = [] - entity1 = SmartSimEntity("entity1", "python", LaunchSettings("slurm")) + entity1 = SmartSimEntity("entity1", LaunchSettings("slurm")) launch_settings1 = LaunchSettings("slurm") - entity2 = SmartSimEntity("entity2", "python", LaunchSettings("pals")) + entity2 = SmartSimEntity("entity2", LaunchSettings("pals")) launch_settings2 = LaunchSettings("pals") pair1 = MPMDPair(entity1, launch_settings1) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index c76b49363..370b67db7 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -210,6 +210,9 @@ def test_invalid_exclude_hostlist_format(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_aprun_command(AprunLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + path, cmd = _as_aprun_command( + AprunLaunchArguments(args), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index e3f159b7f..38ee11486 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -66,7 +66,7 @@ def test_dragon_class_methods(function, value, flag, result): @pytest.mark.parametrize("cpu_affinity", (NOT_SET, [1], [1, 2, 3])) @pytest.mark.parametrize("gpu_affinity", (NOT_SET, [1], [1, 2, 3])) def test_formatting_launch_args_into_request( - mock_echo_executable, nodes, tasks_per_node, cpu_affinity, gpu_affinity + mock_echo_executable, nodes, tasks_per_node, cpu_affinity, gpu_affinity, test_dir ): launch_args = DragonLaunchArguments({}) if nodes is not NOT_SET: @@ -77,7 +77,9 @@ def test_formatting_launch_args_into_request( launch_args.set_cpu_affinity(cpu_affinity) if gpu_affinity is not NOT_SET: launch_args.set_gpu_affinity(gpu_affinity) - req, policy = _as_run_request_args_and_policy(launch_args, mock_echo_executable, {}) + req, policy = _as_run_request_args_and_policy( + launch_args, mock_echo_executable, test_dir, {} + ) expected_args = { k: v @@ -88,7 +90,7 @@ def test_formatting_launch_args_into_request( if v is not NOT_SET } expected_run_req = DragonRunRequestView( - exe="echo", exe_args=["hello", "world"], path="/tmp", env={}, **expected_args + exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, **expected_args ) assert req.exe == expected_run_req.exe assert req.exe_args == expected_run_req.exe_args @@ -96,6 +98,7 @@ def test_formatting_launch_args_into_request( assert req.tasks_per_node == expected_run_req.tasks_per_node assert req.hostlist == expected_run_req.hostlist assert req.pmi_enabled == expected_run_req.pmi_enabled + assert req.path == expected_run_req.path expected_run_policy_args = { k: v diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 3d18ea462..48de0e7b5 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -142,6 +142,9 @@ def test_format_env_vars(): assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] -def test_formatting_returns_original_exe(mock_echo_executable): - cmd = _as_local_command(LocalLaunchArguments({}), mock_echo_executable, {}) +def test_formatting_returns_original_exe(mock_echo_executable, test_dir): + path, cmd = _as_local_command( + LocalLaunchArguments({}), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == ("echo", "hello", "world") + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 2e2dddf78..eec915860 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -119,6 +119,9 @@ def test_launch_args(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_jsrun_command(JsrunLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + path, cmd = _as_jsrun_command( + JsrunLaunchArguments(args), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 362d21f06..ff5200eca 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -283,6 +283,9 @@ def test_invalid_hostlist_format(launcher): ), ), ) -def test_formatting_launch_args(mock_echo_executable, cls, fmt, cmd, args, expected): - fmt_cmd = fmt(cls(args), mock_echo_executable, {}) +def test_formatting_launch_args( + mock_echo_executable, cls, fmt, cmd, args, expected, test_dir +): + path, fmt_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}) assert tuple(fmt_cmd) == (cmd,) + expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index db66fa829..64b9dc7f1 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -131,6 +131,9 @@ def test_invalid_hostlist_format(): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_pals_command(PalsMpiexecLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + path, cmd = _as_pals_command( + PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 538f2aca4..1c21e3d01 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -316,6 +316,9 @@ def test_set_het_groups(monkeypatch): ), ), ) -def test_formatting_launch_args(mock_echo_executable, args, expected): - cmd = _as_srun_command(SlurmLaunchArguments(args), mock_echo_executable, {}) +def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): + path, cmd = _as_srun_command( + SlurmLaunchArguments(args), mock_echo_executable, test_dir, {} + ) assert tuple(cmd) == expected + assert path == test_dir diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index 3f170dfcb..4eb578a71 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -58,7 +58,6 @@ def test_ensemble_user_created_strategy(mock_launcher_settings, test_dir): "test_ensemble", "echo", ("hello", "world"), - path=test_dir, permutation_strategy=user_created_function, ).as_jobs(mock_launcher_settings) assert len(jobs) == 1 @@ -72,7 +71,6 @@ def test_ensemble_without_any_members_raises_when_cast_to_jobs( "test_ensemble", "echo", ("hello", "world"), - path=test_dir, file_parameters=_2x2_PARAMS, permutation_strategy="random", max_permutations=30, @@ -86,7 +84,6 @@ def test_strategy_error_raised_if_a_strategy_that_dne_is_requested(test_dir): "test_ensemble", "echo", ("hello",), - path=test_dir, permutation_strategy="THIS-STRATEGY-DNE", )._create_applications() @@ -105,7 +102,6 @@ def test_replicated_applications_have_eq_deep_copies_of_parameters(params, test_ "test_ensemble", "echo", ("hello",), - path=test_dir, replicas=4, file_parameters=params, )._create_applications() @@ -151,7 +147,6 @@ def test_all_perm_strategy( "test_ensemble", "echo", ("hello", "world"), - path=test_dir, file_parameters=params, exe_arg_parameters=exe_arg_params, permutation_strategy="all_perm", @@ -206,7 +201,6 @@ def test_step_strategy( "test_ensemble", "echo", ("hello", "world"), - path=test_dir, file_parameters=params, exe_arg_parameters=exe_arg_params, permutation_strategy="step", diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 3a971b8d4..fd71f9e99 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -32,7 +32,6 @@ import tempfile import typing as t import uuid -import weakref import pytest @@ -54,6 +53,7 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) + monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: "/tmp/job") yield exp @@ -64,7 +64,7 @@ def dispatcher(): """ d = dispatch.Dispatcher() to_record: dispatch._FormatterType[MockLaunchArgs, LaunchRecord] = ( - lambda settings, exe, env: LaunchRecord(settings, exe, env) + lambda settings, exe, path, env: LaunchRecord(settings, exe, env, path) ) d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) yield d @@ -140,6 +140,7 @@ class LaunchRecord: launch_args: launchArguments.LaunchArguments entity: entity.SmartSimEntity env: t.Mapping[str, str | None] + path: str @classmethod def from_job(cls, job: job.Job): @@ -154,7 +155,8 @@ def from_job(cls, job: job.Job): args = job._launch_settings.launch_args entity = job._entity env = job._launch_settings.env_vars - return cls(args, entity, env) + path = "/tmp/job" + return cls(args, entity, env, path) class MockLaunchArgs(launchArguments.LaunchArguments): @@ -182,9 +184,7 @@ class EchoHelloWorldEntity(entity.SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" def __init__(self): - path = tempfile.TemporaryDirectory() - self._finalizer = weakref.finalize(self, path.cleanup) - super().__init__("test-entity", path, _mock.Mock()) + super().__init__("test-entity", _mock.Mock()) def __eq__(self, other): if type(self) is not type(other): diff --git a/tests/test_generator.py b/tests/test_generator.py new file mode 100644 index 000000000..13d163fc1 --- /dev/null +++ b/tests/test_generator.py @@ -0,0 +1,366 @@ +import filecmp +import itertools +import os +import pathlib +import random +from glob import glob +from os import listdir +from os import path as osp + +import pytest + +from smartsim import Experiment +from smartsim._core.generation.generator import Generator +from smartsim.entity import Application, Ensemble, SmartSimEntity, _mock +from smartsim.entity.files import EntityFiles +from smartsim.launchable import Job +from smartsim.settings import LaunchSettings, dispatch + +# TODO Add JobGroup tests when JobGroup becomes a Launchable + +pytestmark = pytest.mark.group_a + + +def random_id(): + return str(random.randint(1, 100)) + + +@pytest.fixture +def get_gen_copy_dir(fileutils): + yield fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) + + +@pytest.fixture +def get_gen_symlink_dir(fileutils): + yield fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) + + +@pytest.fixture +def get_gen_configure_dir(fileutils): + yield fileutils.get_test_conf_path(osp.join("generator_files", "tag_dir_template")) + + +@pytest.fixture +def generator_instance(test_dir) -> Generator: + """Fixture to create an instance of Generator.""" + root = pathlib.Path(test_dir, "temp_id") + yield Generator(root=root) + + +def test_log_file_path(generator_instance): + """Test if the log_file function returns the correct log path.""" + base_path = "/tmp" + expected_path = osp.join(base_path, "smartsim_params.txt") + assert generator_instance._log_file(base_path) == pathlib.Path(expected_path) + + +def test_generate_job_directory(test_dir, wlmutils, generator_instance): + """Test Generator.generate_job""" + # Create Job + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application( + "app_name", exe="python", run_settings="RunSettings" + ) # Mock RunSettings + job = Job(app, launch_settings) + # Mock id + run_id = "temp_id" + # Call Generator.generate_job + job_run_path = generator_instance.generate_job(job, 0) + assert isinstance(job_run_path, pathlib.Path) + expected_run_path = ( + pathlib.Path(test_dir) + / run_id + / f"{job.__class__.__name__.lower()}s" + / f"{app.name}-{0}" + / "run" + ) + assert job_run_path == expected_run_path + expected_log_path = ( + pathlib.Path(test_dir) + / run_id + / f"{job.__class__.__name__.lower()}s" + / f"{app.name}-{0}" + / "log" + ) + assert osp.isdir(expected_run_path) + assert osp.isdir(expected_log_path) + # Assert smartsim params file created + assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) + # Assert smartsim params correctly written to + with open(expected_log_path / "smartsim_params.txt", "r") as file: + content = file.read() + assert "Generation start date and time:" in content + + +def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): + """Test that Job directory was created from Experiment._generate.""" + # Create Experiment + exp = Experiment(name="experiment_name", exp_path=test_dir) + # Create Job + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job = Job(app, launch_settings) + # Generate Job directory + job_index = 1 + job_execution_path = exp._generate(generator_instance, job, job_index) + # Assert Job run directory exists + assert osp.isdir(job_execution_path) + # Assert Job log directory exists + head, _ = os.path.split(job_execution_path) + expected_log_path = pathlib.Path(head) / "log" + assert osp.isdir(expected_log_path) + + +def test_generate_copy_file(generator_instance, fileutils, wlmutils): + """Test that attached copy files are copied into Job directory""" + # Create the Job and attach copy generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + script = fileutils.get_test_conf_path("sleep.py") + app.attach_generator_files(to_copy=script) + job = Job(app, launch_settings) + + # Create the experiment + path = generator_instance.generate_job(job, 1) + expected_file = pathlib.Path(path) / "sleep.py" + assert osp.isfile(expected_file) + + +def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance): + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + app.attach_generator_files(to_copy=get_gen_copy_dir) + job = Job(app, launch_settings) + + # Call Generator.generate_job + path = generator_instance.generate_job(job, 1) + expected_folder = path / "to_copy_dir" + assert osp.isdir(expected_folder) + + +def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlink_dir): + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name", "python", run_settings="RunSettings") # Mock RunSettings + # Attach directory to Application + app.attach_generator_files(to_symlink=get_gen_symlink_dir) + # Create Job + job = Job(app, launch_settings) + + # Call Generator.generate_job + path = generator_instance.generate_job(job, 1) + expected_folder = path / "to_symlink_dir" + assert osp.isdir(expected_folder) + assert expected_folder.is_symlink() + assert os.fspath(expected_folder.resolve()) == osp.realpath(get_gen_symlink_dir) + # Combine symlinked file list and original file list for comparison + for written, correct in itertools.zip_longest( + listdir(get_gen_symlink_dir), listdir(expected_folder) + ): + # For each pair, check if the filenames are equal + assert written == correct + + +def test_generate_symlink_file(get_gen_symlink_dir, wlmutils, generator_instance): + # Create the Job and attach generator file + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + app = Application("name", "python", "RunSettings") + # Path of directory to symlink + symlink_dir = get_gen_symlink_dir + # Get a list of all files in the directory + symlink_files = sorted(glob(symlink_dir + "/*")) + # Attach directory to Application + app.attach_generator_files(to_symlink=symlink_files) + # Create Job + job = Job(app, launch_settings) + + # Call Generator.generate_job + path = generator_instance.generate_job(job, 1) + expected_file = path / "mock2.txt" + assert osp.isfile(expected_file) + assert expected_file.is_symlink() + assert os.fspath(expected_file.resolve()) == osp.join( + osp.realpath(get_gen_symlink_dir), "mock2.txt" + ) + + +def test_generate_configure(fileutils, wlmutils, generator_instance): + # Directory of files to configure + conf_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "marked/") + ) + # Retrieve a list of files for configuration + tagged_files = sorted(glob(conf_path + "/*")) + # Retrieve directory of files to compare after Experiment.generate_experiment completion + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + # Retrieve list of files in correctly tagged directory for comparison + correct_files = sorted(glob(correct_path + "/*")) + # Initialize a Job + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + param_dict = { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + "VALID": "valid", + } + app = Application("name_1", "python", "RunSettings", params=param_dict) + app.attach_generator_files(to_configure=tagged_files) + job = Job(app, launch_settings) + + # Call Generator.generate_job + path = generator_instance.generate_job(job, 0) + # Retrieve the list of configured files in the test directory + configured_files = sorted(glob(str(path) + "/*")) + # Use filecmp.cmp to check that the corresponding files are equal + for written, correct in itertools.zip_longest(configured_files, correct_files): + assert filecmp.cmp(written, correct) + + +def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_instance): + """Test that Job directory was created from Experiment.""" + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + for i, job in enumerate(job_list): + job_run_path = exp._generate(generator_instance, job, i) + head, _ = os.path.split(job_run_path) + expected_log_path = pathlib.Path(head) / "log" + assert osp.isdir(job_run_path) + assert osp.isdir(pathlib.Path(expected_log_path)) + + +def test_generate_ensemble_directory(wlmutils, generator_instance): + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + for i, job in enumerate(job_list): + # Call Generator.generate_job + path = generator_instance.generate_job(job, i) + # Assert run directory created + assert osp.isdir(path) + # Assert smartsim params file created + head, _ = os.path.split(path) + expected_log_path = pathlib.Path(head) / "log" + assert osp.isdir(expected_log_path) + assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) + # Assert smartsim params correctly written to + with open(expected_log_path / "smartsim_params.txt", "r") as file: + content = file.read() + assert "Generation start date and time:" in content + + +def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: random_id(), + ) + ensemble = Ensemble("ensemble-name", "echo", replicas=2) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + job_dir = listdir(jobs_dir) + for ensemble_dir in job_dir: + run_path = os.path.join(jobs_dir, ensemble_dir, "run") + log_path = os.path.join(jobs_dir, ensemble_dir, "log") + assert osp.isdir(run_path) + assert osp.isdir(log_path) + + +def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: random_id(), + ) + ensemble = Ensemble( + "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) + ) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + job_dir = listdir(jobs_dir) + for ensemble_dir in job_dir: + copy_folder_path = os.path.join(jobs_dir, ensemble_dir, "run", "to_copy_dir") + assert osp.isdir(copy_folder_path) + + +def test_generate_ensemble_symlink( + test_dir, wlmutils, monkeypatch, get_gen_symlink_dir +): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: random_id(), + ) + ensemble = Ensemble( + "ensemble-name", + "echo", + replicas=2, + files=EntityFiles(symlink=get_gen_symlink_dir), + ) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + job_dir = listdir(jobs_dir) + for ensemble_dir in job_dir: + sym_file_path = pathlib.Path(jobs_dir) / ensemble_dir / "run" / "to_symlink_dir" + assert osp.isdir(sym_file_path) + assert sym_file_path.is_symlink() + assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) + + +def test_generate_ensemble_configure( + test_dir, wlmutils, monkeypatch, get_gen_configure_dir +): + monkeypatch.setattr( + "smartsim.settings.dispatch._LauncherAdapter.start", + lambda launch, exe, job_execution_path, env: random_id(), + ) + params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} + # Retrieve a list of files for configuration + tagged_files = sorted(glob(get_gen_configure_dir + "/*")) + ensemble = Ensemble( + "ensemble-name", + "echo", + replicas=1, + files=EntityFiles(tagged=tagged_files), + file_parameters=params, + ) + launch_settings = LaunchSettings(wlmutils.get_test_launcher()) + job_list = ensemble.as_jobs(launch_settings) + exp = Experiment(name="exp_name", exp_path=test_dir) + exp.start(*job_list) + run_dir = listdir(test_dir) + jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + + def _check_generated(param_0, param_1, dir): + assert osp.isdir(dir) + assert osp.isfile(osp.join(dir, "tagged_0.sh")) + assert osp.isfile(osp.join(dir, "tagged_1.sh")) + + with open(osp.join(dir, "tagged_0.sh")) as f: + line = f.readline() + assert line.strip() == f'echo "Hello with parameter 0 = {param_0}"' + + with open(osp.join(dir, "tagged_1.sh")) as f: + line = f.readline() + assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"' + + _check_generated(0, 3, os.path.join(jobs_dir, "ensemble-name-1-1", "run")) + _check_generated(1, 2, os.path.join(jobs_dir, "ensemble-name-2-2", "run")) + _check_generated(1, 3, os.path.join(jobs_dir, "ensemble-name-3-3", "run")) + _check_generated(0, 2, os.path.join(jobs_dir, "ensemble-name-0-0", "run"))