Addressing PR comments (#528)

Addressing PR comments, added documentation, fixed bugs in tests

 [ committed by @juliaputko ]
 [ reviewed by @AlyssaCote , @amandarichardsonn, @ankona ]

doc/api/smartsim_api.rst

@@ -25,6 +25,7 @@ Experiment
    Experiment.finished
    Experiment.get_status
    Experiment.reconnect_orchestrator
+   Experiment.preview
    Experiment.summary
 
 .. autoclass:: Experiment

doc/changelog.rst

@@ -18,6 +18,7 @@ To be released at some future point in time
 
 Description
 
+- Preview entities on experiment before start
 - Change default path for entities
 - Drop Python 3.8 support
 - Update watchdog dependency
@@ -41,6 +42,8 @@ Description
 
 Detailed Notes
 
+- Added preview functionality to Experiment, including preview of all entities,
+  active infrastructure and client configuration. (SmartSim-PR525_)
 - The default path for an entity is now the path to the experiment / the
   entity name. create_database and create_ensemble now have path arguments.
   All path arguments are compatible with relative paths. Relative paths are
@@ -104,6 +107,7 @@ Detailed Notes
   handler. SmartSim will now attempt to kill any launched jobs before calling
   the previously registered signal handler. (SmartSim-PR535_)
 
+.. _SmartSim-PR525: https://github.com/CrayLabs/SmartSim/pull/525
 .. _SmartSim-PR533: https://github.com/CrayLabs/SmartSim/pull/533
 .. _SmartSim-PR544: https://github.com/CrayLabs/SmartSim/pull/544
 .. _SmartSim-PR540: https://github.com/CrayLabs/SmartSim/pull/540

doc/experiment.rst

@@ -126,6 +126,9 @@ the ``Experiment`` post-creation methods.
    * - ``get_status``
      - ``exp.get_status(*args)``
      - Retrieve Entity Status
+   * - ``preview``
+     - ``exp.preview(*args, ...)``
+     - Preview an experiment
 
 .. _orchestrator_exp_docs:
 
@@ -329,6 +332,9 @@ Example
   *Generating*
    - the ``Orchestrator`` output directory
    - the ``Model`` output directory
+  *Preview*
+   - the ``Orchestrator`` contents
+   - the ``Model`` contents
   *Starting*
    - an in-memory database (standalone ``Orchestrator``)
    - an application (``Model``)
@@ -418,6 +424,68 @@ Generating
   The ``Experiment.generate`` call places the `.err` and `.out` log files in the entity
   subdirectories within the main ``Experiment`` directory.
 
+Preview
+=======
+.. compound::
+  Optionally, we preview the Experiment. The ``Experiment.preview`` aggregates multiple pieces of information to give users
+  insight into what and how entities will be launched before the experiment is started. Any instance of ``Model``, ``Ensemble``,
+  or ``Orchestrator`` created by the Experiment can be passed as an argument to the preview method.
+  We preview the ``Orchestrator`` and ``Model`` entities by passing the ``Orchestrator`` and ``Model`` instances to ``exp.preview``:
+
+  .. literalinclude:: tutorials/doc_examples/experiment_doc_examples/exp.py
+    :language: python
+    :linenos:
+    :lines: 20-21
+
+  *  `verbosity_level="info"` instructs preview to display user-defined fields and entities.
+  *  `verbosity_level="debug"` instructs preview to display user-defined field and entities and auto-generated fields.
+  *  `verbosity_level="developer"` instructs preview to display user-defined field and entities, auto-generated fields, and run commands.
+  *  `output_format="plain_text"` sets the output format. The only accepted output format is 'plain_text'.
+  *  `output_filename="test_name.txt"` specifies name of file and extension to write preview data to. If no output filename is set, the preview will be output to stdout.
+
+When executed, the preview shows the following in stdout:
+
+::
+
+  === Experiment Overview ===
+
+    Experiment Name: example-experiment
+      Experiment Path: absolute/path/to/SmartSim/example-experiment
+      Launcher: local
+
+  === Entity Preview ===
+
+    == Orchestrators ==
+
+      = Database Identifier: orchestrator =
+          Path: absolute/path/to/SmartSim/example-experiment/orchestrator
+          Shards: 1
+          TCP/IP Port(s):
+            6379
+          Network Interface: ib0
+          Type: redis
+          Executable: absolute/path/to/SmartSim/smartsim/_core/bin/redis-server
+
+    == Models ==
+
+      = Model Name: hello_world =
+          Path: absolute/path/to/SmartSim/example-experiment/hello_world
+          Executable: /bin/echo
+          Executable Arguments:
+            Hello
+            World
+          Client Configuration:
+            Database Identifier: orchestrator
+              Database Backend: redis
+              TCP/IP Port(s):
+                6379
+              Type: Standalone
+            Outgoing Key Collision Prevention (Key Prefixing):
+              Tensors: Off
+              Datasets: Off
+              ML Models/Torch Scripts: Off
+              Aggregation Lists: Off
+
 Starting
 ========
 .. compound::
@@ -428,7 +496,7 @@ Starting
   .. literalinclude:: tutorials/doc_examples/experiment_doc_examples/exp.py
     :language: python
     :linenos:
-    :lines: 20-21
+    :lines: 23-24
 
 Stopping
 ========
@@ -439,7 +507,7 @@ Stopping
   .. literalinclude:: tutorials/doc_examples/experiment_doc_examples/exp.py
     :language: python
     :linenos:
-    :lines: 23-26
+    :lines: 26-27
 
   Notice that we use the ``Experiment.summary`` function to print
   the summary of the workflow.
@@ -454,4 +522,4 @@ When you run the experiment, the following output will appear::
 .. note::
   Failure to tear down the ``Orchestrator`` at the end of an ``Experiment``
   may lead to ``Orchestrator`` launch failures if another ``Experiment`` is
-  started on the same node.
+  started on the same node.

doc/tutorials/doc_examples/experiment_doc_examples/exp.py

@@ -17,10 +17,13 @@
 # Generate the output directory
 exp.generate(standalone_database, model, overwrite=True)
 
+# Preview the experiment
+exp.preview(standalone_database, model, verbosity_level="debug")
+
 # Launch the Orchestrator then Model instance
 exp.start(standalone_database, model)
 
 # Clobber the Orchestrator
 exp.stop(standalone_database)
 # Log the summary of the Experiment
-smartsim_logger.info(exp.summary())
+smartsim_logger.info(exp.summary())

smartsim/_core/control/controller.py

@@ -145,7 +145,8 @@ def start(
             self.poll(5, True, kill_on_interrupt=kill_on_interrupt)
 
     @property
-    def active_orch_dict(self) -> t.Dict[str, Job]:
+    def active_orchestrator_jobs(self) -> t.Dict[str, Job]:
+        """Return active orchestrator jobs."""
         return {**self._jobs.db_jobs}
 
     @property

smartsim/_core/control/previewrenderer.py

@@ -57,11 +57,15 @@ class Verbosity(str, Enum):
 
 @pass_eval_context
 def as_toggle(_eval_ctx: u.F, value: bool) -> str:
+    """Return "On" if value returns True,
+    and "Off" is value returns False.
+    """
     return "On" if value else "Off"
 
 
 @pass_eval_context
 def get_ifname(_eval_ctx: u.F, value: t.List[str]) -> str:
+    """Extract Network Interface from orchestrator run settings."""
     if value:
         for val in value:
             if "ifname=" in val:
@@ -72,6 +76,7 @@ def get_ifname(_eval_ctx: u.F, value: t.List[str]) -> str:
 
 @pass_eval_context
 def get_dbtype(_eval_ctx: u.F, value: str) -> str:
+    """Extract data base type."""
     if value:
         if "-cli" in value:
             db_type, _ = value.split("/")[-1].split("-", 1)
@@ -81,14 +86,34 @@ def get_dbtype(_eval_ctx: u.F, value: str) -> str:
 
 @pass_eval_context
 def is_list(_eval_ctx: u.F, value: str) -> bool:
+    """Return True if item is of type list, and False
+    otherwise, to determine how Jinja template should
+    render an item.
+    """
     return isinstance(value, list)
 
 
+def render_to_file(content: str, filename: str) -> None:
+    """Output preview to a file if an output filename
+    is specified.
+
+    :param content: The rendered preview.
+    :type content: str
+    :param filename: The name of the file to write the preview to.
+    "type filename: str
+    """
+    filename = find_available_filename(filename)
+
+    with open(filename, "w", encoding="utf-8") as prev_file:
+        prev_file.write(content)
+
+
 def render(
     exp: "Experiment",
     manifest: t.Optional[Manifest] = None,
     verbosity_level: Verbosity = Verbosity.INFO,
     output_format: Format = Format.PLAINTEXT,
+    output_filename: t.Optional[str] = None,
     active_dbjobs: t.Optional[t.Dict[str, Job]] = None,
 ) -> str:
     """
@@ -103,9 +128,11 @@ def render(
     :type output_format: _OutputFormatString
     """
 
-    verbosity_level = _check_verbosity_level(Verbosity(verbosity_level))
+    verbosity_level = Verbosity(verbosity_level)
+
+    _check_output_format(output_format)
 
-    loader = jinja2.PackageLoader("templates")
+    loader = jinja2.PackageLoader("smartsim.templates")
     env = jinja2.Environment(loader=loader, autoescape=True)
 
     env.filters["as_toggle"] = as_toggle
@@ -114,17 +141,14 @@ def render(
     env.filters["is_list"] = is_list
     env.globals["Verbosity"] = Verbosity
 
-    version = f"_{output_format}"
-    tpl_path = f"preview/base{version}.template"
-
-    _check_output_format(output_format)
+    tpl_path = f"preview/{output_format.value}/base.template"
 
     tpl = env.get_template(tpl_path)
 
     if verbosity_level == Verbosity.INFO:
         logger.warning(
-            "Only showing user set parameters. Some entity fields are "
-            "truncated. To view truncated fields: use verbosity_level "
+            "Only showing user set parameters. Some internal entity "
+            "fields are truncated. To view truncated fields: use verbosity_level "
             "'developer' or 'debug.'"
         )
 
@@ -135,18 +159,15 @@ def render(
         config=CONFIG,
         verbosity_level=verbosity_level,
     )
-    return rendered_preview
 
-
-def preview_to_file(content: str, filename: str) -> None:
-    """
-    Output preview to a file if an output filename
-    is specified.
-    """
-    filename = find_available_filename(filename)
-
-    with open(filename, "w", encoding="utf-8") as prev_file:
-        prev_file.write(content)
+    if output_filename:
+        render_to_file(
+            rendered_preview,
+            output_filename,
+        )
+    else:
+        logger.info(rendered_preview)
+    return rendered_preview
 
 
 def find_available_filename(filename: str) -> str:
@@ -168,20 +189,5 @@ def _check_output_format(output_format: Format) -> None:
     Check that a valid file output format is given.
     """
     if not output_format == Format.PLAINTEXT:
-        raise PreviewFormatError(
-            f"The only valid output format currently available is {Format.PLAINTEXT}"
-        )
-
-
-def _check_verbosity_level(
-    verbosity_level: Verbosity,
-) -> Verbosity:
-    """
-    Check that the given verbosity level is valid.
-    """
-    if not isinstance(verbosity_level, Verbosity):
-
-        logger.warning(f"'{verbosity_level}' is an unsupported verbosity level.\
- Setting verbosity to: {Verbosity.INFO}")
-        return Verbosity.INFO
-    return verbosity_level
+        raise PreviewFormatError(f"The only valid output format currently available \
+is {Format.PLAINTEXT.value}")

smartsim/entity/dbobject.py

@@ -197,8 +197,6 @@ def __init__(
         if not script and not script_path:
             raise ValueError("Either script or script_path must be provided")
 
-        self.script_path = script_path
-
     @property
     def script(self) -> t.Optional[str]:
         return self.func
@@ -278,7 +276,6 @@ def __init__(
         self.min_batch_timeout = min_batch_timeout
         self.tag = tag
         self.inputs, self.outputs = self._check_tensor_args(inputs, outputs)
-        self.model_file = model_file
 
     @property
     def model(self) -> t.Optional[bytes]:

smartsim/entity/ensemble.py

@@ -103,6 +103,7 @@ def __init__(
         self._key_prefixing_enabled = True
         self.batch_settings = init_default({}, batch_settings, BatchSettings)
         self.run_settings = init_default({}, run_settings, RunSettings)
+        self.replicas: str
 
         super().__init__(name, str(path), perm_strat=perm_strat, **kwargs)
 
@@ -120,6 +121,7 @@ def _initialize_entities(self, **kwargs: t.Any) -> None:
         """
         strategy = self._set_strategy(kwargs.pop("perm_strat"))
         replicas = kwargs.pop("replicas", None)
+        self.replicas = replicas
 
         # if a ensemble has parameters and run settings, create
         # the ensemble and assign run_settings to each member

smartsim/error/errors.py

@@ -151,8 +151,8 @@ class SmartSimCLIActionCancelled(SmartSimError):
     """Raised when a `smart` CLI command is terminated"""
 
 
-class PreviewException(Exception):
-    """Raised when a part of preview isn't support by SmartSim yet"""
+class PreviewException(SSUnsupportedError):
+    """Raised when a part of preview isn't supported by SmartSim yet"""
 
 
 class PreviewFormatError(PreviewException):