Update dataloaders to use Aggregation Lists

doc/changelog.rst

@@ -22,6 +22,7 @@ This section details changes made in the development branch that have not yet be
 
 Description
 
+- Update ML data loaders to make use of SmartRedis's aggregation lists
 - Drop support for Ray
 - Allow for models to be launched independently as batch jobs
 - Update to current version of Redis
@@ -31,6 +32,7 @@ Description
 
 Detailed Notes
 
+- The PyTorch and TensorFlow data loaders were update to make use of aggregation lists. This breaks their API, but makes them easier to use. (PR264_)
 - The support for Ray was dropped, as its most recent versions caused problems when deployed through SmartSim.
   We plan to release a separate add-on library to accomplish the same results. If
   you are interested in getting the Ray launch functionality back in your workflow, please get in touch with us! (PR263_)
@@ -49,6 +51,7 @@ Detailed Notes
 - The release of RedisAI 1.2.7 allows us to update support for recent versions of PyTorch, Tensorflow, and ONNX (PR234_)
 - Make installation of correct Torch backend more reliable according to instruction from PyTorch
 
+.. _PR264: https://github.com/CrayLabs/SmartSim/pull/264
 .. _PR263: https://github.com/CrayLabs/SmartSim/pull/263
 .. _PR258: https://github.com/CrayLabs/SmartSim/pull/258
 .. _PR256: https://github.com/CrayLabs/SmartSim/pull/256

smartsim/_core/control/controller.py

@@ -290,7 +290,6 @@ def _launch(self, manifest):
                 raise SmartSimError(msg)
             self._launch_orchestrator(orchestrator)
 
-
         if self.orchestrator_active:
             self._set_dbobjects(manifest)
 

smartsim/_core/control/manifest.py

@@ -50,7 +50,6 @@ def __init__(self, *args):
         self._check_names(self._deployables)
         self._check_entity_lists_nonempty()
 
-
     @property
     def db(self):
         """Return Orchestrator instances in Manifest
@@ -69,7 +68,6 @@ def db(self):
                 _db = deployable
         return _db
 
-
     @property
     def models(self):
         """Return Model instances in Manifest
@@ -83,7 +81,6 @@ def models(self):
                 _models.append(deployable)
         return _models
 
-
     @property
     def ensembles(self):
         """Return Ensemble instances in Manifest
@@ -103,7 +100,6 @@ def ensembles(self):
 
         return _ensembles
 
-
     @property
     def all_entity_lists(self):
         """All entity lists, including ensembles and
@@ -119,7 +115,6 @@ def all_entity_lists(self):
 
         return _all_entity_lists
 
-
     def _check_names(self, deployables):
         used = []
         for deployable in deployables:
@@ -130,7 +125,6 @@ def _check_names(self, deployables):
                 raise SmartSimError("User provided two entities with the same name")
             used.append(name)
 
-
     def _check_types(self, deployables):
         for deployable in deployables:
             if not (
@@ -141,15 +135,13 @@ def _check_types(self, deployables):
                     f"Entity has type {type(deployable)}, not SmartSimEntity or EntityList"
                 )
 
-
     def _check_entity_lists_nonempty(self):
         """Check deployables for sanity before launching"""
 
         for entity_list in self.all_entity_lists:
             if len(entity_list) < 1:
                 raise ValueError(f"{entity_list.name} is empty. Nothing to launch.")
 
-
     def __str__(self):
         s = ""
         e_header = "=== Ensembles ===\n"

smartsim/experiment.py

@@ -774,7 +774,12 @@ def summary(self, format="github"):
                 )
         else:
             return tabulate(
-                values, headers, showindex=True, tablefmt=format, missingval="None", disable_numparse=True
+                values,
+                headers,
+                showindex=True,
+                tablefmt=format,
+                missingval="None",
+                disable_numparse=True,
             )
 
     def _launch_summary(self, manifest):

smartsim/ml/__init__.py

@@ -24,9 +24,4 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from .data import (
-    DynamicDataDownloader,
-    StaticDataDownloader,
-    TrainingDataUploader,
-    form_name,
-)
+from .data import DataDownloader, DataInfo, TrainingDataUploader, form_name