Remove references to limit_app_cpus and throw exception

conftest.py

@@ -27,7 +27,6 @@
 import json
 import os
 import inspect
-import shutil
 import pytest
 import psutil
 import shutil
@@ -44,6 +43,7 @@
 from smartsim._core.config import CONFIG
 from smartsim.error import SSConfigError
 from subprocess import run
+import sys
 
 
 # Globals, yes, but its a testing file
@@ -113,7 +113,8 @@ def pytest_sessionfinish(session, exitstatus):
     returning the exit status to the system.
     """
     if exitstatus == 0:
-        shutil.rmtree(test_dir)
+        # shutil.rmtree(test_dir)
+        pass
     else:
         # kill all spawned processes in case of error
         kill_all_test_spawned_processes()
@@ -591,3 +592,38 @@ class MLUtils:
     def get_test_device():
         global test_device
         return test_device
+
+@pytest.fixture
+def coloutils():
+    return ColoUtils
+
+class ColoUtils:
+    def setup_test_colo(fileutils, db_type, exp_name, db_args, launcher="local"):
+        """Setup things needed for setting up the colo pinning tests"""
+
+        exp = Experiment(exp_name, launcher=launcher)
+
+        # get test setup
+        test_dir = fileutils.make_test_dir()
+        sr_test_script = fileutils.get_test_conf_path("send_data_local_smartredis.py")
+
+        # Create an app with a colo_db which uses 1 db_cpu
+        colo_settings = exp.create_run_settings(exe=sys.executable, exe_args=sr_test_script)
+        colo_model = exp.create_model(f"colocated_model", colo_settings)
+        colo_model.set_path(test_dir)
+
+        if db_type in ['tcp', "deprecated"]:
+            db_args["port"] = 6780
+            db_args["ifname"] = "lo"
+
+        colocate_fun = {
+            "tcp": colo_model.colocate_db_tcp,
+            "deprecated": colo_model.colocate_db,
+            "uds":colo_model.colocate_db_uds
+        }
+        colocate_fun[db_type](**db_args)
+
+        # assert model will launch with colocated db
+        assert colo_model.colocated
+        # Check to make sure that limit_db_cpus made it into the colo settings
+        return exp, colo_model

doc/changelog.rst

@@ -25,6 +25,7 @@ Description
 
 A full list of changes and detailed notes can be found below:
 
+- Replace `limit_app_cpus` with `limit_db_cpus` for co-located orchestrators
 - Remove wait time associated with Experiment launch summary
 - Update and rename Redis conf file
 - Migrate from redis-py-cluster to redis-py
@@ -40,22 +41,27 @@ Detailed notes
 
 - Typehints have been added. A makefile target `make check-mypy` executes static
   analysis with mypy. (PR295_, PR303_)
+- Replace `limit_app_cpus` with `limit_db_cpus` for co-located orchestrators.
+  This resolves some incorrect behavior/assumptions about how the application
+  would be pinned.  Instead, users should directly specify the binding options in
+  their application using the options appropriate for their launcher
 - Simplify code in `random_permutations` parameter generation strategy (PR300_)
 - Remove wait time associated with Experiment launch summary (PR298_)
 - Update Redis conf file to conform with Redis v7.0.5 conf file (PR293_)
 - Migrate from redis-py-cluster to redis-py for cluster status checks (PR292_)
 - Update full test suite to no longer require a tensorflow wheel to be available
   at test time. (PR291_)
 - Correct spelling of colocated in doc strings (PR290_)
-- Deprecated launcher-specific orchestrators, constants, and ML 
+- Deprecated launcher-specific orchestrators, constants, and ML
   utilities were removed. (PR289_)
 - Relax the coloredlogs version to be greater than 10.0 (PR288_)
 - Update the Github Actions runner image from `macos-10.15`` to `macos-12``. The
-  former began deprecation in May 2022 and was finally removed in May 2023. (PR285_)
-- The Fortran tutorials had not been fully updated to show how to handle 
+- The Fortran tutorials had not been fully updated to show how to handle
   return/error codes. These have now all been updated. (PR284_)
-- Orchestrator and Colocated DB now accept a list of interfaces to bind to. The 
+- Orchestrator and Colocated DB now accept a list of interfaces to bind to. The
   argument name is still `interface` for backward compatibility reasons. (PR281_)
+- Typehints have been added to public APIs. A makefile target to execute static
+  analysis with mypy is available `make check-mypy`. (PR295_)
 
 .. _PR303: https://github.com/CrayLabs/SmartSim/pull/303
 .. _PR300: https://github.com/CrayLabs/SmartSim/pull/300

doc/orchestrator.rst

@@ -97,22 +97,24 @@ Here is an example of creating a simple model that is colocated with an
   colo_model.colocate_db_uds(
           db_cpus=1,              # cpus given to the database on each node
           debug=False             # include debug information (will be slower)
-          pin_db_cpus=True,       # Pin the affinities of the co-located database
-                                  # to specific processors
+          limit_db_cpus=True,     # Limit the database to a specific set of cpus
+          ifname=network_interface # specify network interface(s) to use (i.e. "ib0" or ["ib0", "lo"])
   )
   exp.start(colo_model)
 
 
-By default, SmartSim will attempt to make sure that the database and the application
-do not fight over resources by taking over the affinity mapping process locally on
-each node. This can be disabled by setting ``pin_db_cpus`` to ``False``.
+By default, SmartSim will pin the database to the first _N_ CPUs according to ``db_cpus``. By
+specifying the optional argument ``db_cpu_list``, an alternative pinning can be specified (
+following the format used by ``taskset``). For example, ``db_cpu_list=0-2,5`` limits the database
+to be run on processors 0, 1, 2 and 5. Setting ``limit_db_cpus=False`` disables pinning for the database
+kernel scheduler. For optimal performance, most users will want to also modify the RunSettings for
+the model to pin their application to cores not occupied by the database.
 
-Especially for multicore machines, it can be useful to pin the CPUs used by the database.
-This is enabled by default and the database will automatically be bound to the first
-logical processors starting from 0 to ``db_cpus-1``. To specify a custom set of cpus,
-set ``db_cpu_list`` following the same syntax as the Linux command ``taskset -c``. The
-RunSettings for the simulation should similarly be bound (e.g. in Slurm using
-``cpu-bind``) to bind the application to a separate set of cores.
+.. note::
+
+  Pinning _only_ affects the co-located deployment because both the application and the database
+  are sharing the same compute node. For the clustered deployment, a shard occupies the entirerty
+  of the node.
 
 Redis
 =====

smartsim/entity/model.py

@@ -202,6 +202,9 @@ def colocate_db_uds(
         :type db_cpus: int, optional
         :param limit_db_cpus: whether to limit the number of cpus used by the database defaults to True
         :type limit_db_cpus: bool, optional
+        :param db_cpu_list: lists the cpus which the database can be run on. Follows `taskset -c` syntax
+                            e.g. '0-2,5' specifies processors 0, 1, 2, and 5
+        :type db_cpu_list: str, optional
         :param debug: launch Model with extra debug information about the colocated db
         :type debug: bool, optional
         :param kwargs: additional keyword arguments to pass to the orchestrator database
@@ -262,6 +265,9 @@ def colocate_db_tcp(
         :type db_cpus: int, optional
         :param limit_db_cpus: whether to limit the number of cpus used by the database, defaults to True
         :type limit_db_cpus: bool, optional
+        :param db_cpu_list: lists the cpus which the database can be run on. Follows `taskset -c` syntax
+                            e.g. '0-2,5' specifies processors 0, 1, 2, and 5
+        :type db_cpu_list: str, optional
         :param debug: launch Model with extra debug information about the colocated db
         :type debug: bool, optional
         :param kwargs: additional keyword arguments to pass to the orchestrator database
@@ -297,10 +303,14 @@ def _set_colocated_db_settings(
         if hasattr(self.run_settings, "_prep_colocated_db"):
             self.run_settings._prep_colocated_db(common_options["cpus"])
 
-        # TODO list which db settings can be extras
+        if "limit_app_cpus" in common_options:
+            raise SSUnsupportedError(
+                "Pinning of app CPUs via limit_app_cpus is no supported. Modify RunSettings " +
+                "instead using the correct binding option for your launcher."
+            )
 
+        # TODO list which db settings can be extras
         cpus = common_options["cpus"]
-        print(cpus)
         # Deal with cases where the database should be pinned to cpus
         # (1) if the user set a db_cpu_list, but not limit_db_cpus
         if common_options["db_cpu_list"] and not common_options["limit_db_cpus"]:
@@ -324,7 +334,6 @@ def _set_colocated_db_settings(
                     f"pinning to processors {cpu_list}"
                 )
             )
-            print(cpu_list)
             common_options["db_cpu_list"] = cpu_list
 
         colo_db_config = {}

tests/backends/test_dbmodel.py

@@ -311,7 +311,6 @@ def test_colocated_db_model_tf(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port(),
         db_cpus=1,
-        limit_app_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -370,7 +369,7 @@ def test_colocated_db_model_pytorch(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port(),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -413,7 +412,7 @@ def test_colocated_db_model_ensemble(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port(),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -425,7 +424,7 @@ def test_colocated_db_model_ensemble(fileutils, wlmutils):
         entity.colocate_db(
             port=wlmutils.get_test_port() + i,
             db_cpus=1,
-            limit_app_cpus=False,
+            limit_db_cpus=False,
             debug=True,
             ifname="lo",
         )
@@ -455,7 +454,7 @@ def test_colocated_db_model_ensemble(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port() + len(colo_ensemble),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -509,7 +508,7 @@ def test_colocated_db_model_ensemble_reordered(fileutils, wlmutils):
         entity.colocate_db(
             wlmutils.get_test_port() + i,
             db_cpus=1,
-            limit_app_cpus=False,
+            limit_db_cpus=False,
             debug=True,
             ifname="lo",
         )
@@ -528,7 +527,7 @@ def test_colocated_db_model_ensemble_reordered(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port() + len(colo_ensemble),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -565,7 +564,7 @@ def test_colocated_db_model_errors(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port(),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -585,7 +584,7 @@ def test_colocated_db_model_errors(fileutils, wlmutils):
         entity.colocate_db(
             port=wlmutils.get_test_port() + i,
             db_cpus=1,
-            limit_app_cpus=False,
+            limit_db_cpus=False,
             debug=True,
             ifname="lo",
         )
@@ -614,7 +613,7 @@ def test_colocated_db_model_errors(fileutils, wlmutils):
             entity.colocate_db(
                 port=wlmutils.get_test_port() + i,
                 db_cpus=1,
-                limit_app_cpus=False,
+                limit_db_cpus=False,
                 debug=True,
                 ifname="lo",
             )

tests/backends/test_dbscript.py

@@ -153,7 +153,7 @@ def test_colocated_db_script(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port(),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -204,7 +204,7 @@ def test_colocated_db_script_ensemble(fileutils, wlmutils):
         entity.colocate_db(
             port=wlmutils.get_test_port() + i,
             db_cpus=1,
-            limit_app_cpus=False,
+            limit_db_cpus=False,
             debug=True,
             ifname="lo",
         )
@@ -214,7 +214,7 @@ def test_colocated_db_script_ensemble(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port() + len(colo_ensemble),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -268,7 +268,7 @@ def test_colocated_db_script_ensemble_reordered(fileutils, wlmutils):
         entity.colocate_db(
             port=wlmutils.get_test_port() + i,
             db_cpus=1,
-            limit_app_cpus=False,
+            limit_db_cpus=False,
             debug=True,
             ifname="lo",
         )
@@ -278,7 +278,7 @@ def test_colocated_db_script_ensemble_reordered(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port() + len(colo_ensemble),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -315,7 +315,7 @@ def test_db_script_errors(fileutils, wlmutils):
     colo_model.colocate_db(
         port=wlmutils.get_test_port(),
         db_cpus=1,
-        limit_app_cpus=False,
+        limit_db_cpus=False,
         debug=True,
         ifname="lo",
     )
@@ -335,7 +335,7 @@ def test_db_script_errors(fileutils, wlmutils):
         entity.colocate_db(
             port=wlmutils.get_test_port() + i,
             db_cpus=1,
-            limit_app_cpus=False,
+            limit_db_cpus=False,
             debug=True,
             ifname="lo",
         )
@@ -358,7 +358,7 @@ def test_db_script_errors(fileutils, wlmutils):
             entity.colocate_db(
                 port=wlmutils.get_test_port() + i,
                 db_cpus=1,
-                limit_app_cpus=False,
+                limit_db_cpus=False,
                 debug=True,
                 ifname="lo",
             )

tests/conftest/setup_test_colo/colocated_model.out

@@ -0,0 +1,4 @@
+SmartRedis Library@19-19-36:WARNING: Environment variable SR_LOG_FILE is not set. Defaulting to stdout
+SmartRedis Library@19-19-36:WARNING: Environment variable SR_LOG_LEVEL is not set. Defaulting to INFO
+default@19-19-36:ERROR: Unable to connect to backend database: Failed to connect to Redis: Connection refused
+Test worked! Sent and received array: [1 2 3 4]