More easily discoverable dependencies (#635)

setup.py used to define dependencies in a way that was not amenable to
code scanning tools. Direct dependencies now appear directly in the
setup call and the definition of the SmartRedis version has been
removed. Additionally, the code scanning tool was failing to detect some
of the dependencies due to the existence of the requirements-doc.txt
file. These requirements are now listed in the `docs` extra.

[ committed by @ashao ]
[ reviewed by @ankona ]

.readthedocs.yaml

@@ -23,7 +23,7 @@ build:
       - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis
       - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard
     post_create_environment:
-      - python -m pip install .[dev]
+      - python -m pip install .[dev,docs]
       - cd smartredis; python -m pip install .
       - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran
       - ln -s smartredis/examples ./examples
@@ -37,7 +37,3 @@ build:
 sphinx:
   configuration: doc/conf.py
   fail_on_warning: true
-
-python:
-   install:
-   - requirements: doc/requirements-doc.txt

doc/changelog.md

@@ -15,6 +15,7 @@ To be released at some future point in time
 
 Description
 
+- Make dependencies more discoverable in setup.py
 - Add hardware pinning capability when using dragon
 - Pin NumPy version to 1.x
 - New launcher support for SGE (and similar derivatives)
@@ -25,6 +26,14 @@ Description
 
 Detailed Notes
 
+- setup.py used to define dependencies in a way that was not amenable
+  to code scanning tools. Direct dependencies now appear directly
+  in the setup call and the definition of the SmartRedis version
+  has been removed
+  ([SmartSim-PR635](https://github.com/CrayLabs/SmartSim/pull/635))
+- The separate definition of dependencies for the docs in
+  requirements-doc.txt is now defined as an extra.
+  ([SmartSim-PR635](https://github.com/CrayLabs/SmartSim/pull/635))
 - The new major version release of Numpy is incompatible with modules
   compiled against Numpy 1.x. For both SmartSim and SmartRedis we
   request a 1.x version of numpy. This is needed in SmartSim because

docker/docs/dev/Dockerfile

@@ -55,8 +55,7 @@ RUN git clone https://github.com/CrayLabs/SmartDashboard.git --branch develop --
     && rm -rf ~/.cache/pip
 
 # Install docs dependencies and SmartSim
-RUN python -m pip install -r doc/requirements-doc.txt \
-    && NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .
+RUN NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .[docs]
 
 # Note this is needed to ensure that the Sphinx builds. Can be removed with newer Tensorflow
 RUN python -m pip install typing_extensions==4.6.1

setup.py

@@ -165,25 +165,9 @@ def has_ext_modules(_placeholder):
 
 
 # Define needed dependencies for the installation
-deps = [
-    "packaging>=24.0",
-    "psutil>=5.7.2",
-    "coloredlogs>=10.0",
-    "tabulate>=0.8.9",
-    "redis>=4.5",
-    "tqdm>=4.50.2",
-    "filelock>=3.4.2",
-    "protobuf~=3.20",
-    "jinja2>=3.1.2",
-    "watchdog>=4.0.0",
-    "pydantic==1.10.14",
-    "pyzmq>=25.1.2",
-    "pygithub>=2.3.0",
-    "numpy<2"
-]
 
 # Add SmartRedis at specific version
-deps.append("smartredis>={}".format(versions.SMARTREDIS))
+# install_requires.append("smartredis>={}".format(versions.SMARTREDIS))
 
 extras_require = {
     "dev": [
@@ -205,6 +189,24 @@ def has_ext_modules(_placeholder):
         "types-setuptools",
         "typing_extensions>=4.1.0",
     ],
+    "docs": [
+        "Sphinx==6.2.1",
+        "breathe==4.35.0",
+        "sphinx-fortran==1.1.1",
+        "sphinx-book-theme==1.0.1",
+        "sphinx-copybutton==0.5.2",
+        "sphinx-tabs==3.4.4",
+        "nbsphinx==0.9.3",
+        "docutils==0.18.1",
+        "torch==2.0.1",
+        "tensorflow==2.13.1",
+        "ipython",
+        "jinja2==3.1.2",
+        "sphinx-design",
+        "pypandoc",
+        "sphinx-autodoc-typehints",
+        "myst_parser",
+    ],
     # see smartsim/_core/_install/buildenv.py for more details
     **versions.ml_extras_required(),
 }
@@ -213,7 +215,23 @@ def has_ext_modules(_placeholder):
 # rest in setup.cfg
 setup(
     version=smartsim_version,
-    install_requires=deps,
+    install_requires=[
+        "packaging>=24.0",
+        "psutil>=5.7.2",
+        "coloredlogs>=10.0",
+        "tabulate>=0.8.9",
+        "redis>=4.5",
+        "tqdm>=4.50.2",
+        "filelock>=3.4.2",
+        "protobuf~=3.20",
+        "jinja2>=3.1.2",
+        "watchdog>=4.0.0",
+        "pydantic==1.10.14",
+        "pyzmq>=25.1.2",
+        "pygithub>=2.3.0",
+        "numpy<2",
+        "smartredis>=0.5,<0.6",
+    ],
     cmdclass={
         "build_py": SmartSimBuild,
         "install": InstallPlatlib,

smartsim/_core/_install/buildenv.py

@@ -242,7 +242,7 @@ class Versioner:
     ``smart build`` command to determine which dependency versions
     to look for and download.
 
-    Default versions for SmartSim, SmartRedis, Redis, and RedisAI are
+    Default versions for SmartSim, Redis, and RedisAI are
     all set here. Setting a default version for RedisAI also dictates
     default versions of the machine learning libraries.
     """
@@ -252,7 +252,6 @@ class Versioner:
 
     # Versions
     SMARTSIM = Version_(get_env("SMARTSIM_VERSION", "0.7.0"))
-    SMARTREDIS = Version_(get_env("SMARTREDIS_VERSION", "0.5.3"))
     SMARTSIM_SUFFIX = get_env("SMARTSIM_SUFFIX", "")
 
     # Redis
@@ -284,7 +283,6 @@ class Versioner:
     def as_dict(self, db_name: DbEngine = "REDIS") -> t.Dict[str, t.Tuple[str, ...]]:
         pkg_map = {
             "SMARTSIM": self.SMARTSIM,
-            "SMARTREDIS": self.SMARTREDIS,
             db_name: self.REDIS,
             "REDISAI": self.REDISAI,
             "TORCH": self.TORCH,