Redis removal

.github/workflows/run_tests.yml

@@ -117,7 +117,7 @@ jobs:
       - name: Install SmartSim (with ML backends)
         run: |
           python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis
-          python -m pip install .[dev,ml]
+          python -m pip install .[dev,mypy,ml]
 
       - name: Install ML Runtimes with Smart (with pt, tf, and onnx support)
         if: contains( matrix.os, 'ubuntu' ) || contains( matrix.os, 'macos-12')
@@ -129,7 +129,6 @@ jobs:
 
       - name: Run mypy
         run: |
-          python -m pip install .[mypy]
           make check-mypy
 
       # TODO: Re-enable static analysis once API is firmed up

doc/changelog.md

@@ -15,6 +15,7 @@ To be released at some future point in time
 
 Description
 
+- Mitigate dependency installation issues
 - Fix internal host name representation for Dragon backend
 - Make dependencies more discoverable in setup.py
 - Add hardware pinning capability when using dragon
@@ -27,6 +28,12 @@ Description
 
 Detailed Notes
 
+- Installation of mypy or dragon in separate build actions caused
+  some dependencies (typing_extensions, numpy) to be upgraded and
+  caused runtime failures. The build actions were tweaked to include
+  all optional dependencies to be considered by pip during resolution.
+  Additionally, the numpy version was capped on dragon installations.
+  ([SmartSim-PR653](https://github.com/CrayLabs/SmartSim/pull/653))
 - setup.py used to define dependencies in a way that was not amenable
   to code scanning tools. Direct dependencies now appear directly
   in the setup call and the definition of the SmartRedis version

smartsim/_core/_cli/build.py

@@ -46,7 +46,6 @@
 )
 from smartsim._core._install.builder import BuildError, Device
 from smartsim._core.config import CONFIG
-from smartsim._core.utils.helpers import installed_redisai_backends
 from smartsim.error import SSConfigError
 from smartsim.log import get_logger
 
@@ -60,12 +59,13 @@
 
 def check_py_onnx_version(versions: Versioner) -> None:
     """Check Python environment for ONNX installation"""
+    # TODO: Add new baceknd for skl2onnx, onnxmltools, and scikit-learn
     _check_packages_in_python_env(
         {
             "onnx": Version_(versions.ONNX),
-            "skl2onnx": Version_(versions.REDISAI.skl2onnx),
-            "onnxmltools": Version_(versions.REDISAI.onnxmltools),
-            "scikit-learn": Version_(getattr(versions.REDISAI, "scikit-learn")),
+            # "skl2onnx": Version_(versions.None.skl2onnx),
+            # "onnxmltools": Version_(versions.None.onnxmltools),
+            # "scikit-learn": Version_(getattr(versions.None, "scikit-learn")),
         },
     )
 
@@ -75,43 +75,14 @@ def check_py_tf_version(versions: Versioner) -> None:
     _check_packages_in_python_env({"tensorflow": Version_(versions.TENSORFLOW)})
 
 
-def check_backends_install() -> bool:
-    """Checks if backends have already been installed.
-    Logs details on how to proceed forward
-    if the RAI_PATH environment variable is set or if
-    backends have already been installed.
-    """
-    rai_path = os.environ.get("RAI_PATH", "")
-    installed = installed_redisai_backends()
-    msg = ""
-
-    if rai_path and installed:
-        msg = (
-            f"There is no need to build. backends are already built and "
-            f"specified in the environment at 'RAI_PATH': {CONFIG.redisai}"
-        )
-    elif rai_path and not installed:
-        msg = (
-            "Before running 'smart build', unset your RAI_PATH environment "
-            "variable with 'unset RAI_PATH'."
-        )
-    elif not rai_path and installed:
-        msg = (
-            "If you wish to re-run `smart build`, you must first run `smart clean`."
-            " The following backend(s) must be removed: " + ", ".join(installed)
-        )
-
-    if msg:
-        logger.error(msg)
 
-    return not bool(msg)
 
 
 def build_feature_store(
     build_env: BuildEnv, versions: Versioner, keydb: bool, verbose: bool
 ) -> None:
     # check feature store installation
-    feature_store_name = "KeyDB" if keydb else "Redis"
+    feature_store_name = "KeyDB" if keydb else None
     feature_store_builder = builder.FeatureStoreBuilder(
         build_env(),
         jobs=build_env.JOBS,
@@ -120,112 +91,17 @@ def build_feature_store(
         malloc=build_env.MALLOC,
         verbose=verbose,
     )
+    # TODO: Add new feature store
     if not feature_store_builder.is_built:
-        logger.info(
-            f"Building {feature_store_name} version {versions.REDIS} "
-            f"from {versions.REDIS_URL}"
-        )
-        feature_store_builder.build_from_git(versions.REDIS_URL, versions.REDIS_BRANCH)
+        # logger.info(
+        #     f"Building {feature_store_name} version {versions.None} "
+        #     f"from {versions.None}"
+        # )
+        #feature_store_builder.build_from_git(versions.None, versions.None)
         feature_store_builder.cleanup()
     logger.info(f"{feature_store_name} build complete!")
 
 
-def build_redis_ai(
-    build_env: BuildEnv,
-    versions: Versioner,
-    device: Device,
-    use_torch: bool = True,
-    use_tf: bool = True,
-    use_onnx: bool = False,
-    torch_dir: t.Union[str, Path, None] = None,
-    libtf_dir: t.Union[str, Path, None] = None,
-    verbose: bool = False,
-    torch_with_mkl: bool = True,
-) -> None:
-    # make sure user isn't trying to do something silly on MacOS
-    if build_env.PLATFORM == "darwin" and device == Device.GPU:
-        raise BuildError("SmartSim does not support GPU on MacOS")
-
-    # decide which runtimes to build
-    print("\nML Backends Requested")
-    backends_table = [
-        ["PyTorch", versions.TORCH, color_bool(use_torch)],
-        ["TensorFlow", versions.TENSORFLOW, color_bool(use_tf)],
-        ["ONNX", versions.ONNX, color_bool(use_onnx)],
-    ]
-    print(tabulate(backends_table, tablefmt="fancy_outline"), end="\n\n")
-    print(f"Building for GPU support: {color_bool(device == Device.GPU)}\n")
-
-    if not check_backends_install():
-        sys.exit(1)
-
-    # TORCH
-    if use_torch and torch_dir:
-        torch_dir = Path(torch_dir).resolve()
-        if not torch_dir.is_dir():
-            raise SetupError(
-                f"Could not find requested user Torch installation: {torch_dir}"
-            )
-
-    # TF
-    if use_tf and libtf_dir:
-        libtf_dir = Path(libtf_dir).resolve()
-        if not libtf_dir.is_dir():
-            raise SetupError(
-                f"Could not find requested user TF installation: {libtf_dir}"
-            )
-
-    build_env_dict = build_env()
-
-    rai_builder = builder.RedisAIBuilder(
-        build_env=build_env_dict,
-        jobs=build_env.JOBS,
-        _os=builder.OperatingSystem.from_str(platform.system()),
-        architecture=builder.Architecture.from_str(platform.machine()),
-        torch_dir=str(torch_dir) if torch_dir else "",
-        libtf_dir=str(libtf_dir) if libtf_dir else "",
-        build_torch=use_torch,
-        build_tf=use_tf,
-        build_onnx=use_onnx,
-        verbose=verbose,
-        torch_with_mkl=torch_with_mkl,
-    )
-
-    if rai_builder.is_built:
-        logger.info("RedisAI installed. Run `smart clean` to remove.")
-    else:
-        # get the build environment, update with CUDNN env vars
-        # if present and building for GPU, otherwise warn the user
-        if device == Device.GPU:
-            gpu_env = build_env.get_cudnn_env()
-            cudnn_env_vars = [
-                "CUDNN_LIBRARY",
-                "CUDNN_INCLUDE_DIR",
-                "CUDNN_INCLUDE_PATH",
-                "CUDNN_LIBRARY_PATH",
-            ]
-            if not gpu_env:
-                logger.warning(
-                    "CUDNN environment variables not found.\n"
-                    f"Looked for {cudnn_env_vars}"
-                )
-            else:
-                build_env_dict.update(gpu_env)
-        # update RAI build env with cudnn env vars
-        rai_builder.env = build_env_dict
-
-        logger.info(
-            f"Building RedisAI version {versions.REDISAI}"
-            f" from {versions.REDISAI_URL}"
-        )
-
-        # NOTE: have the option to add other builds here in the future
-        # like "from_tarball"
-        rai_builder.build_from_git(
-            versions.REDISAI_URL, versions.REDISAI_BRANCH, device
-        )
-        logger.info("ML Backends and RedisAI build complete!")
-
 
 def check_py_torch_version(versions: Versioner, device: Device = Device.CPU) -> None:
     """Check Python environment for TensorFlow installation"""
@@ -360,16 +236,8 @@ def _format_incompatible_python_env_message(
 
 
 def _configure_keydb_build(versions: Versioner) -> None:
-    """Configure the redis versions to be used during the build operation"""
-    versions.REDIS = Version_("6.2.0")
-    versions.REDIS_URL = "https://github.com/EQ-Alpha/KeyDB"
-    versions.REDIS_BRANCH = "v6.2.0"
-
-    CONFIG.conf_path = Path(CONFIG.core_path, "config", "keydb.conf")
-    if not CONFIG.conf_path.resolve().is_file():
-        raise SSConfigError(
-            "Database configuration file at REDIS_CONF could not be found"
-        )
+    """Configure the feature store versions to be used during the build operation"""
+    ...
 
 
 # pylint: disable-next=too-many-statements
@@ -403,7 +271,7 @@ def execute(
         _configure_keydb_build(versions)
 
     if verbose:
-        fs_name: DbEngine = "KEYDB" if keydb else "REDIS"
+        fs_name: DbEngine = "KEYDB" if keydb else None
         logger.info("Version Information:")
         vers = versions.as_dict(fs_name=fs_name)
         version_names = list(vers.keys())
@@ -422,27 +290,15 @@ def execute(
 
     try:
         if not args.only_python_packages:
-            # REDIS/KeyDB
+            # KeyDB
             build_feature_store(build_env, versions, keydb, verbose)
 
-            # REDISAI
-            build_redis_ai(
-                build_env,
-                versions,
-                device,
-                pt,
-                tf,
-                onnx,
-                args.torch_dir,
-                args.libtensorflow_dir,
-                verbose=verbose,
-                torch_with_mkl=args.torch_with_mkl,
-            )
     except (SetupError, BuildError) as e:
         logger.error(str(e))
         return os.EX_SOFTWARE
-
-    backends = installed_redisai_backends()
+
+    # TODO: Add new backend 
+    backends = None
     backends_str = ", ".join(s.capitalize() for s in backends) if backends else "No"
     logger.info(f"{backends_str} backend(s) built")
 
@@ -523,7 +379,7 @@ def configure_parser(parser: argparse.ArgumentParser) -> None:
         "--keydb",
         action="store_true",
         default=False,
-        help="Build KeyDB instead of Redis",
+        help="Build KeyDB",
     )
     parser.add_argument(
         "--no_torch_with_mkl",

smartsim/_core/_cli/cli.py

@@ -103,12 +103,12 @@ def register_menu_items(self, menu_items: t.List[MenuItemConfig]) -> None:
         for item in menu_items:
             self._register_menu_item(item)
 
-
+# TODO: Add new path for dbcli
 def default_cli() -> SmartCli:
     menu = [
         MenuItemConfig(
             "build",
-            "Build SmartSim dependencies (Redis, RedisAI, Dragon, ML runtimes)",
+            "Build SmartSim dependencies (Dragon, ML runtimes)",
             build_execute,
             build_parser,
         ),
@@ -120,7 +120,7 @@ def default_cli() -> SmartCli:
         ),
         MenuItemConfig(
             "dbcli",
-            "Print the path to the redis-cli binary",
+            "Print the path to the -cli binary",
             dbcli_execute,
         ),
         MenuItemConfig(

smartsim/_core/_cli/info.py

@@ -21,7 +21,6 @@ def execute(
         tabulate(
             [
                 ["SmartSim", _fmt_py_pkg_version("smartsim")],
-                ["SmartRedis", _fmt_py_pkg_version("smartredis")],
             ],
             headers=["Name", "Version"],
             tablefmt="fancy_outline",
@@ -36,15 +35,9 @@ def execute(
         fs_table.append(["Location", str(fs_path)])
     print(tabulate(fs_table, tablefmt="fancy_outline"), end="\n\n")
 
-    print("Redis AI Configuration:")
-    rai_path = _helpers.redis_install_base().parent / "redisai.so"
-    rai_table = [["Status", _fmt_installed_redis_ai(rai_path)]]
-    if rai_path.is_file():
-        rai_table.append(["Location", str(rai_path)])
-    print(tabulate(rai_table, tablefmt="fancy_outline"), end="\n\n")
-
     print("Machine Learning Backends:")
-    backends = _helpers.installed_redisai_backends()
+    #TODO: add a backend
+    backends = None
     print(
         tabulate(
             [
@@ -79,12 +72,6 @@ def _fmt_installed_fs(fs_path: t.Optional[pathlib.Path]) -> str:
     return _helpers.colorize(fs_name.upper(), "green")
 
 
-def _fmt_installed_redis_ai(rai_path: pathlib.Path) -> str:
-    if not rai_path.is_file():
-        return _MISSING_DEP
-    return _helpers.colorize("Installed", "green")
-
-
 def _fmt_py_pkg_version(pkg_name: str) -> str:
     try:
         return _helpers.colorize(_BuildEnv.get_py_package_version(pkg_name), "green")

smartsim/_core/_cli/scripts/dragon_install.py

@@ -182,7 +182,7 @@ def install_package(asset_dir: pathlib.Path) -> int:
         logger.info(f"Installing package: {wheel_path.absolute()}")
 
         try:
-            pip("install", "--force-reinstall", str(wheel_path))
+            pip("install", "--force-reinstall", str(wheel_path), "numpy<2")
             wheel_path = next(wheels, None)
         except Exception:
             logger.error(f"Unable to install from {asset_dir}")

smartsim/_core/_cli/utils.py

@@ -91,20 +91,14 @@ def clean(core_path: Path, _all: bool = False) -> int:
 
     lib_path = core_path / "lib"
     if lib_path.is_dir():
-        # remove RedisAI
-        rai_path = lib_path / "redisai.so"
-        if rai_path.is_file():
-            rai_path.unlink()
-            logger.info("Successfully removed existing RedisAI installation")
-
         backend_path = lib_path / "backends"
         if backend_path.is_dir():
             shutil.rmtree(backend_path, ignore_errors=True)
             logger.info("Successfully removed ML runtimes")
 
     bin_path = core_path / "bin"
     if bin_path.is_dir() and _all:
-        files_to_remove = ["redis-server", "redis-cli", "keydb-server", "keydb-cli"]
+        files_to_remove = ["keydb-server", "keydb-cli"]
         removed = False
         for _file in files_to_remove:
             file_path = bin_path.joinpath(_file)
@@ -121,7 +115,7 @@ def clean(core_path: Path, _all: bool = False) -> int:
 def get_fs_path() -> t.Optional[Path]:
     bin_path = get_install_path() / "_core" / "bin"
     for option in bin_path.iterdir():
-        if option.name in ("redis-cli", "keydb-cli"):
+        if option.name in ("keydb-cli"):
             return option
     return None