From 9e893d0f6acdf43821910cfb94cfc892bb773fde Mon Sep 17 00:00:00 2001
From: Anuradha Karuppiah <anuradhak@nvidia.com>
Date: Tue, 27 Aug 2024 16:42:22 +0000
Subject: [PATCH 1/5] draft: add a recipe for creating the morpheus_core conda
 package

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>
---
 .../morpheus_core/conda_build_config.yaml     |  32 +++
 ci/conda/recipes/morpheus_core/meta.yaml      | 140 ++++++++++++
 .../morpheus_core/morpheus_core_build.sh      |  88 ++++++++
 ci/conda/recipes/run_conda_core_build.sh      | 117 ++++++++++
 ci/scripts/github/common_core.sh              | 205 ++++++++++++++++++
 ci/scripts/github/conda_core.sh               |  61 ++++++
 ci/scripts/run_ci_local.sh                    |   2 +-
 7 files changed, 644 insertions(+), 1 deletion(-)
 create mode 100644 ci/conda/recipes/morpheus_core/conda_build_config.yaml
 create mode 100644 ci/conda/recipes/morpheus_core/meta.yaml
 create mode 100644 ci/conda/recipes/morpheus_core/morpheus_core_build.sh
 create mode 100755 ci/conda/recipes/run_conda_core_build.sh
 create mode 100755 ci/scripts/github/common_core.sh
 create mode 100755 ci/scripts/github/conda_core.sh

diff --git a/ci/conda/recipes/morpheus_core/conda_build_config.yaml b/ci/conda/recipes/morpheus_core/conda_build_config.yaml
new file mode 100644
index 0000000000..4b051dc074
--- /dev/null
+++ b/ci/conda/recipes/morpheus_core/conda_build_config.yaml
@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+c_compiler_version:
+  - 11.2
+
+cxx_compiler_version:
+  - 11.2
+
+cuda_compiler:
+  - cuda-nvcc
+
+cuda_compiler_version:
+  - 12.1
+
+python:
+  - 3.10
+
+rapids_version:
+  - 24.02
diff --git a/ci/conda/recipes/morpheus_core/meta.yaml b/ci/conda/recipes/morpheus_core/meta.yaml
new file mode 100644
index 0000000000..0fca54dcb1
--- /dev/null
+++ b/ci/conda/recipes/morpheus_core/meta.yaml
@@ -0,0 +1,140 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
+{% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
+
+package:
+  name: morpheus_core
+  version: {{ version }}
+
+source:
+  git_url: ../../../..
+
+outputs:
+
+  - name: morpheus_core
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+      script_env:
+        # Note: Setting a default value here will overwrite the value from the
+        # shell! Defaults should be set in the run_conda_build.sh script
+        - CMAKE_CUDA_ARCHITECTURES
+        - MORPHEUS_CACHE_DIR
+        - MORPHEUS_PYTHON_BUILD_STUBS
+        - MORPHEUS_SUPPORT_DOCA
+        - MORPHEUS_BUILD_MORPHEUS_LLM
+        - PARALLEL_LEVEL
+      run_exports:
+        - {{ pin_subpackage("morpheus_core", max_pin="x.x") }}
+    script: morpheus_core_build.sh
+
+    requirements:
+      build:
+        - {{ compiler("c") }}
+        - {{ compiler("cuda") }}
+        - {{ compiler("cxx") }}
+        - automake =1.16.5 # Needed for DOCA build
+        - ccache
+        - cmake =3.27
+        - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application
+        - cuda-version {{ cuda_compiler_version }}.*
+        - libtool # Needed for DOCA build
+        - ninja =1.11
+        - pkg-config =0.29 # for mrc cmake
+        - sysroot_linux-64 =2.17
+      host:
+        # CUDA dependencies
+        - cuda-cudart-dev {{ cuda_compiler_version }}.*
+        - cuda-libraries-dev {{ cuda_compiler_version }}.* # required by matx for libcublas-dev
+        - cuda-nvml-dev {{ cuda_compiler_version }}.*
+        - cuda-nvrtc-dev {{ cuda_compiler_version }}.*
+        - cuda-nvtx-dev {{ cuda_compiler_version }}.*
+        - cuda-version {{ cuda_compiler_version }}.*
+        # Non-CUDA dependencies
+        - cudf {{ rapids_version }}
+        - cython 3.0.*
+        - glog 0.6.*
+        - libcudf {{ rapids_version }}
+        - librdkafka >=1.9.2,<1.10.0a0
+        - mrc {{ minor_version }}
+        - nlohmann_json 3.11.*
+        - pip
+        - pybind11-stubgen 0.10.5
+        - python {{ python }}
+        - rapidjson 1.1.0
+        - scikit-build 0.17.6
+        - versioneer-518
+        - zlib 1.2.13 # required to build triton client
+      run:
+        # Runtime only requirements. This + setup.py is the definitive runtime requirement list
+        # This should be synced with `runtime` in dependencies.yaml
+        - appdirs
+        - beautifulsoup4
+        - click >=8
+        - cuda-version {{ cuda_compiler_version }}.*
+        - cudf
+        - cupy # Version determined from cudf
+        - datacompy =0.10.*
+        - dill =0.3.7
+        - docker-py =5.0.*
+        - elasticsearch ==8.9.0
+        - feedparser =6.0.*
+        - grpcio =1.59.*
+        - mlflow>=2.10.0,<3
+        - mrc
+        - networkx=2.8.8
+        - numpydoc =1.5.*
+        - pydantic
+        - pluggy =1.3.*
+        - python
+        - python-confluent-kafka >=1.9.2,<1.10.0a0
+        - python-graphviz
+        - pytorch-cuda
+        - pytorch * *cuda*
+        - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed
+        - requests
+        - requests-cache =1.1.*
+        - scikit-learn =1.3.2.*
+        - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3
+        - tqdm =4.*
+        - tritonclient =2.34.*
+        - typing_utils =0.1.*
+        - watchdog =3.0.*
+        - websockets
+    test:
+      requires:
+        # test that cuml can be installed in the env
+        - cuml {{ rapids_version }}.*
+        - pynvml
+        - pytest-asyncio
+        - pytest-benchmark
+        - pytest-cov
+        - pytest =7.4.4
+      source_files:
+        - pyproject.toml
+        - scripts/fetch_data.py
+      imports:
+        - morpheus
+      commands:
+          - echo # make sure test requirements get installed
+
+about:
+  home: https://github.com/nv-morpheus/Morpheus
+  license: Apache-2.0
+  license_family: Apache
+  license_file: LICENSE
+  summary: Morpheus Cybersecurity Library
diff --git a/ci/conda/recipes/morpheus_core/morpheus_core_build.sh b/ci/conda/recipes/morpheus_core/morpheus_core_build.sh
new file mode 100644
index 0000000000..5ef4920b9e
--- /dev/null
+++ b/ci/conda/recipes/morpheus_core/morpheus_core_build.sh
@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It is assumed that this script is executed from the root of the repo directory by conda-build
+# (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake)
+
+# Need to ensure this value is set before checking it in the if block
+MORPHEUS_SUPPORT_DOCA=-OFF
+MORPHEUS_BUILD_MORPHEUS_LLM=-OFF
+
+# This will store all of the cmake args. Make sure to prepend args to allow
+# incoming values to overwrite them
+CMAKE_ARGS=${CMAKE_ARGS:-""}
+
+export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
+export USE_SCCACHE=${USE_SCCACHE:-""}
+
+# Check for some mrc environment variables. Append to front of args to allow users to overwrite them
+if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
+   # Set the cache variable, then set the Staging prefix to allow for host searching
+   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
+
+   # Double check that the cache dir has been created
+   mkdir -p ${MORPHEUS_CACHE_DIR}
+fi
+
+CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
+CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS=-"ON"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"RAPIDS"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11
+CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}"
+
+if [[ "${USE_SCCACHE}" == "1" ]]; then
+   CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"
+fi
+
+echo "CC          : ${CC}"
+echo "CXX         : ${CXX}"
+echo "CUDAHOSTCXX : ${CUDAHOSTCXX}"
+echo "CUDA        : ${CUDA}"
+echo "CMAKE_ARGS  : ${CMAKE_ARGS}"
+
+echo "========Begin Env========"
+env
+echo "========End Env========"
+
+BUILD_DIR="build-conda"
+
+# Check if the build directory already exists. And if so, delete the
+# CMakeCache.txt and CMakeFiles to ensure a clean configuration
+if [[ -d "./${BUILD_DIR}" ]]; then
+   echo "Deleting old CMake files at ./${BUILD_DIR}"
+   rm -rf "./${BUILD_DIR}/CMakeCache.txt"
+   rm -rf "./${BUILD_DIR}/CMakeFiles"
+fi
+
+# Run configure
+cmake -B ${BUILD_DIR} \
+   ${CMAKE_ARGS} \
+   --log-level=verbose \
+   .
+
+# Build the components
+cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
+
+# Install just the mprpheus core python wheel components
+${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl
diff --git a/ci/conda/recipes/run_conda_core_build.sh b/ci/conda/recipes/run_conda_core_build.sh
new file mode 100755
index 0000000000..b9e51ead36
--- /dev/null
+++ b/ci/conda/recipes/run_conda_core_build.sh
@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+NUMARGS=$#
+ARGS=$*
+
+function hasArg {
+   (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
+function get_version() {
+   echo "$(git describe --tags | grep -o -E '^([^-]*?)')"
+}
+
+# Color variables
+export b="\033[0;36m"
+export g="\033[0;32m"
+export r="\033[0;31m"
+export e="\033[0;90m"
+export y="\033[0;33m"
+export x="\033[0m"
+
+# Ensure yes is always selected otherwise it can stop halfway through on a prelink message
+export CONDA_ALWAYS_YES=true
+
+# Change this to switch between build/mambabuild/debug
+export CONDA_COMMAND=${CONDA_COMMAND:-"mambabuild"}
+
+# Get the path to the morpheus git folder
+export MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)}
+
+# Export script_env variables that must be set for conda build
+export CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:-"RAPIDS"}
+export MORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS:-"ON"}
+export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"}
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}
+export MORPHEUS_SUPPORT_DOCA=-OFF
+export MORPHEUS_BUILD_MORPHEUS_LLM=-OFF
+
+# Set CONDA_CHANNEL_ALIAS to mimic the conda config channel_alias property during the build
+CONDA_CHANNEL_ALIAS=${CONDA_CHANNEL_ALIAS:-""}
+export USE_SCCACHE=${USE_SCCACHE:-""}
+
+export CMAKE_GENERATOR="Ninja"
+
+# Export variables for the cache
+export CCACHE_DIR="${MORPHEUS_CACHE_DIR}/ccache"
+export CCACHE_NOHASHDIR=1
+
+# Ensure the necessary folders exist before continuing
+mkdir -p ${MORPHEUS_CACHE_DIR}
+mkdir -p ${CCACHE_DIR}
+
+# Local builds use ccache
+# ci builds will use sccache which is a ccache work-alike but uses an S3 backend
+# (https://github.com/mozilla/sccache)
+if [[ "${USE_SCCACHE}" == "" ]]; then
+   # Export CCACHE variables
+   export CMAKE_C_COMPILER_LAUNCHER="ccache"
+   export CMAKE_CXX_COMPILER_LAUNCHER="ccache"
+   export CMAKE_CUDA_COMPILER_LAUNCHER="ccache"
+else
+   export CMAKE_C_COMPILER_LAUNCHER="sccache"
+   export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
+   export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
+fi
+
+# Holds the arguments in an array to allow for complex json objects
+CONDA_ARGS_ARRAY=()
+
+# Some default args
+CONDA_ARGS_ARRAY+=("--use-local")
+
+if [[ "${CONDA_COMMAND}" == "mambabuild" || "${CONDA_COMMAND}" == "build" ]]; then
+   # Remove the timestamp from the work folder to allow caching to work better
+   CONDA_ARGS_ARRAY+=("--build-id-pat" "{n}-{v}")
+fi
+
+# And default channels (should match dependencies.yaml) with optional channel alias
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}conda-forge")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}rapidsai")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}rapidsai-nightly")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/dev")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}pytorch")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}defaults")
+
+if [[ ${NUMARGS} == 0 ]]; then
+   echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}"
+   echo -e "${r}   morpheus_core${x}"
+   echo -e "${r}Exiting...${x}"
+   exit 12
+fi
+
+if hasArg morpheus_core; then
+   # Set GIT_VERSION to set the project version inside of meta.yaml
+   export GIT_VERSION="$(get_version)"
+
+   echo "Running conda-build for morpheus_core v${GIT_VERSION}..."
+   set -x
+   conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ci/conda/recipes/morpheus_core
+   set +x
+fi
diff --git a/ci/scripts/github/common_core.sh b/ci/scripts/github/common_core.sh
new file mode 100755
index 0000000000..b018cca22e
--- /dev/null
+++ b/ci/scripts/github/common_core.sh
@@ -0,0 +1,205 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function print_env_vars() {
+    rapids-logger "Environ:"
+    env | grep -v -E "AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|GH_TOKEN|NGC_API_KEY" | sort
+}
+
+rapids-logger "Env Setup"
+print_env_vars
+rapids-logger "---------"
+mkdir -p ${WORKSPACE_TMP}
+source /opt/conda/etc/profile.d/conda.sh
+export MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)}
+cd ${MORPHEUS_ROOT}
+conda activate morpheus
+
+# For non-gpu hosts nproc will correctly report the number of cores we are able to use
+# On a GPU host however nproc will report the total number of cores and PARALLEL_LEVEL
+# will be defined specifying the subset we are allowed to use.
+NUM_CORES=$(nproc)
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-${NUM_CORES}}
+rapids-logger "Procs: ${NUM_CORES}"
+/usr/bin/lscpu
+
+rapids-logger "Memory"
+/usr/bin/free -g
+
+rapids-logger "User Info"
+id
+
+# S3 vars
+export S3_URL="s3://rapids-downloads/ci/morpheus"
+export DISPLAY_URL="https://downloads.rapids.ai/ci/morpheus"
+export ARTIFACT_ENDPOINT="/pull-request/${PR_NUM}/${GIT_COMMIT}/${NVARCH}"
+export ARTIFACT_URL="${S3_URL}${ARTIFACT_ENDPOINT}"
+
+if [[ "${LOCAL_CI}" == "1" ]]; then
+    export DISPLAY_ARTIFACT_URL="${LOCAL_CI_TMP}"
+else
+    export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}"
+fi
+
+# Set sccache env vars
+export SCCACHE_S3_KEY_PREFIX=morpheus-${NVARCH}
+export SCCACHE_BUCKET=rapids-sccache-east
+export SCCACHE_REGION="us-east-2"
+export SCCACHE_IDLE_TIMEOUT=32768
+#export SCCACHE_LOG=debug
+
+# Set the build flags
+export BUILD_DIR=${BUILD_DIR:-build}
+
+# Turn OFF examples, tests, benchmarks, doca, llm here. They may be on in the runner-docker env
+_FLAGS=()
+_FLAGS+=("-B" "${BUILD_DIR}")
+_FLAGS+=("-G" "Ninja")
+_FLAGS+=("-DCMAKE_MESSAGE_CONTEXT_SHOW=ON")
+_FLAGS+=("-DMORPHEUS_CUDA_ARCHITECTURES=RAPIDS")
+_FLAGS+=("-DMORPHEUS_USE_CONDA=ON")
+_FLAGS+=("-DMORPHEUS_USE_CCACHE=ON")
+_FLAGS+=("-DMORPHEUS_PYTHON_INPLACE_BUILD=OFF")
+_FLAGS+=("-DMORPHEUS_PYTHON_BUILD_STUBS=ON")
+if [[ "${LOCAL_CI}" == "" ]]; then
+    _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)")
+fi
+export CMAKE_BUILD_ALL_FEATURES="${_FLAGS[@]}"
+unset _FLAGS
+
+export FETCH_STATUS=0
+
+print_env_vars
+
+function update_conda_env() {
+    # Deactivate the environment first before updating
+    conda deactivate
+
+
+    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
+        rapids-logger "Checking for updates to conda env"
+
+
+        # Update the packages
+        # use conda instead of mamba due to bug: https://github.com/mamba-org/mamba/issues/3059
+        rapids-conda-retry env update -n morpheus --prune -q --file "$1" --solver=libmamba
+    fi
+
+    # Finally, reactivate
+    conda activate morpheus
+
+    rapids-logger "Final Conda Environment"
+    show_conda_info
+}
+
+function fetch_base_branch_gh_api() {
+    # For PRs, $GIT_BRANCH is like: pull-request/989
+    REPO_NAME=$(basename "${GITHUB_REPOSITORY}")
+    ORG_NAME="${GITHUB_REPOSITORY_OWNER}"
+    PR_NUM="${GITHUB_REF_NAME##*/}"
+
+    rapids-logger "Retrieving base branch from GitHub API"
+    [[ -n "$GH_TOKEN" ]] && CURL_HEADERS=('-H' "Authorization: token ${GH_TOKEN}")
+    RESP=$(
+    curl -s \
+        -H "Accept: application/vnd.github.v3+json" \
+        "${CURL_HEADERS[@]}" \
+        "${GITHUB_API_URL}/repos/${ORG_NAME}/${REPO_NAME}/pulls/${PR_NUM}"
+    )
+
+    export BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref')
+
+    # Change target is the branch name we are merging into but due to the weird way jenkins does
+    # the checkout it isn't recognized by git without the origin/ prefix
+    export CHANGE_TARGET="origin/${BASE_BRANCH}"
+}
+
+function fetch_base_branch_local() {
+    rapids-logger "Retrieving base branch from git"
+    if [[ "${USE_HOST_GIT}" == "0" ]]; then
+        git remote add upstream ${GIT_UPSTREAM_URL}
+        git fetch upstream --tags
+    fi
+
+    source ${MORPHEUS_ROOT}/ci/scripts/common.sh
+    export BASE_BRANCH=$(get_base_branch)
+    export CHANGE_TARGET="upstream/${BASE_BRANCH}"
+}
+
+function fetch_base_branch() {
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        fetch_base_branch_local
+    else
+        fetch_base_branch_gh_api
+    fi
+
+    rapids-logger "Base branch: ${BASE_BRANCH}"
+}
+
+function show_conda_info() {
+
+    rapids-logger "Check Conda info"
+    conda info
+    conda config --show-sources
+    conda list --show-channel-urls
+}
+
+function log_toolchain() {
+    rapids-logger "Check versions"
+    python3 --version
+    x86_64-conda-linux-gnu-cc --version
+    x86_64-conda-linux-gnu-c++ --version
+    cmake --version
+    ninja --version
+    sccache --version
+}
+
+function log_sccache_stats() {
+    if [[ "${LOCAL_CI}" == "" ]]; then
+        rapids-logger "sccache usage for morpheus build:"
+        sccache --show-stats
+    fi
+}
+
+function upload_artifact() {
+    FILE_NAME=$1
+    BASE_NAME=$(basename "${FILE_NAME}")
+    rapids-logger "Uploading artifact: ${BASE_NAME}"
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        cp ${FILE_NAME} "${LOCAL_CI_TMP}/${BASE_NAME}"
+    else
+        aws s3 cp --only-show-errors "${FILE_NAME}" "${ARTIFACT_URL}/${BASE_NAME}"
+        echo "- ${DISPLAY_ARTIFACT_URL}/${BASE_NAME}" >> ${GITHUB_STEP_SUMMARY}
+    fi
+}
+
+function download_artifact() {
+    ARTIFACT=$1
+    rapids-logger "Downloading ${ARTIFACT} from ${DISPLAY_ARTIFACT_URL}"
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        cp "${LOCAL_CI_TMP}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
+    else
+        aws s3 cp --only-show-errors "${ARTIFACT_URL}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
+    fi
+}
+
+function set_job_summary_preamble() {
+    if [[ "${LOCAL_CI}" == "" ]]; then
+        msg="Note: NVIDIA VPN access is required to view these URLs."
+        echo $msg >> ${GITHUB_STEP_SUMMARY}
+        rapids-logger $msg
+    fi
+}
diff --git a/ci/scripts/github/conda_core.sh b/ci/scripts/github/conda_core.sh
new file mode 100755
index 0000000000..3991ce948d
--- /dev/null
+++ b/ci/scripts/github/conda_core.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CI_SCRIPT_ARGS="$@"
+source ${WORKSPACE}/ci/scripts/github/common_core.sh
+
+cd ${MORPHEUS_ROOT}
+
+fetch_base_branch
+
+# Its important that we are in the base environment for the build
+rapids-logger "Activating Base Conda Environment"
+
+# Deactivate any extra environments (There can be a few on the stack)
+while [[ "${CONDA_SHLVL:-0}" -gt 1 ]]; do
+   echo "Deactivating conda environment ${CONDA_DEFAULT_ENV}"
+   conda deactivate
+done
+
+# Ensure at least base is activated
+if [[ "${CONDA_DEFAULT_ENV}" != "base" ]]; then
+   echo "Activating base conda environment"
+   conda activate base
+fi
+
+# Print the info just to be sure base is active
+conda info
+
+rapids-logger "Building Conda Package morpheus_core"
+
+# Run the conda build, and upload to conda forge if requested
+export MORPHEUS_PYTHON_BUILD_STUBS=OFF
+export CONDA_ARGS="--skip-existing"
+${MORPHEUS_ROOT}/ci/conda/recipes/run_conda_core_build.sh morpheus_core "${CI_SCRIPT_ARGS}"
+
+# If we didn't receive the upload argument, upload the artifact to S3
+if [[ " ${CI_SCRIPT_ARGS} " =~ " upload " ]]; then
+   rapids-logger "Building Conda Package morpheus_core... Done"
+else
+   # if we didn't receive the upload argument, we can still upload the artifact to S3
+   tar cfj "${WORKSPACE_TMP}/conda_morpheus_core.tar.bz" "${RAPIDS_CONDA_BLD_OUTPUT_DIR}"
+   ls -lh ${WORKSPACE_TMP}/
+
+   rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
+   upload_artifact "${WORKSPACE_TMP}/conda_morpheus_core.tar.bz"
+fi
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 1b6816f5d1..7bf1d3d0eb 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -23,7 +23,7 @@ case "$1" in
     "all" )
         STAGES=("checks" "build" "docs" "test" "conda")
         ;;
-    "checks" | "build" | "docs" | "test" | "conda" | "bash" )
+    "checks" | "build" | "docs" | "test" | "conda" | "conda_core" | "bash" )
         STAGES=("$1")
         ;;
     * )

From e3cc22eb99a1ec4c3d8e9901efc5cc56814a3087 Mon Sep 17 00:00:00 2001
From: Anuradha Karuppiah <anuradhak@nvidia.com>
Date: Wed, 28 Aug 2024 00:05:46 +0000
Subject: [PATCH 2/5] fix mrc flag import failure

This is a workaround to the following error -
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
CMake Error at
build-conda/_deps/rapids-cmake-src/rapids-cmake/cmake/make_global.cmake:59
(set_target_properties):^M
  Attempt to promote imported target "gflags" to global scope (by
setting^M
  IMPORTED_GLOBAL) which is not built in this directory.^M
Call Stack (most recent call first):^M
  build-conda/_deps/rapids-cmake-src/rapids-cmake/find/package.cmake:129
(rapids_cmake_make_global)^M
  /Morpheus/.cache/cpm/mrc/9f8a50df0cf94cf470a5cdc01861ed63f701e27f/cmake/dependencies.cmake:53
(rapids_find_package)^M
  /Morpheus/.cache/cpm/mrc/9f8a50df0cf94cf470a5cdc01861ed63f701e27f/CMakeLists.txt:148
(include)
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>
---
 cmake/dependencies.cmake | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake
index 5bc821773e..2b59d30593 100644
--- a/cmake/dependencies.cmake
+++ b/cmake/dependencies.cmake
@@ -51,6 +51,14 @@ if(MORPHEUS_BUILD_BENCHMARKS)
   )
 endif()
 
+# gflags
+# ======
+rapids_find_package(gflags REQUIRED
+  GLOBAL_TARGETS gflags
+  BUILD_EXPORT_SET ${PROJECT_NAME}-exports
+  INSTALL_EXPORT_SET ${PROJECT_NAME}-exports
+)
+
 # glog
 # ====
 morpheus_utils_configure_glog()

From e0c31ba5d5fd3d756296845816cb652f16b40e2e Mon Sep 17 00:00:00 2001
From: Anuradha Karuppiah <anuradhak@nvidia.com>
Date: Thu, 29 Aug 2024 03:27:36 +0000
Subject: [PATCH 3/5] dependencies: add a morpheus_core key

This include all non-pip runtime dependencies

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>
---
 .../morpheus_core_cuda-121_arch-x86_64.yaml   | 50 +++++++++++++++++++
 dependencies.yaml                             | 23 +++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 conda/environments/morpheus_core_cuda-121_arch-x86_64.yaml

diff --git a/conda/environments/morpheus_core_cuda-121_arch-x86_64.yaml b/conda/environments/morpheus_core_cuda-121_arch-x86_64.yaml
new file mode 100644
index 0000000000..5a276c89a6
--- /dev/null
+++ b/conda/environments/morpheus_core_cuda-121_arch-x86_64.yaml
@@ -0,0 +1,50 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- conda-forge
+- huggingface
+- rapidsai
+- rapidsai-nightly
+- nvidia
+- nvidia/label/dev
+- pytorch
+dependencies:
+- appdirs
+- beautifulsoup4=4.12
+- click>=8
+- cuda-cudart=12.1
+- cuda-nvrtc=12.1
+- cuda-nvtx=12.1
+- cuda-version=12.1
+- cudf=24.02
+- cupy
+- datacompy=0.10
+- dill=0.3.7
+- docker-py=5.0
+- elasticsearch==8.9.0
+- feedparser=6.0
+- grpcio-status=1.59
+- grpcio=1.59
+- mlflow>=2.10.0,<3
+- mrc=24.10
+- networkx=2.8.8
+- numpydoc=1.5
+- pip
+- pluggy=1.3
+- pydantic
+- python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-graphviz
+- python=3.10
+- pytorch-cuda
+- pytorch=*=*cuda*
+- rapids-dask-dependency=24.02
+- requests
+- requests-cache=1.1
+- scikit-learn=1.3.2
+- sqlalchemy<2.0
+- tqdm=4
+- tritonclient=2.34
+- typing_utils=0.1
+- watchdog=3.0
+- websockets
+name: morpheus_core_cuda-121_arch-x86_64
diff --git a/dependencies.yaml b/dependencies.yaml
index e8f5525696..0d0fc02afa 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -37,6 +37,7 @@ files:
       - example-llms
       - python
       - runtime
+      - runtime-pip
       - test_python_morpheus
 
   # Common dependencies for developing with Morpheus.
@@ -59,6 +60,7 @@ files:
       - docs
       - python
       - runtime
+      - runtime-pip
       - test_python_morpheus
 
   # Dependencies that are needed to build Morpheus.
@@ -79,6 +81,7 @@ files:
       - development
       - python
       - runtime
+      - runtime-pip
       - test_python_morpheus
 
   # Dependencies that are needed to run Morpheus tests.
@@ -100,6 +103,7 @@ files:
       - example-llms
       - python
       - runtime
+      - runtime-pip
       - test_python_morpheus
 
   # Dependencies that are needed to generate the Morpheus documentation.
@@ -123,6 +127,7 @@ files:
       - example-llms
       - python
       - runtime
+      - runtime-pip
 
   # Only the dependencies that are needed to run core Morpheus. This is the smallest set of dependencies needed to run
   # a Morpheus pipeline.
@@ -138,6 +143,17 @@ files:
       - cve-mitigation
       - python
       - runtime
+      - runtime-pip
+
+  morpheus_core:
+    output: conda
+    matrix:
+      cuda: ["12.1"]
+      arch: [x86_64]
+    includes:
+      - cudatoolkit
+      - python
+      - runtime
 
   # Dependencies that are needed to run all Morpheus examples.
   # Includes: runtime
@@ -154,6 +170,7 @@ files:
       - example-llms
       - python
       - runtime
+      - runtime-pip
 
   # Dependencies which are needed to run the model generation scripts in the `models` directory.
   # Includes: none
@@ -313,6 +330,7 @@ dependencies:
   # Runtime dependencies for Morpheus. Included in nearly all output files so dependencies should
   # be added only if it is needed to run the core Morpheus library.
   # This should be synced with `run` in ci/conda/recipes/morpheus/meta.yaml
+  # skip pip packages
   runtime:
     common:
       - output_types: [conda]
@@ -356,6 +374,11 @@ dependencies:
           - watchdog=3.0
           - websockets
           - pip
+
+  runtime-pip:
+    common:
+      - output_types: [conda]
+        packages:
           - pip:
             - databricks-cli < 0.100
             - databricks-connect

From ba7e397014eeea31aa9aaa17c15c1905c58566e4 Mon Sep 17 00:00:00 2001
From: Anuradha Karuppiah <anuradhak@nvidia.com>
Date: Wed, 4 Sep 2024 17:32:23 +0000
Subject: [PATCH 4/5] Add a docker stage for building the morpheus core conda
 package

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>
---
 docker/Dockerfile | 44 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4f69391258..f6b65027a0 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -16,8 +16,8 @@
 # limitations under the License.
 
 # ============== LEGEND =================================================== #
-#                         [conda_bld_morpheus]
-#                               /
+#       [conda_bld_morpheus_core] [conda_bld_morpheus]
+#               /                  /
 # [base] -> [conda_env] -> [base_extended] -> [runtime_conda_create] -> [runtime]
 #                               \
 #                         [conda_env_dev] -> [development] -> [development_pydbg]
@@ -149,6 +149,46 @@ ENTRYPOINT [ "/opt/conda/bin/tini", "--", "/workspace/docker/entrypoint.sh" ]
 # Reset the shell back to normal
 SHELL ["/bin/bash", "-c"]
 
+# ============ Stage: conda_bld_morpheus_core ============
+# Now build the morpheus conda package
+FROM conda_env as conda_bld_morpheus_core
+
+ARG MORPHEUS_ROOT_HOST
+ARG CUDA_MAJOR_VER
+ARG CUDA_MINOR_VER
+
+ENV GIT_CLONE_PROTECTION_ACTIVE=false
+
+RUN --mount=type=cache,id=apt,target=/var/cache/apt \
+    apt update && \
+    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
+    apt install --no-install-recommends -y \
+        cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        cuda-cudart-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy the source
+COPY . ./
+
+RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locked \
+    --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
+    # Install git-lfs before running the build to avoid errors during conda build
+    /opt/conda/bin/mamba install -y -n base -c conda-forge "git-lfs" &&\
+    source activate base &&\
+    git lfs install &&\
+    # Need to get around recent versions of git locking paths until they are deemed safe
+    git config --global --add safe.directory "*" &&\
+    # Change to the morpheus directory and build the conda package
+    cd ${MORPHEUS_ROOT_HOST} &&\
+    MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld ./ci/conda/recipes/run_conda_core_build.sh morpheus_core
+
 # ============ Stage: base_extended ============
 # Base environment with any optional extensions. ex. DOCA libs
 FROM conda_env as base_extended

From 14aafbe9709e3aa6b2d0da4f135bb3655ebe66fa Mon Sep 17 00:00:00 2001
From: Anuradha Karuppiah <anuradhak@nvidia.com>
Date: Wed, 4 Sep 2024 20:29:01 +0000
Subject: [PATCH 5/5] Add an option to upload the package to anaconda

Upload is done using the CONDA_TOKEN env var

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>
---
 ci/conda/recipes/run_conda_core_build.sh | 25 ++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ci/conda/recipes/run_conda_core_build.sh b/ci/conda/recipes/run_conda_core_build.sh
index b9e51ead36..5c3c230ac9 100755
--- a/ci/conda/recipes/run_conda_core_build.sh
+++ b/ci/conda/recipes/run_conda_core_build.sh
@@ -82,6 +82,31 @@ fi
 # Holds the arguments in an array to allow for complex json objects
 CONDA_ARGS_ARRAY=()
 
+if hasArg upload; then
+   # Set the conda token
+   CONDA_TOKEN=${CONDA_TOKEN:?"CONDA_TOKEN must be set to allow upload"}
+
+   # Get the label to apply to the package
+   CONDA_PKG_LABEL=${CONDA_PKG_LABEL:-"dev"}
+
+   # Ensure we have anaconda-client installed for upload
+   if [[ -z "$(conda list | grep anaconda-client)" ]]; then
+      echo -e "${y}anaconda-client not found and is required for up. Installing...${x}"
+
+      mamba install -y anaconda-client
+   fi
+
+   echo -e "${y}Uploading morpheus_core conda package${x}"
+
+   # Add the conda token needed for uploading
+   CONDA_ARGS_ARRAY+=("--token" "${CONDA_TOKEN}")
+
+   if [[ -n "${CONDA_PKG_LABEL}" ]]; then
+      CONDA_ARGS_ARRAY+=("--label" "${CONDA_PKG_LABEL}")
+      echo -e "${y}   Using label: ${CONDA_PKG_LABEL}${x}"
+   fi
+fi
+
 # Some default args
 CONDA_ARGS_ARRAY+=("--use-local")