Use dependencies.yaml as source-of-truth for environment files. (#1441)

Adds rapids-dependency-file-generator pre-commit checks for local and CI, and updates scripts to use the generated environment files. Contributes to #1389 ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - Christopher Harris (https://github.com/cwharris) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: #1441
nv-morpheus · Dec 21, 2023 · b49ff8d · b49ff8d
1 parent 0bd9e48
commit b49ff8d
Show file tree

Hide file tree

Showing 18 changed files with 499 additions and 61 deletions.
diff --git a/.devcontainer/opt/morpheus/bin/post-attach-command.sh b/.devcontainer/opt/morpheus/bin/post-attach-command.sh
@@ -27,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus}
 sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
-then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml; \
+then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml; \
 fi
diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml
@@ -132,7 +132,6 @@ jobs:
       env:
         NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
         PARALLEL_LEVEL: '10'
-        MERGE_EXAMPLES_YAML: '1'
     strategy:
       fail-fast: true
 
@@ -164,8 +163,6 @@ jobs:
         username: '$oauthtoken'
         password: ${{ secrets.NGC_API_KEY }}
       image: ${{ inputs.container }}
-      env:
-        MERGE_DOCS_YAML: '1'
     strategy:
       fail-fast: true
 

diff --git a/.github/workflows/pull_request.yml → .github/workflows/pr.yaml b/.github/workflows/pull_request.yml → .github/workflows/pr.yaml
@@ -42,6 +42,15 @@ permissions:
   statuses: none
 
 jobs:
+
+  pr-builder:
+    needs:
+      - prepare
+      - checks
+      - ci_pipe
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+
   prepare:
     # Executes the get-pr-info action to determine if the PR has the skip-ci label, if the action fails we assume the
     # PR does not have the label
@@ -57,11 +66,18 @@ jobs:
     outputs:
       has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }}
 
+  checks:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    # Only run the CI pipeline if the PR does not have the skip-ci label
+    if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }}
+    with:
+      enable_check_generated_files: false
+
   ci_pipe:
     name: CI Pipeline
     needs: [prepare]
     uses: ./.github/workflows/ci_pipe.yml
-
     # Only run the CI pipeline if the PR does not have the skip-ci label
     if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }}
     with:

diff --git a/.gitmodules b/.gitmodules
@@ -2,7 +2,7 @@
 	path = external/morpheus-visualizations
 	url = https://github.com/nv-morpheus/morpheus-visualizations.git
 	branch = branch-24.03
-[submodule "morpheus_utils"]
+[submodule "external/utilities"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
 	branch = branch-24.03
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+repos:
+      - repo: https://github.com/rapidsai/dependency-file-generator
+        rev: v1.7.1
+        hooks:
+            - id: rapids-dependency-file-generator
+              args: ["--clean"]
+
+default_language_version:
+      python: python3
diff --git a/ci/check_style.sh b/ci/check_style.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+rapids-logger "Create checks conda environment"
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key checks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n checks
+conda activate checks
+
+# Run pre-commit checks
+pre-commit run --all-files --show-diff-on-failure
diff --git a/ci/conda/recipes/python-dbg/meta.yaml b/ci/conda/recipes/python-dbg/meta.yaml
@@ -16,7 +16,7 @@
 {% set version = environ.get('MORPHEUS_PYTHON_VER') %}
 
 package:
-  name: python
+  name: morpheus-python-dbg
   version: {{ version }}
 
 source:

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -77,8 +77,8 @@ fi
 # Root CMakeLists.txt
 sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt
 
-# Conda environment file
-sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" docker/conda/environments/cuda11.8_dev.yml
+# Depedencies file
+sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" dependencies.yaml
 
 # examples/digital_fingerprinting
 sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" \

diff --git a/ci/scripts/copyright.py b/ci/scripts/copyright.py
@@ -48,7 +48,8 @@
     re.compile(r"^[^ \/\n]*build[^ \/\n]*\/.*$"),  # Ignore any build*/ folder
     re.compile(r"^external\/.*$"),  # Ignore external
     re.compile(r"[^ \/\n]*docs/source/(_lib|_modules|_templates)/.*$"),
-    re.compile(r"PULL_REQUEST_TEMPLATE.md"),  # Ignore the PR template
+    re.compile(r"PULL_REQUEST_TEMPLATE.md"),  # Ignore the PR template,
+    re.compile(r"[^ \/\n]*conda/environments/.*\.yaml$"),  # Ignore generated environment files
 ]
 
 # this will break starting at year 10000, which is probably OK :)

diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh
@@ -18,7 +18,12 @@ set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key build \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 log_toolchain
 

diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh
@@ -18,7 +18,12 @@ set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key build \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 log_toolchain
 

diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh
@@ -61,10 +61,6 @@ export SCCACHE_REGION="us-east-2"
 export SCCACHE_IDLE_TIMEOUT=32768
 #export SCCACHE_LOG=debug
 
-export CONDA_ENV_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml
-export CONDA_EXAMPLES_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_examples.yml
-export CONDA_DOCS_YML=${MORPHEUS_ROOT}/docs/conda_docs.yml
-
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
 
 export FETCH_STATUS=0
@@ -75,31 +71,10 @@ function update_conda_env() {
     # Deactivate the environment first before updating
     conda deactivate
 
-    ENV_YAML=${CONDA_ENV_YML}
-    if [[ "${MERGE_EXAMPLES_YAML}" == "1" || "${MERGE_DOCS_YAML}" == "1" ]]; then
-        # Merge the dev, docs and examples envs, otherwise --prune will remove the examples packages
-        ENV_YAML=${WORKSPACE_TMP}/merged_env.yml
-        YAMLS="${CONDA_ENV_YML}"
-        if [[ "${MERGE_EXAMPLES_YAML}" == "1" ]]; then
-            YAMLS="${YAMLS} ${CONDA_EXAMPLES_YML}"
-        fi
-        if [[ "${MERGE_DOCS_YAML}" == "1" ]]; then
-            YAMLS="${YAMLS} ${CONDA_DOCS_YML}"
-        fi
-
-        rapids-logger "Merging conda envs: ${YAMLS}"
-        conda run -n morpheus --live-stream conda-merge ${YAMLS} > ${ENV_YAML}
-    fi
-
-    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
-        rapids-logger "Checking for updates to conda env"
+    rapids-logger "Checking for updates to conda env"
 
-        # Remove default/conflicting channels from base image
-        rm /opt/conda/.condarc
-
-        # Update the packages
-        rapids-mamba-retry env update -n morpheus --prune -q --file ${ENV_YAML}
-    fi
+    # Update the packages
+    rapids-mamba-retry env update -n morpheus --prune -q --file "$1"
 
     # Finally, reactivate
     conda activate morpheus
@@ -132,6 +107,7 @@ function fetch_base_branch_gh_api() {
 
 function fetch_base_branch_local() {
     rapids-logger "Retrieving base branch from git"
+    git remote remove upstream
     git remote add upstream ${GIT_UPSTREAM_URL}
     git fetch upstream --tags
     source ${MORPHEUS_ROOT}/ci/scripts/common.sh

diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh
@@ -18,7 +18,12 @@ set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key docs \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 download_artifact "wheel.tar.bz"
 

diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh
@@ -19,7 +19,12 @@ set -e
 source ${WORKSPACE}/ci/scripts/github/common.sh
 /usr/bin/nvidia-smi
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key test \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 rapids-logger "Check versions"
 python3 --version
@@ -69,8 +74,8 @@ rapids-logger "Pulling LFS assets"
 git lfs install
 ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation
 
-# List missing files
-rapids-logger "Listing missing files"
+# Listing LFS-known files
+rapids-logger "Listing LFS-known files"
 git lfs ls-files
 
 REPORTS_DIR="${WORKSPACE_TMP}/reports"

diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -exo pipefail
+
 case "$1" in
     "" )
         STAGES=("bash")
@@ -72,16 +74,12 @@ for STAGE in "${STAGES[@]}"; do
     DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}"
     if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then
         CONTAINER="${TEST_CONTAINER}"
-        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all"
-        if [[ "${STAGE}" == "test" ]]; then
-            DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_EXAMPLES_YAML=1 --cap-add=sys_nice"
-        fi
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia"
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --gpus all"
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice"
     else
         CONTAINER="${BUILD_CONTAINER}"
         DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc"
-        if [[ "${STAGE}" == "docs" ]]; then
-            DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_DOCS_YAML=1"
-        fi
     fi
 
     if [[ "${STAGE}" == "bash" ]]; then