diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml index 34dfbd38cd..01e15b7e4c 100644 --- a/.github/workflows/ci_pipe.yml +++ b/.github/workflows/ci_pipe.yml @@ -21,14 +21,18 @@ on: run_check: required: true type: boolean - conda_run_build: - description: 'Runs the conda-build stage to ensure the conda package builds successfully' + conda_core_run_build: + description: 'Runs the stage to build the morpheus-core conda package' required: true type: boolean conda_upload_label: - description: 'The label to use when uploading the conda package. Leave empty to disable uploading' + description: 'The label to use when uploading the morpheus conda packages. Leave empty to disable uploading' required: true type: string + conda_run_build: + description: 'Runs the conda-build stage to build the conda package with all morpheus components' + required: true + type: boolean container: required: true type: string @@ -36,6 +40,8 @@ on: required: true type: string secrets: + CONDA_TOKEN: + required: true NGC_API_KEY: required: true @@ -198,9 +204,9 @@ jobs: shell: bash run: ./morpheus/ci/scripts/github/docs.sh - package: - name: Package - if: ${{ inputs.conda_run_build }} + package-core: + name: Package Core + if: ${{ inputs.conda_core_run_build }} needs: [check, documentation, test] runs-on: linux-amd64-cpu16 timeout-minutes: 60 @@ -228,10 +234,44 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-duration-seconds: 43200 # 12h - - name: conda + - name: Build morpheus-core conda package shell: bash env: CONDA_TOKEN: "${{ secrets.CONDA_TOKEN }}" SCRIPT_ARGS: "${{ inputs.conda_upload_label != '' && 'upload' || '' }}" CONDA_PKG_LABEL: "${{ inputs.conda_upload_label }}" - run: ./morpheus/ci/scripts/github/conda.sh $SCRIPT_ARGS + run: ./morpheus/ci/scripts/github/conda_core.sh $SCRIPT_ARGS + + package: + name: Package All + if: ${{ inputs.conda_run_build }} + needs: [check, documentation, test] + runs-on: linux-amd64-cpu16 + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.container }} + strategy: + fail-fast: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'morpheus' + fetch-depth: 0 + submodules: 'recursive' + + - name: Get AWS credentials using OIDC + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + + - name: conda + shell: bash + run: ./morpheus/ci/scripts/github/conda.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 42a261fdc0..d7c748c181 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -68,6 +68,7 @@ jobs: is_main_branch: ${{ github.ref_name == 'main' }} is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }} has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }} + has_conda_core_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-core-build') || false }} has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }} pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }} @@ -89,13 +90,16 @@ jobs: with: # Run checks for any PR branch run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }} - # Run conda-build for main/dev branches and PRs with the conda-build label + # Build morpheus-core conda package. This is done for main/dev branches and + # for PRs with the conda-core-build label + conda_core_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_core_build_label) }} + # Upload morpheus-core conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches + conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }} + # Run morpheus conda package, with all components. This is done for main/dev + # branches and for PRs with the conda-build label. conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }} - # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches - # Disable conda upload for now, once we have morpheus packages in conda forge set the value to - # !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' - conda_upload_label: "" container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240614 test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240614 secrets: + CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }} NGC_API_KEY: ${{ secrets.NGC_API_KEY }} diff --git a/ci/conda/recipes/morpheus-core/conda_build_config.yaml b/ci/conda/recipes/morpheus-core/conda_build_config.yaml new file mode 100644 index 0000000000..4b051dc074 --- /dev/null +++ b/ci/conda/recipes/morpheus-core/conda_build_config.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +c_compiler_version: + - 11.2 + +cxx_compiler_version: + - 11.2 + +cuda_compiler: + - cuda-nvcc + +cuda_compiler_version: + - 12.1 + +python: + - 3.10 + +rapids_version: + - 24.02 diff --git a/ci/conda/recipes/morpheus-core/meta.yaml b/ci/conda/recipes/morpheus-core/meta.yaml new file mode 100644 index 0000000000..f72004b95a --- /dev/null +++ b/ci/conda/recipes/morpheus-core/meta.yaml @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} + +package: + name: morpheus-core + version: {{ version }} + +source: + git_url: ../../../.. + +outputs: + + - name: morpheus-core + build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + # Note: Setting a default value here will overwrite the value from the + # shell! Defaults should be set in the run_conda_build.sh script + - CMAKE_CUDA_ARCHITECTURES + - MORPHEUS_CACHE_DIR + - MORPHEUS_PYTHON_BUILD_STUBS + - MORPHEUS_SUPPORT_DOCA + - MORPHEUS_BUILD_MORPHEUS_LLM + - PARALLEL_LEVEL + run_exports: + - {{ pin_subpackage("morpheus-core", max_pin="x.x") }} + script: morpheus_core_build.sh + + requirements: + build: + - {{ compiler("c") }} + - {{ compiler("cuda") }} + - {{ compiler("cxx") }} + - automake =1.16.5 # Needed for DOCA build + - ccache + - cmake =3.27 + - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application + - cuda-version {{ cuda_compiler_version }}.* + - libtool # Needed for DOCA build + - ninja =1.11 + - pkg-config =0.29 # for mrc cmake + - sysroot_linux-64 =2.17 + host: + # CUDA dependencies + - cuda-cudart-dev {{ cuda_compiler_version }}.* + - cuda-libraries-dev {{ cuda_compiler_version }}.* # required by matx for libcublas-dev + - cuda-nvml-dev {{ cuda_compiler_version }}.* + - cuda-nvrtc-dev {{ cuda_compiler_version }}.* + - cuda-nvtx-dev {{ cuda_compiler_version }}.* + - cuda-version {{ cuda_compiler_version }}.* + # Non-CUDA dependencies + - cudf {{ rapids_version }} + - cython 3.0.* + - glog 0.6.* + - libcudf {{ rapids_version }} + - librdkafka >=1.9.2,<1.10.0a0 + - mrc {{ minor_version }} + - nlohmann_json 3.11.* + - pip + - pybind11-stubgen 0.10.5 + - python {{ python }} + - rapidjson 1.1.0 + - scikit-build 0.17.6 + - versioneer-518 + - zlib 1.2.13 # required to build triton client + run: + # Runtime only requirements. This + setup.py is the definitive runtime requirement list + # This should be synced with `runtime` in dependencies.yaml + - appdirs + - beautifulsoup4 + - click >=8 + - cuda-version {{ cuda_compiler_version }}.* + - cudf + - cupy # Version determined from cudf + - datacompy =0.10.* + - dill =0.3.7 + - docker-py =5.0.* + - elasticsearch ==8.9.0 + - feedparser =6.0.* + - grpcio =1.59.* + - mlflow>=2.10.0,<3 + - mrc + - networkx=2.8.8 + - numpydoc =1.5.* + - pydantic + - pluggy =1.3.* + - python + - python-confluent-kafka >=1.9.2,<1.10.0a0 + - python-graphviz + - pytorch-cuda + - pytorch * *cuda* + - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed + - requests + - requests-cache =1.1.* + - scikit-learn =1.3.2.* + - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3 + - tqdm =4.* + - tritonclient =2.34.* + - typing_utils =0.1.* + - watchdog =3.0.* + - websockets + test: + requires: + # test that cuml can be installed in the env + - cuml {{ rapids_version }}.* + - pynvml + - pytest-asyncio + - pytest-benchmark + - pytest-cov + - pytest =7.4.4 + source_files: + - pyproject.toml + - scripts/fetch_data.py + imports: + - morpheus + commands: + - echo # make sure test requirements get installed + +about: + home: https://github.com/nv-morpheus/Morpheus + license: Apache-2.0 + license_family: Apache + license_file: LICENSE + summary: Morpheus Cybersecurity Core Library diff --git a/ci/conda/recipes/morpheus-core/morpheus_core_build.sh b/ci/conda/recipes/morpheus-core/morpheus_core_build.sh new file mode 100644 index 0000000000..5ef4920b9e --- /dev/null +++ b/ci/conda/recipes/morpheus-core/morpheus_core_build.sh @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# It is assumed that this script is executed from the root of the repo directory by conda-build +# (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake) + +# Need to ensure this value is set before checking it in the if block +MORPHEUS_SUPPORT_DOCA=-OFF +MORPHEUS_BUILD_MORPHEUS_LLM=-OFF + +# This will store all of the cmake args. Make sure to prepend args to allow +# incoming values to overwrite them +CMAKE_ARGS=${CMAKE_ARGS:-""} + +export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..) +export USE_SCCACHE=${USE_SCCACHE:-""} + +# Check for some mrc environment variables. Append to front of args to allow users to overwrite them +if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then + # Set the cache variable, then set the Staging prefix to allow for host searching + CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}" + + # Double check that the cache dir has been created + mkdir -p ${MORPHEUS_CACHE_DIR} +fi + +CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" +CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS=-"ON"} ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"RAPIDS"} ${CMAKE_ARGS}" +CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" +CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11 +CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}" + +if [[ "${USE_SCCACHE}" == "1" ]]; then + CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}" +fi + +echo "CC : ${CC}" +echo "CXX : ${CXX}" +echo "CUDAHOSTCXX : ${CUDAHOSTCXX}" +echo "CUDA : ${CUDA}" +echo "CMAKE_ARGS : ${CMAKE_ARGS}" + +echo "========Begin Env========" +env +echo "========End Env========" + +BUILD_DIR="build-conda" + +# Check if the build directory already exists. And if so, delete the +# CMakeCache.txt and CMakeFiles to ensure a clean configuration +if [[ -d "./${BUILD_DIR}" ]]; then + echo "Deleting old CMake files at ./${BUILD_DIR}" + rm -rf "./${BUILD_DIR}/CMakeCache.txt" + rm -rf "./${BUILD_DIR}/CMakeFiles" +fi + +# Run configure +cmake -B ${BUILD_DIR} \ + ${CMAKE_ARGS} \ + --log-level=verbose \ + . + +# Build the components +cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install + +# Install just the mprpheus core python wheel components +${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 6b6923b735..3f96d70736 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -48,8 +48,6 @@ export CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:-"RAPIDS"} export MORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS:-"ON"} export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"} export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)} -export MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} -export MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON} # Set CONDA_CHANNEL_ALIAS to mimic the conda config channel_alias property during the build CONDA_CHANNEL_ALIAS=${CONDA_CHANNEL_ALIAS:-""} @@ -82,6 +80,31 @@ fi # Holds the arguments in an array to allow for complex json objects CONDA_ARGS_ARRAY=() +if hasArg upload; then + # Set the conda token + CONDA_TOKEN=${CONDA_TOKEN:?"CONDA_TOKEN must be set to allow upload"} + + # Get the label to apply to the package + CONDA_PKG_LABEL=${CONDA_PKG_LABEL:-"dev"} + + # Ensure we have anaconda-client installed for upload + if [[ -z "$(conda list | grep anaconda-client)" ]]; then + echo -e "${y}anaconda-client not found and is required for up. Installing...${x}" + + mamba install -y anaconda-client + fi + + echo -e "${y}Uploading conda package${x}" + + # Add the conda token needed for uploading + CONDA_ARGS_ARRAY+=("--token" "${CONDA_TOKEN}") + + if [[ -n "${CONDA_PKG_LABEL}" ]]; then + CONDA_ARGS_ARRAY+=("--label" "${CONDA_PKG_LABEL}") + echo -e "${y} Using label: ${CONDA_PKG_LABEL}${x}" + fi +fi + # Some default args CONDA_ARGS_ARRAY+=("--use-local") @@ -103,12 +126,15 @@ CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}defa if [[ ${NUMARGS} == 0 ]]; then echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}" echo -e "${r} morpheus${x}" + echo -e "${r} morpheus-core${x}" echo -e "${r} pydebug${x}" echo -e "${r}Exiting...${x}" exit 12 fi if hasArg morpheus; then + export MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} + export MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON} # Set GIT_VERSION to set the project version inside of meta.yaml export GIT_VERSION="$(get_version)" @@ -118,6 +144,18 @@ if hasArg morpheus; then set +x fi +if hasArg morpheus-core; then + export MORPHEUS_SUPPORT_DOCA=-OFF + export MORPHEUS_BUILD_MORPHEUS_LLM=-OFF + # Set GIT_VERSION to set the project version inside of meta.yaml + export GIT_VERSION="$(get_version)" + + echo "Running conda-build for morpheus-core v${GIT_VERSION}..." + set -x + conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ci/conda/recipes/morpheus-core + set +x +fi + if hasArg pydebug; then export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2) diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh index e959eb1040..7babe4a40a 100755 --- a/ci/scripts/github/build.sh +++ b/ci/scripts/github/build.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/cmake_all.sh rapids-dependency-file-generator \ --output conda \ diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index e9bfaa6752..6b6e579feb 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/cmake_all.sh rapids-dependency-file-generator \ --output conda \ diff --git a/ci/scripts/github/cmake_all.sh b/ci/scripts/github/cmake_all.sh new file mode 100644 index 0000000000..86b0d65dcd --- /dev/null +++ b/ci/scripts/github/cmake_all.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_FLAGS=() +_FLAGS+=("-B" "${BUILD_DIR}") +_FLAGS+=("-G" "Ninja") +_FLAGS+=("-DCMAKE_MESSAGE_CONTEXT_SHOW=ON") +_FLAGS+=("-DMORPHEUS_CUDA_ARCHITECTURES=RAPIDS") +_FLAGS+=("-DMORPHEUS_USE_CONDA=ON") +_FLAGS+=("-DMORPHEUS_USE_CCACHE=ON") +_FLAGS+=("-DMORPHEUS_PYTHON_INPLACE_BUILD=OFF") +_FLAGS+=("-DMORPHEUS_PYTHON_BUILD_STUBS=ON") +_FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=ON") +_FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=ON") +_FLAGS+=("-DMORPHEUS_BUILD_TESTS=ON") +_FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_LLM=ON") +if [[ "${LOCAL_CI}" == "" ]]; then + _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)") +fi +export CMAKE_BUILD_ALL_FEATURES="${_FLAGS[@]}" +unset _FLAGS + +if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then + export CMAKE_BUILD_ALL_FEATURES="${CMAKE_BUILD_ALL_FEATURES} -DMORPHEUS_SUPPORT_DOCA=ON" +fi diff --git a/ci/scripts/github/cmake_core.sh b/ci/scripts/github/cmake_core.sh new file mode 100644 index 0000000000..20d250e45c --- /dev/null +++ b/ci/scripts/github/cmake_core.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_FLAGS=() +_FLAGS+=("-B" "${BUILD_DIR}") +_FLAGS+=("-G" "Ninja") +_FLAGS+=("-DCMAKE_MESSAGE_CONTEXT_SHOW=ON") +_FLAGS+=("-DMORPHEUS_CUDA_ARCHITECTURES=RAPIDS") +_FLAGS+=("-DMORPHEUS_USE_CONDA=ON") +_FLAGS+=("-DMORPHEUS_USE_CCACHE=ON") +_FLAGS+=("-DMORPHEUS_PYTHON_INPLACE_BUILD=OFF") +_FLAGS+=("-DMORPHEUS_PYTHON_BUILD_STUBS=ON") +_FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=OFF") +_FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=OFF") +_FLAGS+=("-DMORPHEUS_BUILD_TESTS=OFF") +_FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF") +_FLAGS+=("-DMORPHEUS_SUPPORT_DOCA=OFF") +if [[ "${LOCAL_CI}" == "" ]]; then + _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)") +fi +export CMAKE_BUILD_ALL_FEATURES="${_FLAGS[@]}" +unset _FLAGS diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 66d44fdb8c..5d7cf3f698 100755 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -64,29 +64,6 @@ export SCCACHE_IDLE_TIMEOUT=32768 # Set the build flags export BUILD_DIR=${BUILD_DIR:-build} -_FLAGS=() -_FLAGS+=("-B" "${BUILD_DIR}") -_FLAGS+=("-G" "Ninja") -_FLAGS+=("-DCMAKE_MESSAGE_CONTEXT_SHOW=ON") -_FLAGS+=("-DMORPHEUS_CUDA_ARCHITECTURES=RAPIDS") -_FLAGS+=("-DMORPHEUS_USE_CONDA=ON") -_FLAGS+=("-DMORPHEUS_USE_CCACHE=ON") -_FLAGS+=("-DMORPHEUS_PYTHON_INPLACE_BUILD=OFF") -_FLAGS+=("-DMORPHEUS_PYTHON_BUILD_STUBS=ON") -_FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=ON") -_FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=ON") -_FLAGS+=("-DMORPHEUS_BUILD_TESTS=ON") -_FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_LLM=ON") -if [[ "${LOCAL_CI}" == "" ]]; then - _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)") -fi -export CMAKE_BUILD_ALL_FEATURES="${_FLAGS[@]}" -unset _FLAGS - -if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then - export CMAKE_BUILD_ALL_FEATURES="${CMAKE_BUILD_ALL_FEATURES} -DMORPHEUS_SUPPORT_DOCA=ON" -fi - export FETCH_STATUS=0 print_env_vars diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh index 4114bd9ab9..5be051fade 100755 --- a/ci/scripts/github/conda.sh +++ b/ci/scripts/github/conda.sh @@ -18,6 +18,7 @@ set -e CI_SCRIPT_ARGS="$@" source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/cmake_all.sh cd ${MORPHEUS_ROOT} diff --git a/ci/scripts/github/conda_core.sh b/ci/scripts/github/conda_core.sh new file mode 100755 index 0000000000..b2006b6162 --- /dev/null +++ b/ci/scripts/github/conda_core.sh @@ -0,0 +1,62 @@ +#!/usr/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CI_SCRIPT_ARGS="$@" +source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/cmake_core.sh + +cd ${MORPHEUS_ROOT} + +fetch_base_branch + +# Its important that we are in the base environment for the build +rapids-logger "Activating Base Conda Environment" + +# Deactivate any extra environments (There can be a few on the stack) +while [[ "${CONDA_SHLVL:-0}" -gt 1 ]]; do + echo "Deactivating conda environment ${CONDA_DEFAULT_ENV}" + conda deactivate +done + +# Ensure at least base is activated +if [[ "${CONDA_DEFAULT_ENV}" != "base" ]]; then + echo "Activating base conda environment" + conda activate base +fi + +# Print the info just to be sure base is active +conda info + +rapids-logger "Building Conda Package morpheus-core" + +# Run the conda build, and upload to conda forge if requested +export MORPHEUS_PYTHON_BUILD_STUBS=OFF +export CONDA_ARGS="--skip-existing" +${MORPHEUS_ROOT}/ci/conda/recipes/run_conda_build.sh morpheus-core "${CI_SCRIPT_ARGS}" + +# If we didn't receive the upload argument, upload the artifact to S3 +if [[ " ${CI_SCRIPT_ARGS} " =~ " upload " ]]; then + rapids-logger "Building Conda Package morpheus-core... Done" +else + # if we didn't receive the upload argument, we can still upload the artifact to S3 + tar cfj "${WORKSPACE_TMP}/conda_morpheus_core.tar.bz" "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" + ls -lh ${WORKSPACE_TMP}/ + + rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" + upload_artifact "${WORKSPACE_TMP}/conda_morpheus_core.tar.bz" +fi diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh index f771c34265..a0e2057fe8 100755 --- a/ci/scripts/github/docs.sh +++ b/ci/scripts/github/docs.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/cmake_all.sh rapids-dependency-file-generator \ --output conda \ diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh index 38aabd5cd5..2e3388e611 100755 --- a/ci/scripts/github/test.sh +++ b/ci/scripts/github/test.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/cmake_all.sh /usr/bin/nvidia-smi rapids-dependency-file-generator \ diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 1b6816f5d1..8f7df827da 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -21,13 +21,13 @@ case "$1" in STAGES=("bash") ;; "all" ) - STAGES=("checks" "build" "docs" "test" "conda") + STAGES=("checks" "build" "docs" "test" "conda_core" "conda") ;; - "checks" | "build" | "docs" | "test" | "conda" | "bash" ) + "checks" | "build" | "docs" | "test" | "conda_core" | "conda" | "bash" ) STAGES=("$1") ;; * ) - echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", \"docs\", \"test\", \"conda\", or \"bash\"" + echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", \"docs\", \"test\", \"conda_core\", \"conda\", or \"bash\"" exit 1 ;; esac diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 5bc821773e..2b59d30593 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -51,6 +51,14 @@ if(MORPHEUS_BUILD_BENCHMARKS) ) endif() +# gflags +# ====== +rapids_find_package(gflags REQUIRED + GLOBAL_TARGETS gflags + BUILD_EXPORT_SET ${PROJECT_NAME}-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-exports +) + # glog # ==== morpheus_utils_configure_glog() diff --git a/docker/Dockerfile b/docker/Dockerfile index 4f69391258..e9a51ecccf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,8 +16,8 @@ # limitations under the License. # ============== LEGEND =================================================== # -# [conda_bld_morpheus] -# / +# [conda_bld_morpheus_core] [conda_bld_morpheus] +# / / # [base] -> [conda_env] -> [base_extended] -> [runtime_conda_create] -> [runtime] # \ # [conda_env_dev] -> [development] -> [development_pydbg] @@ -149,6 +149,52 @@ ENTRYPOINT [ "/opt/conda/bin/tini", "--", "/workspace/docker/entrypoint.sh" ] # Reset the shell back to normal SHELL ["/bin/bash", "-c"] +# ============ Stage: conda_bld_morpheus_core ============ +# Now build the morpheus conda package +FROM conda_env as conda_bld_morpheus_core + +ARG MORPHEUS_ROOT_HOST +ARG CUDA_MAJOR_VER +ARG CUDA_MINOR_VER + +# to upload the image use - +# --build-arg CONDA_TOKEN= +# --build-arg CONDA_UPLOAD=upload +ARG CONDA_TOKEN +ARG CONDA_UPLOAD + +ENV GIT_CLONE_PROTECTION_ACTIVE=false + +RUN --mount=type=cache,id=apt,target=/var/cache/apt \ + apt update && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ + apt install --no-install-recommends -y \ + cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + cuda-cudart-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ + rm -rf /var/lib/apt/lists/* + +# Copy the source +COPY . ./ + +RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locked \ + --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ + # Install git-lfs before running the build to avoid errors during conda build + /opt/conda/bin/mamba install -y -n base -c conda-forge "git-lfs" &&\ + source activate base &&\ + git lfs install &&\ + # Need to get around recent versions of git locking paths until they are deemed safe + git config --global --add safe.directory "*" &&\ + # Change to the morpheus directory and build the conda package + cd ${MORPHEUS_ROOT_HOST} &&\ + MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_TOKEN=${CONDA_TOKEN} CONDA_BLD_PATH=/opt/conda/conda-bld ./ci/conda/recipes/run_conda_build.sh morpheus-core ${CONDA_UPLOAD} + # ============ Stage: base_extended ============ # Base environment with any optional extensions. ex. DOCA libs FROM conda_env as base_extended