diff --git a/.devcontainer/conda/Dockerfile b/.devcontainer/conda/Dockerfile new file mode 100644 index 000000000..d1ffbce77 --- /dev/null +++ b/.devcontainer/conda/Dockerfile @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base + +ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin" diff --git a/.devcontainer/conda/devcontainer.json b/.devcontainer/conda/devcontainer.json new file mode 100644 index 000000000..219f6748b --- /dev/null +++ b/.devcontainer/conda/devcontainer.json @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +{ + "name": "mrc-dev", + "build": { + "dockerfile": "Dockerfile" + }, + "hostRequirements": { + "gpu": true + }, + "capAdd":[ + "SYS_NICE", + "SYS_PTRACE" + ], + "securityOpt": [ + "seccomp=unconfined" + ], + "runArgs": [ + "--network=host" + ], + "containerEnv": { + "HOST_MRC_ROOT": "${localWorkspaceFolder}", + "MRC_ROOT": "${containerWorkspaceFolder}", + "DEFAULT_CONDA_ENV": "mrc", + "MAMBA_NO_BANNER": "1", + "VAULT_HOST": "https://vault.ops.k8s.rapids.ai" + }, + "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/conda/initialize-command.sh" ], + "remoteUser": "coder", + "mounts": [ + { + "type": "bind", + "source": "${localWorkspaceFolder}/.cache/conda/envs", + "target": "/home/coder/.conda/envs" + }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/../.conda/pkgs", + "target": "/home/coder/.conda/pkgs" + }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs + "target": "/home/coder/.config" + }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/.devcontainer/opt/mrc", + "target": "/opt/mrc" + }, + ], + "features": { + "ghcr.io/devcontainers/features/dotnet:1": {} + }, + "customizations": { + "vscode": { + "extensions": [ + "eamodio.gitlens", + "formulahendry.terminal", + "hbenl.vscode-test-explorer", + "josetr.cmake-language-support-vscode", + "llvm-vs-code-extensions.vscode-clangd", + "matepek.vscode-catch2-test-adapter", + "ms-dotnettools.vscode-dotnet-runtime", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.cpptools-themes", + "ms-vscode.cpptools", + "ms-vscode.test-adapter-converter", + "xaver.clang-format" + ], + "settings": { + "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake", + "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet", + "C_Cpp.intelliSenseEngine": "disabled", + "python.terminal.activateEnvironment": false + } + } + }, +} diff --git a/.devcontainer/conda/initialize-command.sh b/.devcontainer/conda/initialize-command.sh new file mode 100755 index 000000000..807451923 --- /dev/null +++ b/.devcontainer/conda/initialize-command.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# create the parent conda folder so it's found when mounting +mkdir -p ../.conda + +# create a config directory if it does not exist so it's found when mounting +mkdir -p ../.config diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f5244494a..3c3973d03 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -50,13 +50,11 @@ "hostRequirements": { "gpu": true }, - "initializeCommand": [ "./.devcontainer/init.sh" ], + "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ], "mounts": [ + "source=${localEnv:HOME}/.config,target=/home/${localEnv:USER}/.config,type=bind", // Mount the necessary files for GDB pretty-printing to work - "source=${localWorkspaceFolder}/.devcontainer/home/.gdbinit,target=/home/${localEnv:USER}/.gdbinit,type=bind", - "source=${localWorkspaceFolder}/.devcontainer/home/.config/gdb,target=/home/${localEnv:USER}/.config/gdb,type=bind", - // Mount the Github CLI config directory to allow using the Github CLI - "source=${localEnv:HOME}/.config/gh,target=/home/${localEnv:USER}/.config/gh,type=bind", + "source=${localWorkspaceFolder}/.devcontainer/opt/mrc/etc/.gdbinit,target=/home/${localEnv:USER}/.gdbinit,type=bind", ], "features": { "ghcr.io/devcontainers/features/git-lfs": {} diff --git a/.devcontainer/home/.gdbinit b/.devcontainer/home/.gdbinit deleted file mode 100644 index 43e108353..000000000 --- a/.devcontainer/home/.gdbinit +++ /dev/null @@ -1,3 +0,0 @@ -add-auto-load-safe-path /opt/conda - -source ~/.config/gdb/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/initialize-command.sh b/.devcontainer/initialize-command.sh new file mode 100755 index 000000000..807451923 --- /dev/null +++ b/.devcontainer/initialize-command.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# create the parent conda folder so it's found when mounting +mkdir -p ../.conda + +# create a config directory if it does not exist so it's found when mounting +mkdir -p ../.config diff --git a/.devcontainer/opt/mrc/bin/post-attach-command.sh b/.devcontainer/opt/mrc/bin/post-attach-command.sh new file mode 100755 index 000000000..eb00a5061 --- /dev/null +++ b/.devcontainer/opt/mrc/bin/post-attach-command.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ensure our ~/.config directory has the correct permissions. If ~/.config did +# not exist, and you mount ~/.config/gh from the host, then ~/.config will be +# created with root permissions which can break things + +conda_env_find(){ + conda env list | grep "${@}" >/dev/null 2>/dev/null +} + +ENV_NAME=${ENV_NAME:-mrc} + +sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc; + +if conda_env_find "${ENV_NAME}" ; \ + +then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml --prune; \ +else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml; \ +fi diff --git a/.devcontainer/opt/mrc/bin/update-content-command.sh b/.devcontainer/opt/mrc/bin/update-content-command.sh new file mode 100755 index 000000000..2e025db79 --- /dev/null +++ b/.devcontainer/opt/mrc/bin/update-content-command.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ensure our ~/.config directory has the correct permissions. If ~/.config did +# not exist, and you mount ~/.config/gh from the host, then ~/.config will be +# created with root permissions which can break things + +if [[ ! -f ~/.config/.gdbinit ]]; then + cp /opt/mrc/etc/.gdbinit ~/.config/.gdbinit +fi diff --git a/.devcontainer/opt/mrc/conda/Dockerfile b/.devcontainer/opt/mrc/conda/Dockerfile new file mode 100644 index 000000000..d1ffbce77 --- /dev/null +++ b/.devcontainer/opt/mrc/conda/Dockerfile @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base + +ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin" diff --git a/.devcontainer/opt/mrc/conda/devcontainer.json b/.devcontainer/opt/mrc/conda/devcontainer.json new file mode 100644 index 000000000..6046d60a1 --- /dev/null +++ b/.devcontainer/opt/mrc/conda/devcontainer.json @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +{ + "name": "mrc-dev", + "build": { + "dockerfile": "Dockerfile" + }, + "hostRequirements": { + "gpu": true + }, + "capAdd":[ + "SYS_NICE", + "SYS_PTRACE" + ], + "securityOpt": [ + "seccomp=unconfined" + ], + "runArgs": [ + "--network=host" + ], + "containerEnv": { + "HOST_MRC_ROOT": "${localWorkspaceFolder}", + "MRC_ROOT": "${containerWorkspaceFolder}", + "DEFAULT_CONDA_ENV": "mrc", + "MAMBA_NO_BANNER": "1", + "VAULT_HOST": "https://vault.ops.k8s.rapids.ai" + }, + "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ], + "remoteUser": "coder", + "mounts": [ + { + "type": "bind", + "source": "${localWorkspaceFolder}/.cache/conda/envs", + "target": "/home/coder/.conda/envs" + }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/../.conda/pkgs", + "target": "/home/coder/.conda/pkgs" + }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs + "target": "/home/coder/.config" + }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/.devcontainer/opt/mrc", + "target": "/opt/mrc" + }, + ], + "features": { + "ghcr.io/devcontainers/features/dotnet:1": {} + }, + "customizations": { + "vscode": { + "extensions": [ + "eamodio.gitlens", + "formulahendry.terminal", + "hbenl.vscode-test-explorer", + "josetr.cmake-language-support-vscode", + "llvm-vs-code-extensions.vscode-clangd", + "matepek.vscode-catch2-test-adapter", + "ms-dotnettools.vscode-dotnet-runtime", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.cpptools-themes", + "ms-vscode.cpptools", + "ms-vscode.test-adapter-converter", + "xaver.clang-format" + ], + "settings": { + "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake", + "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet", + "C_Cpp.intelliSenseEngine": "disabled", + "python.terminal.activateEnvironment": false + } + } + }, +} diff --git a/.devcontainer/opt/mrc/conda/initialize-command.sh b/.devcontainer/opt/mrc/conda/initialize-command.sh new file mode 100755 index 000000000..807451923 --- /dev/null +++ b/.devcontainer/opt/mrc/conda/initialize-command.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# create the parent conda folder so it's found when mounting +mkdir -p ../.conda + +# create a config directory if it does not exist so it's found when mounting +mkdir -p ../.config diff --git a/.devcontainer/opt/mrc/etc/.gdbinit b/.devcontainer/opt/mrc/etc/.gdbinit new file mode 100644 index 000000000..0fa2be45f --- /dev/null +++ b/.devcontainer/opt/mrc/etc/.gdbinit @@ -0,0 +1,3 @@ +add-auto-load-safe-path /opt/conda + +source /opt/mrc/etc/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py b/.devcontainer/opt/mrc/etc/enable_conda_libstd_pretty_print.py similarity index 100% rename from .devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py rename to .devcontainer/opt/mrc/etc/enable_conda_libstd_pretty_print.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f9db2db1..b437ba321 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,41 +77,41 @@ Some options: #### Clone MRC repository ```bash -export MRC_HOME=$(pwd)/mrc -git clone --recurse-submodules git@github.com:nv-morpheus/mrc.git $MRC_HOME -cd $MRC_HOME +export MRC_ROOT=$(pwd)/mrc +git clone --recurse-submodules git@github.com:nv-morpheus/mrc.git $MRC_ROOT +cd $MRC_ROOT ``` #### Create MRC Conda environment ```bash # note: `mamba` may be used in place of `conda` for better performance. -conda env create -n mrc --file $MRC_HOME/ci/conda/environments/dev_env.yml +conda env create -n mrc --file $MRC_ROOT/ci/conda/environments/dev_env.yml conda activate mrc ``` #### Build MRC ```bash -mkdir $MRC_HOME/build -cd $MRC_HOME/build +mkdir $MRC_ROOT/build +cd $MRC_ROOT/build cmake .. make -j $(nproc) ``` #### Run MRC C++ Tests ```bash -export MRC_TEST_INTERNAL_DATA_PATH=$MRC_HOME/src/tests -$MRC_HOME/build/src/tests/test_mrc_private.x -$MRC_HOME/build/tests/test_mrc.x -$MRC_HOME/build/tests/logging/test_mrc_logging.x +export MRC_TEST_INTERNAL_DATA_PATH=$MRC_ROOT/cpp/mrc/src/tests +$MRC_ROOT/build/cpp/mrcsrc/tests/test_mrc_private.x +$MRC_ROOT/build/cpp/mrctests/test_mrc.x +$MRC_ROOT/build/cpp/mrctests/logging/test_mrc_logging.x ``` ### Install MRC Python ```bash -pip install -e $MRC_HOME/build/python +pip install -e $MRC_ROOT/build/python ``` #### Run MRC Python Tests ```bash -pytest $MRC_HOME/python +pytest $MRC_ROOT/python ``` ### Building API Documentation diff --git a/README.md b/README.md index c7180c15f..0f05e754a 100644 --- a/README.md +++ b/README.md @@ -110,46 +110,46 @@ Installing via source is for more advanced users and is necessary to try MRC fea #### Clone MRC repository ```bash -export MRC_HOME=$(pwd)/mrc -git clone --recurse-submodules git@github.com:nv-morpheus/mrc.git $MRC_HOME -cd $MRC_HOME +export MRC_ROOT=$(pwd)/mrc +git clone --recurse-submodules git@github.com:nv-morpheus/mrc.git $MRC_ROOT +cd $MRC_ROOT ``` #### Create MRC Conda Environment ```bash # note: `mamba` may be used in place of `conda` for better performance. -conda env create -n mrc-dev --file $MRC_HOME/ci/conda/environments/dev_env.yml +conda env create -n mrc-dev --file $MRC_ROOT/ci/conda/environments/dev_env.yml conda activate mrc-dev ``` #### Build MRC ```bash -mkdir $MRC_HOME/build -cd $MRC_HOME/build +mkdir $MRC_ROOT/build +cd $MRC_ROOT/build cmake .. make -j $(nproc) ``` #### Run MRC C++ Tests ```bash -export MRC_TEST_INTERNAL_DATA_PATH=$MRC_HOME/src/tests -$MRC_HOME/build/src/tests/test_mrc_private.x -$MRC_HOME/build/tests/test_mrc.x -$MRC_HOME/build/tests/logging/test_mrc_logging.x +export MRC_TEST_INTERNAL_DATA_PATH=$MRC_ROOT/cpp/mrc/src/tests +$MRC_ROOT/build/cpp/mrc/src/tests/test_mrc_private.x +$MRC_ROOT/build/cpp/mrc/tests/test_mrc.x +$MRC_ROOT/build/cpp/mrc/tests/logging/test_mrc_logging.x ``` #### Install MRC Python Bindings ```bash -pip install -e $MRC_HOME/build/python +pip install -e $MRC_ROOT/build/python ``` #### Run MRC Python Tests ```bash -pytest $MRC_HOME/python +pytest $MRC_ROOT/python ``` ### Docker Installation -A Dockerfile is provided at `$MRC_HOME` and can be built with +A Dockerfile is provided at `$MRC_ROOT` and can be built with ```bash docker build -t mrc:latest . ``` diff --git a/ci/conda/README.md b/ci/conda/README.md index afef29543..56e6045fb 100644 --- a/ci/conda/README.md +++ b/ci/conda/README.md @@ -3,7 +3,7 @@ To build the Conda packages, it's recommended to run the provided scripts from a docker container. To build the container, `mrc-conda-build`, run the following: ```bash -cd ${MRC_HOME} +cd ${MRC_ROOT} docker buildx build --target developement -t mrc-conda-build . ``` @@ -20,10 +20,10 @@ docker run --rm -ti -v $PWD:/work \ mrc-conda-build ./ci/conda/recipes/run_conda_build.sh ``` -This will save the conda packages to `${MRC_HOME}/.conda-bld`. To install from this location, use the following: +This will save the conda packages to `${MRC_ROOT}/.conda-bld`. To install from this location, use the following: ```bash -conda install -c file://${MRC_HOME}/.conda-bld mrc +conda install -c file://${MRC_ROOT}/.conda-bld mrc ``` ## Uploading the Conda Package diff --git a/docs/quickstart/README.md b/docs/quickstart/README.md index ac6f01b99..1000ae9d8 100644 --- a/docs/quickstart/README.md +++ b/docs/quickstart/README.md @@ -23,7 +23,7 @@ If you've alredy followed the MRC library installation instructions in the [main ```bash # Change directory to the quickstart root -cd ${MRC_HOME}/docs/quickstart/ +cd ${MRC_ROOT}/docs/quickstart/ # Update the existing MRC conda environment. Here, we assume the environment is named `mrc`, but you can change this to the environment name you used, if different conda env update -n mrc -f environment_cpp.yml @@ -39,7 +39,7 @@ pip install -e python If you haven't installed MRC or would like to create an entirely new Conda environment for the Quick Start Guide, run: ```bash # Change directory to the quickstart root -cd ${MRC_HOME}/docs/quickstart/ +cd ${MRC_ROOT}/docs/quickstart/ # Create a new MRC conda environment. Here, we assume the environment is named `mrc-quickstart` conda env create -n mrc-quickstart -f environment_cpp.yml @@ -62,7 +62,7 @@ If you encounter errors building the examples, this is mostly likely caused for A comprehensive overview of building MRC from source is provided in the [MRC README](../../README.md#source-installation). To build the Quick Start Guide examples, simply run ```bash -cd ${MRC_HOME}/docs/quickstart +cd ${MRC_ROOT}/docs/quickstart # Build the QSG including the C++ examples ./compile.sh