From 47bb4384974af27326b5f3ea86fc539887f21d28 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 07:23:15 -0700
Subject: [PATCH 01/27] WIP: Adapting local CI scripts from morpheus

---
 ci/scripts/bootstrap_local_ci.sh |  42 ++++++++++++
 ci/scripts/github/common.sh      |  27 +++++++-
 ci/scripts/run_ci_local.sh       | 111 +++++++++++++++++++++++++++++++
 3 files changed, 178 insertions(+), 2 deletions(-)
 create mode 100755 ci/scripts/bootstrap_local_ci.sh
 create mode 100755 ci/scripts/run_ci_local.sh

diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh
new file mode 100755
index 000000000..e8ad0fb46
--- /dev/null
+++ b/ci/scripts/bootstrap_local_ci.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export WORKSPACE_TMP="$(pwd)/ws_tmp"
+mkdir -p ${WORKSPACE_TMP}
+git clone ${GIT_URL} mrc
+cd mrc/
+git checkout ${GIT_BRANCH}
+git pull
+git checkout ${GIT_COMMIT}
+
+export MRC_ROOT=$(pwd)
+export WORKSPACE=${MRC_ROOT}
+export LOCAL_CI=1
+unset CMAKE_CUDA_COMPILER_LAUNCHER
+unset CMAKE_CXX_COMPILER_LAUNCHER
+unset CMAKE_C_COMPILER_LAUNCHER
+
+if [[ "${STAGE}" =~ "build" || ]]; then
+    SCRIPT_NAME="build.sh"
+elif [[ "${STAGE}" =~ "test" || ]]; then
+    SCRIPT_NAME="test.sh"
+else
+    SCRIPT_NAME="${STAGE}.sh"
+fi
+
+if [[ "${STAGE}" != "bash" ]]; then
+    ${MRC_ROOT}/ci/scripts/github/${SCRIPT_NAME}
+fi
diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh
index 02684da2f..930a3d4fb 100644
--- a/ci/scripts/github/common.sh
+++ b/ci/scripts/github/common.sh
@@ -105,7 +105,12 @@ function update_conda_env() {
 
 print_env_vars
 
-function fetch_base_branch() {
+function fetch_base_branch_gh_api() {
+    # For PRs, $GIT_BRANCH is like: pull-request/989
+    REPO_NAME=$(basename "${GITHUB_REPOSITORY}")
+    ORG_NAME="${GITHUB_REPOSITORY_OWNER}"
+    PR_NUM="${GITHUB_REF_NAME##*/}"
+
     rapids-logger "Retrieving base branch from GitHub API"
     [[ -n "$GH_TOKEN" ]] && CURL_HEADERS=('-H' "Authorization: token ${GH_TOKEN}")
     RESP=$(
@@ -115,11 +120,29 @@ function fetch_base_branch() {
         "${GITHUB_API_URL}/repos/${ORG_NAME}/${REPO_NAME}/pulls/${PR_NUM}"
     )
 
-    BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref')
+    export BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref')
 
     # Change target is the branch name we are merging into but due to the weird way jenkins does
     # the checkout it isn't recognized by git without the origin/ prefix
     export CHANGE_TARGET="origin/${BASE_BRANCH}"
+}
+
+function fetch_base_branch_local() {
+    rapids-logger "Retrieving base branch from git"
+    git remote add upstream ${GIT_UPSTREAM_URL}
+    git fetch upstream --tags
+    source ${MRC_ROOT}/ci/scripts/common.sh
+    export BASE_BRANCH=$(get_base_branch)
+    export CHANGE_TARGET="upstream/${BASE_BRANCH}"
+}
+
+function fetch_base_branch() {
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        fetch_base_branch_local
+    else
+        fetch_base_branch_gh_api
+    fi
+
     git submodule update --init --recursive
     rapids-logger "Base branch: ${BASE_BRANCH}"
 }
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
new file mode 100755
index 000000000..0527e96a9
--- /dev/null
+++ b/ci/scripts/run_ci_local.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+case "$1" in
+    "" )
+        STAGES=("bash")
+        ;;
+    "all" )
+        STAGES=("checks" "build-clang" "build-codecov" "build-gcc" "test-clang" "test-codecov" "test-gcc" "docs" "benchmark" "conda")
+        ;;
+    "build" )
+        STAGES=("build-clang" "build-codecov" "build-gcc")
+        ;;
+    "test" )
+        STAGES=("test-clang" "test-codecov" "test-gcc")
+        ;;
+    "checks" | "build-clang" | "build-codecov" | "build-gcc" | "codecov" | "test" | "test-clang" | "test-codecov" | "test-gcc" | "docs" | "benchmark" | "conda" | "bash" )
+        STAGES=("$1")
+        ;;
+    * )
+        echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", " \
+             "\"build-clang\", \"build-codecov\", \"build-gcc\", \"codecov\", \"test\", \"test-clang\", " \
+             "\"test-codecov\", \"test-gcc\", \"docs\", \"benchmark\", \"conda\" or \"bash\""
+        exit 1
+        ;;
+esac
+
+# CI image doesn't contain ssh, need to use https
+function git_ssh_to_https()
+{
+    local url=$1
+    echo $url | sed -e 's|^git@github\.com:|https://github.com/|'
+}
+
+MRC_ROOT=${MRC_ROOT:-$(git rev-parse --show-toplevel)}
+
+GIT_URL=$(git remote get-url origin)
+GIT_URL=$(git_ssh_to_https ${GIT_URL})
+
+GIT_UPSTREAM_URL=$(git remote get-url upstream)
+GIT_UPSTREAM_URL=$(git_ssh_to_https ${GIT_UPSTREAM_URL})
+
+GIT_BRANCH=$(git branch --show-current)
+GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
+
+LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MRC_ROOT}/.tmp/local_ci_tmp}
+CONTAINER_VER=${CONTAINER_VER:-230711}
+CUDA_VER=${CUDA_VER:-11.8}
+DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
+
+BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-${CONTAINER_VER}"
+TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-${CONTAINER_VER}"
+
+ENV_LIST="--env LOCAL_CI_TMP=/ci_tmp"
+ENV_LIST="${ENV_LIST} --env GIT_URL=${GIT_URL}"
+ENV_LIST="${ENV_LIST} --env GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}"
+ENV_LIST="${ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}"
+ENV_LIST="${ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}"
+ENV_LIST="${ENV_LIST} --env PARALLEL_LEVEL=$(nproc)"
+ENV_LIST="${ENV_LIST} --env CUDA_VER=${CUDA_VER}"
+ENV_LIST="${ENV_LIST} --env SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}"
+
+mkdir -p ${LOCAL_CI_TMP}
+cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}
+
+for STAGE in "${STAGES[@]}"; do
+    if [[ "${STAGE}" =~ "clang" || ]]; then
+        ENV_LIST="${ENV_LIST} --env BUILD_CC=clang"
+    elif [[ "${STAGE}" =~ "gcc" ]]; then
+        ENV_LIST="${ENV_LIST} --env BUILD_CC=gcc"
+    elif [[ "${STAGE}" =~ "codecov" ]]; then
+        ENV_LIST="${ENV_LIST} --env BUILD_CC=cc-coverage"
+    fi
+
+    DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}"
+    if [[ "${STAGE}" =~ "test" || "${STAGE}" =~ "codecov" || "${USE_GPU}" == "1" ]]; then
+        CONTAINER="${TEST_CONTAINER}"
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all --cap-add=sys_nice --cap-add=sys_ptrace"
+    else
+        CONTAINER="${BUILD_CONTAINER}"
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc"
+    fi
+
+    if [[ "${STAGE}" == "bash" ]]; then
+        DOCKER_RUN_CMD="bash --init-file /ci_tmp/bootstrap_local_ci.sh"
+    else
+        DOCKER_RUN_CMD="/ci_tmp/bootstrap_local_ci.sh"
+    fi
+
+    echo "Running ${STAGE} stage in ${CONTAINER}"
+    docker run ${DOCKER_RUN_ARGS} ${DOCKER_EXTRA_ARGS} ${CONTAINER} ${DOCKER_RUN_CMD}
+
+    STATUS=$?
+    if [[ ${STATUS} -ne 0 ]]; then
+        echo "Error: docker exited with a non-zero status code for ${STAGE} of ${STATUS}"
+        exit ${STATUS}
+    fi
+done

From 777384890145ea4d3915b4e864b7cc0269542d57 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 07:25:12 -0700
Subject: [PATCH 02/27] Ignore .tmp and used defined clangd

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 1a20325a2..53c9f38e0 100755
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 /build*/
+.tmp
 *.engine
 .Dockerfile
 .gitignore
@@ -17,6 +18,9 @@ include/mrc/version.hpp
 .vscode/settings.json
 .vscode/tasks.json
 
+# Ignore user-defined clangd settings
+.clangd
+
 # Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology
 
 ### C++ ###

From 86e7631d59ae627c85a1e19a6e7eed90a4be12e9 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 07:45:19 -0700
Subject: [PATCH 03/27] Fix syntax errors

---
 ci/scripts/bootstrap_local_ci.sh | 4 ++--
 ci/scripts/run_ci_local.sh       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh
index e8ad0fb46..7a8273a7c 100755
--- a/ci/scripts/bootstrap_local_ci.sh
+++ b/ci/scripts/bootstrap_local_ci.sh
@@ -29,9 +29,9 @@ unset CMAKE_CUDA_COMPILER_LAUNCHER
 unset CMAKE_CXX_COMPILER_LAUNCHER
 unset CMAKE_C_COMPILER_LAUNCHER
 
-if [[ "${STAGE}" =~ "build" || ]]; then
+if [[ "${STAGE}" =~ "build" ]]; then
     SCRIPT_NAME="build.sh"
-elif [[ "${STAGE}" =~ "test" || ]]; then
+elif [[ "${STAGE}" =~ "test" ]]; then
     SCRIPT_NAME="test.sh"
 else
     SCRIPT_NAME="${STAGE}.sh"
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 0527e96a9..593a2d917 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -77,7 +77,7 @@ mkdir -p ${LOCAL_CI_TMP}
 cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}
 
 for STAGE in "${STAGES[@]}"; do
-    if [[ "${STAGE}" =~ "clang" || ]]; then
+    if [[ "${STAGE}" =~ "clang" ]]; then
         ENV_LIST="${ENV_LIST} --env BUILD_CC=clang"
     elif [[ "${STAGE}" =~ "gcc" ]]; then
         ENV_LIST="${ENV_LIST} --env BUILD_CC=gcc"

From a28668435d0ab3c34b7c89299cd9ecb1d9bc3dc2 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 08:28:45 -0700
Subject: [PATCH 04/27] wip

---
 ci/scripts/github/build.sh  | 13 ++++++----
 ci/scripts/github/common.sh | 26 ++++++++++++++++----
 ci/scripts/run_ci_local.sh  | 47 +++++++++++++++++++++++--------------
 3 files changed, 60 insertions(+), 26 deletions(-)

diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh
index e63f04eef..654ce35d5 100755
--- a/ci/scripts/github/build.sh
+++ b/ci/scripts/github/build.sh
@@ -20,7 +20,12 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 update_conda_env
 
-CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DMRC_USE_CCACHE=ON"
+if [[ "${LOCAL_CI}" == "" ]]; then
+    CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DMRC_USE_CCACHE=ON"
+else
+    CMAKE_CACHE_FLAGS=""
+fi
+
 
 rapids-logger "Check versions"
 python3 --version
@@ -59,15 +64,15 @@ cmake --build build --parallel ${PARALLEL_LEVEL}
 rapids-logger "sccache usage for MRC build:"
 sccache --show-stats
 
-if [[ "${BUILD_CC}" != "gcc-coverage" ]]; then
+if [[ "${BUILD_CC}" != "gcc-coverage" || ${LOCAL_CI} == "1" ]]; then
     rapids-logger "Archiving results"
     tar cfj "${WORKSPACE_TMP}/dot_cache.tar.bz" .cache
     tar cfj "${WORKSPACE_TMP}/build.tar.bz" build
     ls -lh ${WORKSPACE_TMP}/
 
     rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-    aws s3 cp --no-progress "${WORKSPACE_TMP}/build.tar.bz" "${ARTIFACT_URL}/build.tar.bz"
-    aws s3 cp --no-progress "${WORKSPACE_TMP}/dot_cache.tar.bz" "${ARTIFACT_URL}/dot_cache.tar.bz"
+    upload_artifact "${WORKSPACE_TMP}/build.tar.bz"
+    upload_artifact "${WORKSPACE_TMP}/dot_cache.tar.bz"
 fi
 
 rapids-logger "Success"
diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh
index 930a3d4fb..cc4629971 100644
--- a/ci/scripts/github/common.sh
+++ b/ci/scripts/github/common.sh
@@ -78,9 +78,11 @@ function update_conda_env() {
     # Deactivate the environment first before updating
     conda deactivate
 
-    # Make sure we have the conda-merge package installed
-    if [[ -z "$(conda list | grep conda-merge)" ]]; then
-        rapids-mamba-retry install -q -n mrc -c conda-forge "conda-merge>=0.2"
+    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
+        # Make sure we have the conda-merge package installed
+        if [[ -z "$(conda list | grep conda-merge)" ]]; then
+            rapids-mamba-retry install -q -n mrc -c conda-forge "conda-merge>=0.2"
+        fi
     fi
 
     # Create a temp directory which we store the combined environment file in
@@ -90,8 +92,10 @@ function update_conda_env() {
     # will clobber the last env update
     conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml
 
-    # Update the conda env with prune remove excess packages (in case one was removed from the env)
-    rapids-mamba-retry env update -n mrc --prune --file ${condatmpdir}/merged_env.yml
+    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
+        # Update the conda env with prune remove excess packages (in case one was removed from the env)
+        rapids-mamba-retry env update -n mrc --prune --file ${condatmpdir}/merged_env.yml
+    fi
 
     # Delete the temp directory
     rm -rf ${condatmpdir}
@@ -166,3 +170,15 @@ function show_conda_info() {
     conda config --show-sources
     conda list --show-channel-urls
 }
+
+function upload_artifact() {
+    FILE_NAME=$1
+    BASE_NAME=$(basename "${FILE_NAME}")
+    rapids-logger "Uploading artifact: ${BASE_NAME}"
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        cp ${FILE_NAME} "${LOCAL_CI_TMP}/${BASE_NAME}"
+    else
+        aws s3 cp --only-show-errors "${FILE_NAME}" "${ARTIFACT_URL}/${BASE_NAME}"
+        echo "- ${DISPLAY_ARTIFACT_URL}/${BASE_NAME}" >> ${GITHUB_STEP_SUMMARY}
+    fi
+}
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 593a2d917..960b9ffcc 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -56,7 +56,7 @@ GIT_UPSTREAM_URL=$(git_ssh_to_https ${GIT_UPSTREAM_URL})
 GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
-LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MRC_ROOT}/.tmp/local_ci_tmp}
+BASE_LOCAL_CI_TMP=${BASE_LOCAL_CI_TMP:-${MRC_ROOT}/.tmp/local_ci_tmp}
 CONTAINER_VER=${CONTAINER_VER:-230711}
 CUDA_VER=${CUDA_VER:-11.8}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
@@ -64,27 +64,40 @@ DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-${CONTAINER_VER}"
 TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-${CONTAINER_VER}"
 
-ENV_LIST="--env LOCAL_CI_TMP=/ci_tmp"
-ENV_LIST="${ENV_LIST} --env GIT_URL=${GIT_URL}"
-ENV_LIST="${ENV_LIST} --env GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}"
-ENV_LIST="${ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}"
-ENV_LIST="${ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}"
-ENV_LIST="${ENV_LIST} --env PARALLEL_LEVEL=$(nproc)"
-ENV_LIST="${ENV_LIST} --env CUDA_VER=${CUDA_VER}"
-ENV_LIST="${ENV_LIST} --env SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}"
+# These variables are common to all stages
+BASE_ENV_LIST="--env LOCAL_CI_TMP=/ci_tmp"
+BASE_ENV_LIST="${BASE_ENV_LIST} --env GIT_URL=${GIT_URL}"
+BASE_ENV_LIST="${BASE_ENV_LIST} --env GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}"
+BASE_ENV_LIST="${BASE_ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}"
+BASE_ENV_LIST="${BASE_ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}"
+BASE_ENV_LIST="${BASE_ENV_LIST} --env PARALLEL_LEVEL=$(nproc)"
+BASE_ENV_LIST="${BASE_ENV_LIST} --env CUDA_VER=${CUDA_VER}"
+BASE_ENV_LIST="${BASE_ENV_LIST} --env SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}"
 
-mkdir -p ${LOCAL_CI_TMP}
-cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}
+mkdir -p ${BASE_LOCAL_CI_TMP}
+cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${BASE_LOCAL_CI_TMP}
 
 for STAGE in "${STAGES[@]}"; do
-    if [[ "${STAGE}" =~ "clang" ]]; then
-        ENV_LIST="${ENV_LIST} --env BUILD_CC=clang"
-    elif [[ "${STAGE}" =~ "gcc" ]]; then
-        ENV_LIST="${ENV_LIST} --env BUILD_CC=gcc"
-    elif [[ "${STAGE}" =~ "codecov" ]]; then
-        ENV_LIST="${ENV_LIST} --env BUILD_CC=cc-coverage"
+    # Take a copy of the base env list, then make stage specific changes
+    ENV_LIST="${BASE_ENV_LIST}"
+
+    if [[ "${STAGE}" =~ clang|gcc ]]; then
+        if [[ "${STAGE}" =~ "clang" ]]; then
+            BUILD_CC="clang"
+        elif [[ "${STAGE}" =~ "codecov" ]]; then
+            BUILD_CC="gcc-coverage"
+        elif [[ "${STAGE}" =~ "gcc" ]]; then
+            BUILD_CC="gcc"
+        fi
+
+        ENV_LIST="${ENV_LIST} --env BUILD_CC=${BUILD_CC}"
+        LOCAL_CI_TMP="${BASE_LOCAL_CI_TMP}/${BUILD_CC}"
+        mkdir -p ${LOCAL_CI_TMP}
+    else
+        LOCAL_CI_TMP="${BASE_LOCAL_CI_TMP}"
     fi
 
+
     DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}"
     if [[ "${STAGE}" =~ "test" || "${STAGE}" =~ "codecov" || "${USE_GPU}" == "1" ]]; then
         CONTAINER="${TEST_CONTAINER}"

From bc87ddc93e6025dd680b25772f0addce3c125dca Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 08:47:28 -0700
Subject: [PATCH 05/27] WIP

---
 ci/scripts/github/build.sh | 6 ++++--
 ci/scripts/run_ci_local.sh | 6 +++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh
index 654ce35d5..300452c05 100755
--- a/ci/scripts/github/build.sh
+++ b/ci/scripts/github/build.sh
@@ -61,8 +61,10 @@ cmake -B build -G Ninja ${CMAKE_FLAGS} .
 rapids-logger "Building MRC"
 cmake --build build --parallel ${PARALLEL_LEVEL}
 
-rapids-logger "sccache usage for MRC build:"
-sccache --show-stats
+if [[ "${LOCAL_CI}" == "" ]]; then
+    rapids-logger "sccache usage for MRC build:"
+    sccache --show-stats
+fi
 
 if [[ "${BUILD_CC}" != "gcc-coverage" || ${LOCAL_CI} == "1" ]]; then
     rapids-logger "Archiving results"
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 960b9ffcc..5382145ea 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -74,9 +74,6 @@ BASE_ENV_LIST="${BASE_ENV_LIST} --env PARALLEL_LEVEL=$(nproc)"
 BASE_ENV_LIST="${BASE_ENV_LIST} --env CUDA_VER=${CUDA_VER}"
 BASE_ENV_LIST="${BASE_ENV_LIST} --env SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}"
 
-mkdir -p ${BASE_LOCAL_CI_TMP}
-cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${BASE_LOCAL_CI_TMP}
-
 for STAGE in "${STAGES[@]}"; do
     # Take a copy of the base env list, then make stage specific changes
     ENV_LIST="${BASE_ENV_LIST}"
@@ -97,6 +94,9 @@ for STAGE in "${STAGES[@]}"; do
         LOCAL_CI_TMP="${BASE_LOCAL_CI_TMP}"
     fi
 
+    mkdir -p ${LOCAL_CI_TMP}
+    cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}
+
 
     DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}"
     if [[ "${STAGE}" =~ "test" || "${STAGE}" =~ "codecov" || "${USE_GPU}" == "1" ]]; then

From 531a61ba60f503c116547430a4b87ac8d145cf36 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 09:12:44 -0700
Subject: [PATCH 06/27] Log compiler

---
 ci/scripts/github/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh
index 300452c05..4650a9c7d 100755
--- a/ci/scripts/github/build.sh
+++ b/ci/scripts/github/build.sh
@@ -54,7 +54,7 @@ fi
 
 show_conda_info
 
-rapids-logger "Configuring for build and test"
+rapids-logger "Configuring for build and test using ${BUILD_CC}"
 git submodule update --init --recursive
 cmake -B build -G Ninja ${CMAKE_FLAGS} .
 

From 6e0ccfe1203a026a8c4755eb8e110efff0002b92 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 09:26:45 -0700
Subject: [PATCH 07/27] WIP [no ci]

---
 ci/scripts/github/build.sh | 2 +-
 ci/scripts/run_ci_local.sh | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh
index 4650a9c7d..300452c05 100755
--- a/ci/scripts/github/build.sh
+++ b/ci/scripts/github/build.sh
@@ -54,7 +54,7 @@ fi
 
 show_conda_info
 
-rapids-logger "Configuring for build and test using ${BUILD_CC}"
+rapids-logger "Configuring for build and test"
 git submodule update --init --recursive
 cmake -B build -G Ninja ${CMAKE_FLAGS} .
 
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 5382145ea..a31ba1d0d 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -27,13 +27,13 @@ case "$1" in
     "test" )
         STAGES=("test-clang" "test-codecov" "test-gcc")
         ;;
-    "checks" | "build-clang" | "build-codecov" | "build-gcc" | "codecov" | "test" | "test-clang" | "test-codecov" | "test-gcc" | "docs" | "benchmark" | "conda" | "bash" )
+    "checks" | "build-clang" | "build-codecov" | "build-gcc" | "test" | "test-clang" | "test-codecov" | "test-gcc" | "docs" | "benchmark" | "conda" | "bash" )
         STAGES=("$1")
         ;;
     * )
         echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", " \
-             "\"build-clang\", \"build-codecov\", \"build-gcc\", \"codecov\", \"test\", \"test-clang\", " \
-             "\"test-codecov\", \"test-gcc\", \"docs\", \"benchmark\", \"conda\" or \"bash\""
+             "\"build-clang\", \"build-codecov\", \"build-gcc\", \"test\", \"test-clang\", \"test-codecov\", " \
+             "\"test-gcc\", \"docs\", \"benchmark\", \"conda\" or \"bash\""
         exit 1
         ;;
 esac
@@ -78,7 +78,7 @@ for STAGE in "${STAGES[@]}"; do
     # Take a copy of the base env list, then make stage specific changes
     ENV_LIST="${BASE_ENV_LIST}"
 
-    if [[ "${STAGE}" =~ clang|gcc ]]; then
+    if [[ "${STAGE}" =~ clang|codecov|gcc ]]; then
         if [[ "${STAGE}" =~ "clang" ]]; then
             BUILD_CC="clang"
         elif [[ "${STAGE}" =~ "codecov" ]]; then

From 19498d9a6f6c7c2faf278014e6a75dfe90fb2446 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 09:49:54 -0700
Subject: [PATCH 08/27] WIP [no ci]

---
 ci/scripts/github/common.sh | 17 ++++++++++++++++-
 ci/scripts/github/test.sh   |  6 +++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh
index cc4629971..964816836 100644
--- a/ci/scripts/github/common.sh
+++ b/ci/scripts/github/common.sh
@@ -56,7 +56,12 @@ export S3_URL="s3://rapids-downloads/ci/mrc"
 export DISPLAY_URL="https://downloads.rapids.ai/ci/mrc"
 export ARTIFACT_ENDPOINT="/pull-request/${PR_NUM}/${GIT_COMMIT}/${NVARCH}/${BUILD_CC}"
 export ARTIFACT_URL="${S3_URL}${ARTIFACT_ENDPOINT}"
-export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}"
+
+if [[ "${LOCAL_CI}" == "1" ]]; then
+    export DISPLAY_ARTIFACT_URL="${LOCAL_CI_TMP}"
+else
+    export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}"
+fi
 
 # Set sccache env vars
 export SCCACHE_S3_KEY_PREFIX=mrc-${NVARCH}-${BUILD_CC}
@@ -182,3 +187,13 @@ function upload_artifact() {
         echo "- ${DISPLAY_ARTIFACT_URL}/${BASE_NAME}" >> ${GITHUB_STEP_SUMMARY}
     fi
 }
+
+function download_artifact() {
+    ARTIFACT=$1
+    rapids-logger "Downloading ${ARTIFACT} from ${DISPLAY_ARTIFACT_URL}"
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        cp "${LOCAL_CI_TMP}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
+    else
+        fetch_s3 "${ARTIFACT_URL}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
+    fi
+}
diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh
index 0aab525a0..40000a516 100755
--- a/ci/scripts/github/test.sh
+++ b/ci/scripts/github/test.sh
@@ -22,8 +22,8 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 update_conda_env
 
 rapids-logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/"
-fetch_s3 "${ARTIFACT_ENDPOINT}/dot_cache.tar.bz" "${WORKSPACE_TMP}/dot_cache.tar.bz"
-fetch_s3 "${ARTIFACT_ENDPOINT}/build.tar.bz" "${WORKSPACE_TMP}/build.tar.bz"
+download_artifact "dot_cache.tar.bz"
+download_artifact "build.tar.bz"
 
 tar xf "${WORKSPACE_TMP}/dot_cache.tar.bz"
 tar xf "${WORKSPACE_TMP}/build.tar.bz"
@@ -60,7 +60,7 @@ cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR})
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-aws s3 cp ${WORKSPACE_TMP}/test_reports.tar.bz "${ARTIFACT_URL}/test_reports.tar.bz"
+upload_artifact ${WORKSPACE_TMP}/test_reports.tar.bz
 
 TEST_RESULTS=$(($CTEST_RESULTS+$PYTEST_RESULTS))
 exit ${TEST_RESULTS}

From 67c01957f86ef49b29c5bd1a9aace1b557a8dace Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 11:08:17 -0700
Subject: [PATCH 09/27] WIP [no ci]

---
 ci/scripts/github/conda.sh | 11 +++++++++++
 ci/scripts/run_ci_local.sh |  4 ++++
 2 files changed, 15 insertions(+)

diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh
index 3b8104ad3..57908d5b6 100755
--- a/ci/scripts/github/conda.sh
+++ b/ci/scripts/github/conda.sh
@@ -37,6 +37,17 @@ fi
 conda info
 
 rapids-logger "Building Conda Package"
+expor
 
 # Run the conda build and upload
 ${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "$@"
+
+rapids-logger "Building Conda Package... Done"
+if [[ "$@" == "" ]]; then
+   # if we didn't receive the upload argument, we can still upload the artifact to S3
+   tar cfj "${WORKSPACE_TMP}/conda.tar.bz" "${RAPIDS_CONDA_BLD_OUTPUT_DIR}"
+   ls -lh ${WORKSPACE_TMP}/
+
+   rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
+   upload_artifact "${WORKSPACE_TMP}/conda.tar.bz"
+fi
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index a31ba1d0d..944f63fad 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -94,6 +94,10 @@ for STAGE in "${STAGES[@]}"; do
         LOCAL_CI_TMP="${BASE_LOCAL_CI_TMP}"
     fi
 
+    if [[ ${STAGE} == "conda" ]]; then
+        ENV_LIST="${ENV_LIST} --env CONDA_ARGS=\"--output-folder=/ci_tmp\""
+    fi
+
     mkdir -p ${LOCAL_CI_TMP}
     cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}
 

From 90526425a009cfe17f8e9ae739fe291122caf3b0 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 11:09:13 -0700
Subject: [PATCH 10/27] WIP [no ci]

---
 ci/scripts/github/conda.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh
index 57908d5b6..7070b5b26 100755
--- a/ci/scripts/github/conda.sh
+++ b/ci/scripts/github/conda.sh
@@ -37,7 +37,6 @@ fi
 conda info
 
 rapids-logger "Building Conda Package"
-expor
 
 # Run the conda build and upload
 ${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "$@"

From 5e979adbc7e2f1fb9ef5f7be012d355e4231e25f Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 11:11:34 -0700
Subject: [PATCH 11/27] WIP [no ci]

---
 ci/scripts/run_ci_local.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 944f63fad..a31ba1d0d 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -94,10 +94,6 @@ for STAGE in "${STAGES[@]}"; do
         LOCAL_CI_TMP="${BASE_LOCAL_CI_TMP}"
     fi
 
-    if [[ ${STAGE} == "conda" ]]; then
-        ENV_LIST="${ENV_LIST} --env CONDA_ARGS=\"--output-folder=/ci_tmp\""
-    fi
-
     mkdir -p ${LOCAL_CI_TMP}
     cp ${MRC_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}
 

From 4a13824f1b939f2f38aab9f7443e893a3ef072a3 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 11:24:18 -0700
Subject: [PATCH 12/27] WIP: Explicitly check upload arg [no ci]

---
 ci/scripts/github/conda.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh
index 7070b5b26..151eab6b5 100755
--- a/ci/scripts/github/conda.sh
+++ b/ci/scripts/github/conda.sh
@@ -42,7 +42,7 @@ rapids-logger "Building Conda Package"
 ${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "$@"
 
 rapids-logger "Building Conda Package... Done"
-if [[ "$@" == "" ]]; then
+if [[ "$@" !~ "upload" ]]; then
    # if we didn't receive the upload argument, we can still upload the artifact to S3
    tar cfj "${WORKSPACE_TMP}/conda.tar.bz" "${RAPIDS_CONDA_BLD_OUTPUT_DIR}"
    ls -lh ${WORKSPACE_TMP}/

From ab685fa6e156d4e488e1b8ad5e14ad0c84e7355c Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 13:23:50 -0700
Subject: [PATCH 13/27] WIP [no ci]

---
 ci/scripts/bootstrap_local_ci.sh | 24 +++++++++++++++---------
 ci/scripts/github/conda.sh       |  8 +++++---
 ci/scripts/run_ci_local.sh       |  4 ++--
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh
index 7a8273a7c..92c02f07e 100755
--- a/ci/scripts/bootstrap_local_ci.sh
+++ b/ci/scripts/bootstrap_local_ci.sh
@@ -25,18 +25,24 @@ git checkout ${GIT_COMMIT}
 export MRC_ROOT=$(pwd)
 export WORKSPACE=${MRC_ROOT}
 export LOCAL_CI=1
+GH_SCRIPT_DIR="${MRC_ROOT}/ci/scripts/github"
+
 unset CMAKE_CUDA_COMPILER_LAUNCHER
 unset CMAKE_CXX_COMPILER_LAUNCHER
 unset CMAKE_C_COMPILER_LAUNCHER
 
-if [[ "${STAGE}" =~ "build" ]]; then
-    SCRIPT_NAME="build.sh"
-elif [[ "${STAGE}" =~ "test" ]]; then
-    SCRIPT_NAME="test.sh"
-else
-    SCRIPT_NAME="${STAGE}.sh"
-fi
-
 if [[ "${STAGE}" != "bash" ]]; then
-    ${MRC_ROOT}/ci/scripts/github/${SCRIPT_NAME}
+    if [[ "${STAGE}" == "benchmark" ]]; then
+        CI_SCRIPT="${GH_SCRIPT_DIR}/pre_benchmark.sh && ${GH_SCRIPT_DIR}/benchmark.sh && ${GH_SCRIPT_DIR}/post_benchmark.sh"
+    else
+        if [[ "${STAGE}" =~ "build" ]]; then
+            CI_SCRIPT="${GH_SCRIPT_DIR}/build.sh"
+        elif [[ "${STAGE}" =~ "test" ]]; then
+            CI_SCRIPT="${GH_SCRIPT_DIR}/test.sh"
+        else
+            CI_SCRIPT="${GH_SCRIPT_DIR}/${STAGE}.sh"
+        fi
+    fi
+
+    ${CI_SCRIPT}
 fi
diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh
index 151eab6b5..dc4e72d65 100755
--- a/ci/scripts/github/conda.sh
+++ b/ci/scripts/github/conda.sh
@@ -16,6 +16,7 @@
 
 set -e
 
+CI_SCRIPT_ARGS = "$@"
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
 # Its important that we are in the base environment for the build
@@ -39,10 +40,11 @@ conda info
 rapids-logger "Building Conda Package"
 
 # Run the conda build and upload
-${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "$@"
+${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "${CI_SCRIPT_ARGS}"
 
-rapids-logger "Building Conda Package... Done"
-if [[ "$@" !~ "upload" ]]; then
+if [["${CI_SCRIPT_ARGS}" =~ "upload" ]]; then
+   rapids-logger "Building Conda Package... Done"
+else
    # if we didn't receive the upload argument, we can still upload the artifact to S3
    tar cfj "${WORKSPACE_TMP}/conda.tar.bz" "${RAPIDS_CONDA_BLD_OUTPUT_DIR}"
    ls -lh ${WORKSPACE_TMP}/
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index a31ba1d0d..21129143c 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -78,12 +78,12 @@ for STAGE in "${STAGES[@]}"; do
     # Take a copy of the base env list, then make stage specific changes
     ENV_LIST="${BASE_ENV_LIST}"
 
-    if [[ "${STAGE}" =~ clang|codecov|gcc ]]; then
+    if [[ "${STAGE}" =~ benchmark|clang|codecov|gcc ]]; then
         if [[ "${STAGE}" =~ "clang" ]]; then
             BUILD_CC="clang"
         elif [[ "${STAGE}" =~ "codecov" ]]; then
             BUILD_CC="gcc-coverage"
-        elif [[ "${STAGE}" =~ "gcc" ]]; then
+        else
             BUILD_CC="gcc"
         fi
 

From b46f4a08673f32af017cd44cc85077ccbdf0be53 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 13:25:50 -0700
Subject: [PATCH 14/27] WIP [no ci]

---
 ci/scripts/github/conda.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh
index dc4e72d65..5a56ee391 100755
--- a/ci/scripts/github/conda.sh
+++ b/ci/scripts/github/conda.sh
@@ -16,7 +16,7 @@
 
 set -e
 
-CI_SCRIPT_ARGS = "$@"
+CI_SCRIPT_ARGS="$@"
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
 # Its important that we are in the base environment for the build

From 555f72ac5e9754f893fe46c60ff575fea1e29755 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 13:32:37 -0700
Subject: [PATCH 15/27] Docker command needs DOCKER_BUILDKIT evn variable to be
 set [no ci]

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 10e3ed7ad..4bdb5c3a3 100644
--- a/README.md
+++ b/README.md
@@ -151,7 +151,7 @@ pytest $MRC_ROOT/python
 ### Docker Installation
 A Dockerfile is provided at `$MRC_ROOT` and can be built with
 ```bash
-docker build -t mrc:latest .
+DOCKER_BUILDKIT=1 docker build -t mrc:latest .
 ```
 To run the container
 ```bash

From 412421016955f3843a3df4c3ee48a3091e8569c5 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 13:33:57 -0700
Subject: [PATCH 16/27] Set new base image [no ci]

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index cae834533..e303f19f9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,7 +16,7 @@
 # limitations under the License.
 
 
-ARG FROM_IMAGE="rapidsai/ci"
+ARG FROM_IMAGE="rapidsai/ci-conda"
 ARG CUDA_VER=11.8.0
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_VER=20.04

From 8ef0b891814dab0af2e9046cdaa13ab88cf6f1ca Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 13:50:56 -0700
Subject: [PATCH 17/27] WIP [no ci]

---
 ci/scripts/github/docs.sh           | 2 +-
 ci/scripts/github/post_benchmark.sh | 2 +-
 ci/scripts/github/pre_benchmark.sh  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh
index 2e0a1f64c..c5f10a53a 100755
--- a/ci/scripts/github/docs.sh
+++ b/ci/scripts/github/docs.sh
@@ -39,6 +39,6 @@ rapids-logger "Tarring the docs"
 tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-aws s3 cp --no-progress "${WORKSPACE_TMP}/docs.tar.bz" "${ARTIFACT_URL}/docs.tar.bz"
+upload_artifact "${WORKSPACE_TMP}/docs.tar.bz"
 
 rapids-logger "Success"
diff --git a/ci/scripts/github/post_benchmark.sh b/ci/scripts/github/post_benchmark.sh
index d08bce2b4..4ac346120 100755
--- a/ci/scripts/github/post_benchmark.sh
+++ b/ci/scripts/github/post_benchmark.sh
@@ -25,6 +25,6 @@ cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/benchmark_reports.tar.bz $(basename ${REPORTS_DIR})
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-aws s3 cp ${WORKSPACE_TMP}/benchmark_reports.tar.bz "${ARTIFACT_URL}/benchmark_reports.tar.bz"
+upload_artifact ${WORKSPACE_TMP}/benchmark_reports.tar.bz
 
 exit $(cat ${WORKSPACE_TMP}/exit_status)
diff --git a/ci/scripts/github/pre_benchmark.sh b/ci/scripts/github/pre_benchmark.sh
index 419df25c2..c9294d2c8 100755
--- a/ci/scripts/github/pre_benchmark.sh
+++ b/ci/scripts/github/pre_benchmark.sh
@@ -21,7 +21,7 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 update_conda_env
 
 rapids-logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/"
-fetch_s3 "${ARTIFACT_ENDPOINT}/build.tar.bz" "${WORKSPACE_TMP}/build.tar.bz"
+download_artifact "build.tar.bz"
 
 tar xf "${WORKSPACE_TMP}/build.tar.bz"
 

From 7c40b71d9fcfb415a27bc1fe3c5b486cdd0d6da6 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 13:58:53 -0700
Subject: [PATCH 18/27] Update image versions [no ci]

---
 .github/workflows/pull_request.yml | 4 ++--
 ci/scripts/run_ci_local.sh         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index f10b02fea..56a519e5f 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -71,9 +71,9 @@ jobs:
       # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       # Build container
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230711
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230920
       # Test container
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230711
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230920
       # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc.
       pr_info: ${{ needs.prepare.outputs.pr_info }}
     secrets:
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 21129143c..1db6209c5 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -57,7 +57,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 BASE_LOCAL_CI_TMP=${BASE_LOCAL_CI_TMP:-${MRC_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-230711}
+CONTAINER_VER=${CONTAINER_VER:-230920}
 CUDA_VER=${CUDA_VER:-11.8}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 

From d4461507497ab45b2507a67c34a58b1dda81a475 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 14:09:09 -0700
Subject: [PATCH 19/27] Fix type-o

---
 .github/workflows/ci_pipe.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml
index 0c6da79f4..6dd1b08c3 100644
--- a/.github/workflows/ci_pipe.yml
+++ b/.github/workflows/ci_pipe.yml
@@ -294,7 +294,7 @@ jobs:
         run: ./mrc/ci/scripts/github/benchmark.sh
       - name: post_benchmark
         shell: bash
-        run: ./mrc/ci/scripts/github/benchmark.sh
+        run: ./mrc/ci/scripts/github/post_benchmark.sh
 
 
   package:

From 1a05b0e557bc669fddd29d4cf41a97fc3f964193 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 14:23:24 -0700
Subject: [PATCH 20/27] Fix defining benchmark script

---
 ci/scripts/bootstrap_local_ci.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh
index 92c02f07e..08a545432 100755
--- a/ci/scripts/bootstrap_local_ci.sh
+++ b/ci/scripts/bootstrap_local_ci.sh
@@ -33,7 +33,12 @@ unset CMAKE_C_COMPILER_LAUNCHER
 
 if [[ "${STAGE}" != "bash" ]]; then
     if [[ "${STAGE}" == "benchmark" ]]; then
-        CI_SCRIPT="${GH_SCRIPT_DIR}/pre_benchmark.sh && ${GH_SCRIPT_DIR}/benchmark.sh && ${GH_SCRIPT_DIR}/post_benchmark.sh"
+        CI_SCRIPT="${WORKSPACE_TMP}/benchmark_ci_script.sh"
+        echo "#!/bin/bash" > ${CI_SCRIPT}
+        echo "${GH_SCRIPT_DIR}/pre_benchmark.sh" >> ${CI_SCRIPT}
+        echo "${GH_SCRIPT_DIR}/benchmark.sh" >> ${CI_SCRIPT}
+        echo "${GH_SCRIPT_DIR}/post_benchmark.sh" >> ${CI_SCRIPT}
+        chmod +x ${CI_SCRIPT}
     else
         if [[ "${STAGE}" =~ "build" ]]; then
             CI_SCRIPT="${GH_SCRIPT_DIR}/build.sh"

From 30442257f731ebc4259f7c81503d5eef960a2f12 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 14:25:35 -0700
Subject: [PATCH 21/27] Add cap-add flags for benchmark [no ci]

---
 ci/scripts/run_ci_local.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index 1db6209c5..d9a521f7a 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -105,6 +105,9 @@ for STAGE in "${STAGES[@]}"; do
     else
         CONTAINER="${BUILD_CONTAINER}"
         DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc"
+        if [[ "${STAGE}" == "benchmark" ]]; then
+            DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice --cap-add=sys_ptrace"
+        fi
     fi
 
     if [[ "${STAGE}" == "bash" ]]; then

From b23f8a44fecfb621fafb942d3197435550dcce23 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 14:46:47 -0700
Subject: [PATCH 22/27] Re-work codecov [no ci]

---
 ci/scripts/bootstrap_local_ci.sh  | 17 ++++++++++++-----
 ci/scripts/github/test_codecov.sh | 27 ++++++++++++++++-----------
 ci/scripts/run_ci_local.sh        | 14 +++++++-------
 3 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh
index 08a545432..86939932b 100755
--- a/ci/scripts/bootstrap_local_ci.sh
+++ b/ci/scripts/bootstrap_local_ci.sh
@@ -32,12 +32,19 @@ unset CMAKE_CXX_COMPILER_LAUNCHER
 unset CMAKE_C_COMPILER_LAUNCHER
 
 if [[ "${STAGE}" != "bash" ]]; then
-    if [[ "${STAGE}" == "benchmark" ]]; then
-        CI_SCRIPT="${WORKSPACE_TMP}/benchmark_ci_script.sh"
+    # benchmark & codecov are composite stages, the rest are composed of a single shell script
+    if [[ "${STAGE}" == "benchmark" || "${STAGE}" == "codecov" ]]; then
+        CI_SCRIPT="${WORKSPACE_TMP}/ci_script.sh"
         echo "#!/bin/bash" > ${CI_SCRIPT}
-        echo "${GH_SCRIPT_DIR}/pre_benchmark.sh" >> ${CI_SCRIPT}
-        echo "${GH_SCRIPT_DIR}/benchmark.sh" >> ${CI_SCRIPT}
-        echo "${GH_SCRIPT_DIR}/post_benchmark.sh" >> ${CI_SCRIPT}
+        if [[ "${STAGE}" == "benchmark" ]]; then
+            echo "${GH_SCRIPT_DIR}/pre_benchmark.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/benchmark.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/post_benchmark.sh" >> ${CI_SCRIPT}
+        else
+            echo "${GH_SCRIPT_DIR}/build.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/test_codecov.sh" >> ${CI_SCRIPT}
+        fi
+
         chmod +x ${CI_SCRIPT}
     else
         if [[ "${STAGE}" =~ "build" ]]; then
diff --git a/ci/scripts/github/test_codecov.sh b/ci/scripts/github/test_codecov.sh
index 4a0ef3ce8..97955859a 100755
--- a/ci/scripts/github/test_codecov.sh
+++ b/ci/scripts/github/test_codecov.sh
@@ -58,13 +58,16 @@ cd ${MRC_ROOT}/build
 # correctly and enabling relative only ignores system and conda files.
 find . -type f -name '*.gcda' -exec x86_64-conda_cos6-linux-gnu-gcov -pbc --source-prefix ${MRC_ROOT} --relative-only {} + 1> /dev/null
 
-rapids-logger "Uploading codecov for C++ tests"
 
-# Get the list of files that we are interested in (Keeps the upload small)
-GCOV_FILES=$(find . -type f \( -iname "cpp#mrc#include#*.gcov" -or -iname "python#*.gcov" -or -iname "cpp#mrc#src#*.gcov" \))
+if [[ "${LOCAL_CI}" == "" ]]; then
+  rapids-logger "Uploading codecov for C++ tests"
 
-# Upload the .gcov files directly to codecov. They do a good job at processing the partials
-/opt/conda/envs/mrc/bin/codecov ${CODECOV_ARGS} -f ${GCOV_FILES} -F cpp
+  # Get the list of files that we are interested in (Keeps the upload small)
+  GCOV_FILES=$(find . -type f \( -iname "cpp#mrc#include#*.gcov" -or -iname "python#*.gcov" -or -iname "cpp#mrc#src#*.gcov" \))
+
+  # Upload the .gcov files directly to codecov. They do a good job at processing the partials
+  /opt/conda/envs/mrc/bin/codecov ${CODECOV_ARGS} -f ${GCOV_FILES} -F cpp
+fi
 
 # Remove the gcov files and any gcda files to reset counters
 find . -type f  \( -iname "*.gcov" -or -iname "*.gcda" \) -exec rm {} \;
@@ -85,13 +88,15 @@ cd ${MRC_ROOT}/build
 # correctly and enabling relative only ignores system and conda files.
 find . -type f -name '*.gcda' -exec x86_64-conda_cos6-linux-gnu-gcov -pbc --source-prefix ${MRC_ROOT} --relative-only {} + 1> /dev/null
 
-rapids-logger "Uploading codecov for Python tests"
+if [[ "${LOCAL_CI}" == "" ]]; then
+  rapids-logger "Uploading codecov for Python tests"
 
-# Get the list of files that we are interested in (Keeps the upload small)
-GCOV_FILES=$(find . -type f \( -iname "cpp#mrc#include#*.gcov" -or -iname "python#*.gcov" -or -iname "cpp#mrc#src#*.gcov" \))
+  # Get the list of files that we are interested in (Keeps the upload small)
+  GCOV_FILES=$(find . -type f \( -iname "cpp#mrc#include#*.gcov" -or -iname "python#*.gcov" -or -iname "cpp#mrc#src#*.gcov" \))
 
-# Upload the .gcov files directly to codecov. They do a good job at processing the partials
-/opt/conda/envs/mrc/bin/codecov ${CODECOV_ARGS} -f ${GCOV_FILES} -F py
+  # Upload the .gcov files directly to codecov. They do a good job at processing the partials
+  /opt/conda/envs/mrc/bin/codecov ${CODECOV_ARGS} -f ${GCOV_FILES} -F py
+fi
 
 # Remove the gcov files and any gcda files to reset counters
 find . -type f  \( -iname "*.gcov" -or -iname "*.gcda" \) -exec rm {} \;
@@ -101,7 +106,7 @@ cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR})
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-aws s3 cp ${WORKSPACE_TMP}/test_reports.tar.bz "${ARTIFACT_URL}/test_reports.tar.bz"
+upload_artifact ${WORKSPACE_TMP}/test_reports.tar.bz
 
 TEST_RESULTS=$(($CTEST_RESULTS+$PYTEST_RESULTS))
 exit ${TEST_RESULTS}
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index d9a521f7a..bb566c463 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -19,21 +19,21 @@ case "$1" in
         STAGES=("bash")
         ;;
     "all" )
-        STAGES=("checks" "build-clang" "build-codecov" "build-gcc" "test-clang" "test-codecov" "test-gcc" "docs" "benchmark" "conda")
+        STAGES=("checks" "build-clang" "build-gcc" "test-clang" "test-gcc" "codecov" "docs" "benchmark" "conda")
         ;;
     "build" )
-        STAGES=("build-clang" "build-codecov" "build-gcc")
+        STAGES=("build-clang" "build-gcc")
         ;;
     "test" )
-        STAGES=("test-clang" "test-codecov" "test-gcc")
+        STAGES=("test-clang" "test-gcc")
         ;;
-    "checks" | "build-clang" | "build-codecov" | "build-gcc" | "test" | "test-clang" | "test-codecov" | "test-gcc" | "docs" | "benchmark" | "conda" | "bash" )
+    "checks" | "build-clang" | "build-gcc" | "test" | "test-clang" | "test-gcc" | "codecov" | "docs" | "benchmark" | "conda" | "bash" )
         STAGES=("$1")
         ;;
     * )
         echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", " \
-             "\"build-clang\", \"build-codecov\", \"build-gcc\", \"test\", \"test-clang\", \"test-codecov\", " \
-             "\"test-gcc\", \"docs\", \"benchmark\", \"conda\" or \"bash\""
+             "\"build-clang\", \"build-gcc\", \"test\", \"test-clang\", \"test-gcc\", \"codecov\"," \
+             "\"docs\", \"benchmark\", \"conda\" or \"bash\""
         exit 1
         ;;
 esac
@@ -81,7 +81,7 @@ for STAGE in "${STAGES[@]}"; do
     if [[ "${STAGE}" =~ benchmark|clang|codecov|gcc ]]; then
         if [[ "${STAGE}" =~ "clang" ]]; then
             BUILD_CC="clang"
-        elif [[ "${STAGE}" =~ "codecov" ]]; then
+        elif [[ "${STAGE}" == "codecov" ]]; then
             BUILD_CC="gcc-coverage"
         else
             BUILD_CC="gcc"

From ea3dbcc606e8c470d06ff80d7e952a41142378f8 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 15:11:59 -0700
Subject: [PATCH 23/27] Formatting fix

---
 ci/scripts/run_ci_local.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
index bb566c463..bae506ccf 100755
--- a/ci/scripts/run_ci_local.sh
+++ b/ci/scripts/run_ci_local.sh
@@ -27,7 +27,8 @@ case "$1" in
     "test" )
         STAGES=("test-clang" "test-gcc")
         ;;
-    "checks" | "build-clang" | "build-gcc" | "test" | "test-clang" | "test-gcc" | "codecov" | "docs" | "benchmark" | "conda" | "bash" )
+    "checks" | "build-clang" | "build-gcc" | "test" | "test-clang" | "test-gcc" | "codecov" | "docs" | "benchmark" | \
+    "conda" | "bash" )
         STAGES=("$1")
         ;;
     * )

From 9f853f04a8d82c76c73be3765f6e8d0b7faee8d7 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Wed, 20 Sep 2023 15:15:43 -0700
Subject: [PATCH 24/27] Fix CR year

---
 ci/scripts/github/post_benchmark.sh | 2 +-
 ci/scripts/github/pre_benchmark.sh  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/github/post_benchmark.sh b/ci/scripts/github/post_benchmark.sh
index 4ac346120..943abc7e0 100755
--- a/ci/scripts/github/post_benchmark.sh
+++ b/ci/scripts/github/post_benchmark.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/ci/scripts/github/pre_benchmark.sh b/ci/scripts/github/pre_benchmark.sh
index c9294d2c8..c14a29144 100755
--- a/ci/scripts/github/pre_benchmark.sh
+++ b/ci/scripts/github/pre_benchmark.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

From 8a2c70dd14345e695144cfa707094b53b301c0b1 Mon Sep 17 00:00:00 2001
From: David Gardner <96306125+dagardner-nv@users.noreply.github.com>
Date: Fri, 22 Sep 2023 14:05:21 -0700
Subject: [PATCH 25/27] Update ci/scripts/bootstrap_local_ci.sh

Co-authored-by: Christopher Harris <xixonia@gmail.com>
---
 ci/scripts/bootstrap_local_ci.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh
index 86939932b..f1ff55bb2 100755
--- a/ci/scripts/bootstrap_local_ci.sh
+++ b/ci/scripts/bootstrap_local_ci.sh
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export WORKSPACE_TMP="$(pwd)/ws_tmp"
+export WORKSPACE_TMP="$(pwd)/.tmp/local_ci_workspace"
 mkdir -p ${WORKSPACE_TMP}
 git clone ${GIT_URL} mrc
 cd mrc/

From ef3fbcda6e1af659b040f16cdd57e5721a92ada3 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Fri, 22 Sep 2023 14:20:50 -0700
Subject: [PATCH 26/27] More explicit checking for script args

---
 ci/scripts/github/conda.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh
index 5a56ee391..36a878528 100755
--- a/ci/scripts/github/conda.sh
+++ b/ci/scripts/github/conda.sh
@@ -42,7 +42,7 @@ rapids-logger "Building Conda Package"
 # Run the conda build and upload
 ${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "${CI_SCRIPT_ARGS}"
 
-if [["${CI_SCRIPT_ARGS}" =~ "upload" ]]; then
+if [[ " ${CI_SCRIPT_ARGS} " =~ " upload " ]]; then
    rapids-logger "Building Conda Package... Done"
 else
    # if we didn't receive the upload argument, we can still upload the artifact to S3

From f5aaf383e10dbb1223ac20572b1b1d72727d73d2 Mon Sep 17 00:00:00 2001
From: David Gardner <dagardner@nvidia.com>
Date: Mon, 25 Sep 2023 09:22:17 -0700
Subject: [PATCH 27/27] Remove fetch_s3 in favor of download_artifact

---
 ci/scripts/github/common.sh | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh
index 964816836..17807bdce 100644
--- a/ci/scripts/github/common.sh
+++ b/ci/scripts/github/common.sh
@@ -156,18 +156,6 @@ function fetch_base_branch() {
     rapids-logger "Base branch: ${BASE_BRANCH}"
 }
 
-function fetch_s3() {
-    ENDPOINT=$1
-    DESTINATION=$2
-    if [[ "${USE_S3_CURL}" == "1" ]]; then
-        curl -f "${DISPLAY_URL}${ENDPOINT}" -o "${DESTINATION}"
-        FETCH_STATUS=$?
-    else
-        aws s3 cp --no-progress "${S3_URL}${ENDPOINT}" "${DESTINATION}"
-        FETCH_STATUS=$?
-    fi
-}
-
 function show_conda_info() {
 
     rapids-logger "Check Conda info"
@@ -194,6 +182,6 @@ function download_artifact() {
     if [[ "${LOCAL_CI}" == "1" ]]; then
         cp "${LOCAL_CI_TMP}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
     else
-        fetch_s3 "${ARTIFACT_URL}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
+        aws s3 cp --no-progress "${ARTIFACT_URL}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
     fi
 }