Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[MXNET-674] Speed up GPU builds in CI #12782

Merged
merged 3 commits into from
Oct 21, 2018
Merged

[MXNET-674] Speed up GPU builds in CI #12782

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5e1a0f4
[MXNET-674] Speed up GPU builds in CI
KellenSunderland Oct 11, 2018
cc9fd66
[MXNET-674] Refactor SMs into shell variable
KellenSunderland Oct 20, 2018
d2307f9
[MXNET-674] Build CMake GPU CI jobs without PTX
KellenSunderland Oct 21, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 74 additions & 64 deletions ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
set -ex

NOSE_COVERAGE_ARGUMENTS="--with-coverage --cover-inclusive --cover-xml --cover-branches --cover-package=mxnet"
CI_CUDA_COMPUTE_CAPABILITIES="-gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_70,code=sm_70"
CI_CMAKE_CUDA_ARCH_BIN="52,70"

clean_repo() {
set -ex
Expand Down Expand Up @@ -293,15 +295,16 @@ build_centos7_gpu() {
# unfortunately this build has problems in 3rdparty dependencies with ccache and make
# build_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_LAPACK=1 \
USE_LAPACK_PATH=/usr/lib64/liblapack.so \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_DIST_KVSTORE=1 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_LAPACK=1 \
USE_LAPACK_PATH=/usr/lib64/liblapack.so \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_DIST_KVSTORE=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand Down Expand Up @@ -520,19 +523,19 @@ build_ubuntu_gpu_tensorrt() {

rm -rf build
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_OPENCV=0 \
USE_DIST_KVSTORE=0 \
USE_TENSORRT=1 \
USE_JEMALLOC=0 \
USE_GPERFTOOLS=0 \
ONNX_NAMESPACE=onnx \
CUDA_ARCH="-gencode arch=compute_70,code=compute_70"\
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_OPENCV=0 \
USE_DIST_KVSTORE=0 \
USE_TENSORRT=1 \
USE_JEMALLOC=0 \
USE_GPERFTOOLS=0 \
ONNX_NAMESPACE=onnx \
CUDA_ARCH="-gencode arch=compute_70,code=compute_70" \
-j$(nproc)
}

Expand All @@ -542,14 +545,15 @@ build_ubuntu_gpu_mkldnn() {
build_ccache_wrappers

make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand All @@ -559,13 +563,14 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
build_ccache_wrappers

make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=0 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=0 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand All @@ -574,14 +579,15 @@ build_ubuntu_gpu_cuda91_cudnn7() {
# unfortunately this build has problems in 3rdparty dependencies with ccache and make
# build_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_CPP_PACKAGE=1 \
USE_DIST_KVSTORE=1 \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_CPP_PACKAGE=1 \
USE_DIST_KVSTORE=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
}

Expand All @@ -608,15 +614,17 @@ build_ubuntu_gpu_cmake_mkldnn() {
set -ex
cd /work/build
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=1 \
-DUSE_MKLDNN=1 \
-DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=1 \
-DUSE_MKLDNN=1 \
-DCMAKE_BUILD_TYPE=Release \
-DCUDA_ARCH_NAME=Manual \
-DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \
-G Ninja \
/work/mxnet

ninja -v
Expand All @@ -629,16 +637,18 @@ build_ubuntu_gpu_cmake() {
set -ex
cd /work/build
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=0 \
-DUSE_MKLDNN=0 \
-DUSE_DIST_KVSTORE=1 \
-DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=0 \
-DUSE_MKLDNN=0 \
-DUSE_DIST_KVSTORE=1 \
-DCMAKE_BUILD_TYPE=Release \
-DCUDA_ARCH_NAME=Manual \
-DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \
-G Ninja \
/work/mxnet

ninja -v
Expand Down