Fix mxnet-native and Docker CD pipelines (apache#17784)

* Fix Jenkinsfile CD pipeline for mxnet-native * Fix cd/python/docker/python_images.sh
MoisesHer · Apr 10, 2020 · 161ee44 · 161ee44
1 parent 7c33239
commit 161ee44
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 27 deletions.
diff --git a/cd/Jenkinsfile_cd_pipeline b/cd/Jenkinsfile_cd_pipeline
@@ -31,7 +31,7 @@ pipeline {
     // Because each pass of the CD pipeline
     // updates Jenkins' state of the release job
     // to avoid crazy issues, we don't allow concurrent builds.
-    disableConcurrentBuilds()  
+    disableConcurrentBuilds()
   }
 
   parameters {
@@ -45,7 +45,6 @@ pipeline {
       steps {
         script {
           cd_utils = load('cd/Jenkinsfile_utils.groovy')
-
           // Update release job state in Jenkins
           cd_utils.update_release_job_state()
         }
@@ -59,7 +58,7 @@ pipeline {
 
             "Static libmxnet based release": {
               stage("Build") {
-                cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)    
+                cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)
               }
               stage("Releases") {
                 cd_utils.error_checked_parallel([
@@ -77,10 +76,9 @@ pipeline {
 
             "Dynamic libmxnet based release": {
               stage("Build") {
-                cd_utils.trigger_release_job("Build dynamic libmxnet", "mxnet_lib/dynamic", params.MXNET_VARIANTS)    
+                cd_utils.trigger_release_job("Build dynamic libmxnet", "mxnet_lib/dynamic", params.MXNET_VARIANTS)
               }
             }
-
           ])
         }
       }

diff --git a/cd/python/docker/python_images.sh b/cd/python/docker/python_images.sh
@@ -45,7 +45,7 @@ fi
 
 build() {
     # NOTE: Ensure the correct context root is passed in when building - Dockerfile expects ./wheel_build
-    docker build -t "${image_name}" --build-arg --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
+    docker build -t "${image_name}" --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
 }
 
 test() {

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
@@ -155,17 +155,6 @@ gather_licenses() {
 }
 
 build_ubuntu_cpu_release() {
-    set -ex
-    cd /work/build
-    cmake \
-        -DUSE_MKL_IF_AVAILABLE=OFF \
-        -DUSE_MKLDNN=OFF \
-        -DUSE_CUDA=OFF \
-        -G Ninja /work/mxnet
-    ninja
-}
-
-build_ubuntu_cpu_mkldnn_release() {
     set -ex
     cd /work/build
     cmake \
@@ -176,20 +165,18 @@ build_ubuntu_cpu_mkldnn_release() {
     ninja
 }
 
-build_ubuntu_gpu_release() {
+build_ubuntu_cpu_native_release() {
     set -ex
     cd /work/build
     cmake \
         -DUSE_MKL_IF_AVAILABLE=OFF \
         -DUSE_MKLDNN=OFF \
-        -DUSE_DIST_KVSTORE=ON \
-        -DUSE_CUDA=ON \
-        -DMXNET_CUDA_ARCH="$CI_CMAKE_CUDA_ARCH" \
+        -DUSE_CUDA=OFF \
         -G Ninja /work/mxnet
     ninja
 }
 
-build_ubuntu_gpu_mkldnn_release() {
+build_ubuntu_gpu_release() {
     set -ex
     cd /work/build
     cmake \
@@ -204,7 +191,7 @@ build_ubuntu_gpu_mkldnn_release() {
 
 # Compiles the dynamic mxnet library
 # Parameters:
-# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, cu100, cu92mkl, etc.
+# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, native, cu100, cu92, etc.
 build_dynamic_libmxnet() {
     set -ex
 
@@ -215,14 +202,13 @@ build_dynamic_libmxnet() {
 
     if [[ ${mxnet_variant} = "cpu" ]]; then
         build_ubuntu_cpu_release
-    elif [[ ${mxnet_variant} = "mkl" ]]; then
-        build_ubuntu_cpu_mkldnn_release
+    elif [[ ${mxnet_variant} = "native" ]]; then
+        build_ubuntu_cpu_native_release
     elif [[ ${mxnet_variant} =~ cu[0-9]+$ ]]; then
         build_ubuntu_gpu_release
-    elif [[ ${mxnet_variant} =~ cu[0-9]+mkl$ ]]; then
-        build_ubuntu_gpu_mkldnn_release
     else
         echo "Error: Unrecognized mxnet variant '${mxnet_variant}'"
+        exit 1
     fi
 }
 

diff --git a/config/distribution/linux_native.cmake b/config/distribution/linux_native.cmake
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(CMAKE_BUILD_TYPE "Distribution" CACHE STRING "Build type")
+set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
+set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
+
+set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
+set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
+set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
+set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
+set(USE_MKLDNN OFF CACHE BOOL "Build with MKL-DNN support")
+set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
+set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
+set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
+set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")