[MXNET-951] Python dockerfiles built on pip binaries and build/release script

docker/docker-python/Dockerfile.mxnet.python.cpu

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for CPU
+
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet

docker/docker-python/Dockerfile.mxnet.python.cpu.mkl

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet CPU with MKL
+
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-mkl

docker/docker-python/Dockerfile.mxnet.python.gpu.cu80

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu80

docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu80mkl

docker/docker-python/Dockerfile.mxnet.python.gpu.cu90

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu90

docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu90mkl

docker/docker-python/Dockerfile.mxnet.python.gpu.cu92

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu92

docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl

@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu92mkl

docker/docker-python/README.md

@@ -0,0 +1,30 @@
+# Release Python Docker Images for MXNet
+
+The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release.  
+It uses the appropriate pip binaries to build different docker images as -
+* cpu
+* cpu_mkl
+* latest (same as cpu)
+* gpu_cu90 
+* gpu_cu90_mkl
+* gpu (same as gpu_cu90)
+
+
+** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. 
+
+Refer: https://pypi.org/project/mxnet/
+
+### Usage
+`./build_python_dockerfile.sh <mxnet_version> <path_to_cloned_mxnet_repo>`
+
+For example: 
+`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet`
+
+** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly.  
+
+### Tests run
+* `/mxnet/tests/python/train/test_conv.py`
+* `/mxnet/example/image-classification/train_mnist.py`
+* `/mxnet/docker/docker-python/test_mxnet.py`: This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. 
+
+Dockerhub credentials will be required to upload images at the end of this script. 

docker/docker-python/build_python_dockerfile.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# Check Params
+programname=$0
+
+function usage {
+    echo "usage: $programname [version] [path]"
+    echo "  [version]  Mxnet Version to build"
+    echo "  [path]     Path to MXNet repository (to run tests)"
+    echo " "
+    exit 1
+}
+
+if [ $# -le 1 ] || [ $# -ge 3 ]
+then
+    usage
+    exit 1
+fi
+
+# Two params provided
+echo "Building Docker Images for Apache MXNet (Incubating) v$1"
+mxnet_version="${1}"
+test_dir="${2}"
+
+docker_build_image(){
+    echo "Building docker image mxnet/python:${1}"
+    docker build -t mxnet/python:${1} -f ${2} .
+}
+
+docker_tag_image(){
+    docker tag mxnet/python:${1} mxnet/python:${2}
+}
+
+docker_test_image_cpu(){
+    echo "Running tests on mxnet/python:${1}"
+    docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+    docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py"
+    docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py"
+}
+
+docker_test_image_gpu(){
+    echo "Running tests on mxnet/python:${1}"
+    nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+    nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu"
+    nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2"
+}
+
+docker_account_login(){
+    docker login
+}
+
+docker_account_logout(){
+    docker logout
+}
+
+docker_push_image(){
+    docker push mxnet/python:${1}
+}
+
+
+# Build and Test dockerfiles - CPU
+docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu"
+docker_test_image_cpu "${mxnet_version}_cpu"
+
+docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl"
+docker_test_image_cpu "${mxnet_version}_cpu_mkl"
+
+docker_tag_image "${mxnet_version}_cpu" "latest"
+docker_test_image_cpu "latest"
+
+
+#Build and Test dockerfiles - GPU
+docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90"
+docker_test_image_gpu "${mxnet_version}_gpu_cu90"
+
+docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl"
+
+docker_tag_image "${mxnet_version}_gpu_cu90" "gpu"
+docker_test_image_gpu "gpu"
+
+docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80"
+docker_test_image_gpu "${mxnet_version}_gpu_cu80"
+
+docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl"
+
+docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92"
+docker_test_image_gpu "${mxnet_version}_gpu_cu92"
+
+docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl"
+
+
+# Push dockerfiles
+echo "All images were successfully built. Now login to dockerhub and push images"
+docker_account_login
+
+docker_push_image "${mxnet_version}_cpu"
+docker_push_image "${mxnet_version}_cpu_mkl"
+docker_push_image "latest"
+docker_push_image "${mxnet_version}_gpu_cu90"
+docker_push_image "${mxnet_version}_gpu_cu90_mkl"
+docker_push_image "gpu"
+docker_push_image "${mxnet_version}_gpu_cu80"
+docker_push_image "${mxnet_version}_gpu_cu80_mkl"
+docker_push_image "${mxnet_version}_gpu_cu92"
+docker_push_image "${mxnet_version}_gpu_cu92_mkl"
+
+docker_account_logout
+
+echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/"