From 9285cae81e06139cccdadb5b4ecd766cf01bb3d9 Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Wed, 30 Jun 2021 20:39:58 -0700 Subject: [PATCH 01/12] Test py3.7 image --- tools/docker/build_wheel.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index 8836afd536..caed262d7a 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -1,7 +1,7 @@ #syntax=docker/dockerfile:1.1.5-experimental ARG TF_VERSION ARG PY_VERSION -FROM gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu18.04-manylinux2010-multipython as base_install +FROM angersson/tensorflow-build:latest-python3.7 as base_install ENV TF_NEED_CUDA="1" # Required for setuptools v50.0.0 From b6a9d8c6bd12a44779207291cee35c3d54036985 Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Thu, 1 Jul 2021 09:37:27 -0700 Subject: [PATCH 02/12] Temp --- tools/docker/build_wheel.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index caed262d7a..7dbf0a43f8 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -16,7 +16,7 @@ RUN cat /usr/bin/lsb_release >> /usr/bin/lsb_release2 RUN mv /usr/bin/lsb_release2 /usr/bin/lsb_release ARG PY_VERSION -RUN ln -sf /usr/local/bin/python$PY_VERSION /usr/bin/python +#RUN ln -sf /usr/local/bin/python$PY_VERSION /usr/bin/python ARG TF_VERSION RUN python -m pip install --default-timeout=1000 tensorflow==$TF_VERSION From 8cabc9fdf30986c4688d405d8e40d482f673cf5a Mon Sep 17 00:00:00 2001 From: bhack Date: Mon, 5 Jul 2021 20:03:07 +0200 Subject: [PATCH 03/12] Test with variable --- tools/docker/build_wheel.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index 7dbf0a43f8..2c5f50614b 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -1,7 +1,7 @@ #syntax=docker/dockerfile:1.1.5-experimental ARG TF_VERSION ARG PY_VERSION -FROM angersson/tensorflow-build:latest-python3.7 as base_install +FROM angersson/tensorflow-build:latest-python$PY_VERSION as base_install ENV TF_NEED_CUDA="1" # Required for setuptools v50.0.0 @@ -11,7 +11,7 @@ ENV SETUPTOOLS_USE_DISTUTILS=stdlib # Fix presented in # https://stackoverflow.com/questions/44967202/pip-is-showing-error-lsb-release-a-returned-non-zero-exit-status-1/44967506 -RUN echo "#! /usr/bin/python2.7" >> /usr/bin/lsb_release2 +RUN echo "#! /usr/bin/python$PY_VERSION" >> /usr/bin/lsb_release2 RUN cat /usr/bin/lsb_release >> /usr/bin/lsb_release2 RUN mv /usr/bin/lsb_release2 /usr/bin/lsb_release From 0bf6e541e66b4e921c7d66e94d72b137ac0f8abf Mon Sep 17 00:00:00 2001 From: bhack Date: Fri, 23 Jul 2021 13:01:54 +0200 Subject: [PATCH 04/12] Switch to the official dockerhub repo --- tools/docker/build_wheel.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index 2c5f50614b..db41a1a3fd 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -1,7 +1,7 @@ #syntax=docker/dockerfile:1.1.5-experimental ARG TF_VERSION ARG PY_VERSION -FROM angersson/tensorflow-build:latest-python$PY_VERSION as base_install +FROM tensorflow/build:latest-python$PY_VERSION as base_install ENV TF_NEED_CUDA="1" # Required for setuptools v50.0.0 From 967ff2445b1cc8c6ca8c7fa0a828ba79ffccf5aa Mon Sep 17 00:00:00 2001 From: bhack Date: Mon, 22 Nov 2021 21:34:18 +0100 Subject: [PATCH 05/12] Remove commented line Test again for https://discuss.tensorflow.org/t/adopting-open-source-dockerfiles-for-official-tf-nightly-ci/6050 --- tools/docker/build_wheel.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index db41a1a3fd..f3e09177f8 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -16,7 +16,6 @@ RUN cat /usr/bin/lsb_release >> /usr/bin/lsb_release2 RUN mv /usr/bin/lsb_release2 /usr/bin/lsb_release ARG PY_VERSION -#RUN ln -sf /usr/local/bin/python$PY_VERSION /usr/bin/python ARG TF_VERSION RUN python -m pip install --default-timeout=1000 tensorflow==$TF_VERSION From 7d4925fb1f1fc6c3ae504252c7e324389b4a8b1e Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Wed, 22 Dec 2021 21:34:46 -0800 Subject: [PATCH 06/12] * Clean-up --- tools/docker/build_wheel.Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index a1df2a93fa..1b204d9981 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -9,14 +9,7 @@ ENV TF_NEED_CUDA="1" # https://github.com/pypa/setuptools/issues/2352 ENV SETUPTOOLS_USE_DISTUTILS=stdlib -# Fix presented in -# https://stackoverflow.com/questions/44967202/pip-is-showing-error-lsb-release-a-returned-non-zero-exit-status-1/44967506 -RUN echo "#! /usr/bin/python$PY_VERSION" >> /usr/bin/lsb_release2 -RUN cat /usr/bin/lsb_release >> /usr/bin/lsb_release2 -RUN mv /usr/bin/lsb_release2 /usr/bin/lsb_release - ARG PY_VERSION - ARG TF_VERSION RUN python -m pip install --default-timeout=1000 tensorflow==$TF_VERSION From d039701e8f5f8bde83a00c3572e5760d574bf9ec Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Wed, 22 Dec 2021 22:13:44 -0800 Subject: [PATCH 07/12] * Clean-up --- tools/docker/build_wheel.Dockerfile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index 1b204d9981..a3f69faeff 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -1,14 +1,8 @@ #syntax=docker/dockerfile:1.1.5-experimental -ARG TF_VERSION ARG PY_VERSION FROM tensorflow/build:latest-python$PY_VERSION as base_install -ENV TF_NEED_CUDA="1" - -# Required for setuptools v50.0.0 -# https://setuptools.readthedocs.io/en/latest/history.html#v50-0-0 -# https://github.com/pypa/setuptools/issues/2352 -ENV SETUPTOOLS_USE_DISTUTILS=stdlib +ENV TF_NEED_CUDA="1" ARG PY_VERSION ARG TF_VERSION RUN python -m pip install --default-timeout=1000 tensorflow==$TF_VERSION @@ -20,10 +14,9 @@ COPY requirements.txt . RUN python -m pip install -r requirements.txt COPY ./ /addons -RUN rm /addons/.bazeliskrc WORKDIR /addons -# ------------------------------------------------------------------- +# -------------------------------------------------------------------K FROM base_install as tfa_gpu_tests CMD ["bash", "tools/testing/build_and_run_tests.sh"] From 626df2c806dee487a42023c0a25971f5bc36c8ac Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Wed, 22 Dec 2021 22:16:08 -0800 Subject: [PATCH 08/12] * Clean-up --- tools/docker/build_wheel.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index a3f69faeff..4ad7e610fc 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -16,7 +16,7 @@ RUN python -m pip install -r requirements.txt COPY ./ /addons WORKDIR /addons -# -------------------------------------------------------------------K +# ------------------------------------------------------------------- FROM base_install as tfa_gpu_tests CMD ["bash", "tools/testing/build_and_run_tests.sh"] From a2e63dc8726b3c90308fac6649ffa848a5a63e5d Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Wed, 22 Dec 2021 22:52:49 -0800 Subject: [PATCH 09/12] * Standardize the dev container --- .github/workflows/release.yml | 2 ++ tools/build_dev_container.sh | 1 + tools/docker/dev_container.Dockerfile | 6 ++---- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9e71f9c50..5bf71ca429 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,6 +124,8 @@ jobs: name: Upload dev container to DockerHub needs: [release-wheel, test-with-bazel] runs-on: ubuntu-18.04 + env: + PY_VERSION: ${{ env.MAX_PY_VERSION }} if: (github.event_name == 'push' && github.ref == 'refs/heads/master') steps: - uses: actions/checkout@v2 diff --git a/tools/build_dev_container.sh b/tools/build_dev_container.sh index 59cf84e7f7..542e26c3f8 100755 --- a/tools/build_dev_container.sh +++ b/tools/build_dev_container.sh @@ -6,6 +6,7 @@ docker build \ -f tools/docker/dev_container.Dockerfile \ --build-arg TF_VERSION=2.7.0 \ --build-arg TF_PACKAGE=tensorflow-cpu \ + --build-arg PY_VERSION=$PY_VERSION \ --no-cache \ --target dev_container_cpu \ -t tfaddons/dev_container:latest-cpu ./ diff --git a/tools/docker/dev_container.Dockerfile b/tools/docker/dev_container.Dockerfile index 2477b682e7..2a4902b030 100644 --- a/tools/docker/dev_container.Dockerfile +++ b/tools/docker/dev_container.Dockerfile @@ -1,11 +1,9 @@ #syntax=docker/dockerfile:1.1.5-experimental -FROM gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu18.04-manylinux2010-multipython as dev_container_cpu +ARG PY_VERSION +FROM tensorflow/build:latest-python$PY_VERSION as dev_container_cpu ARG TF_PACKAGE ARG TF_VERSION -RUN ln -sf /usr/local/bin/python3.9 /usr/bin/python -RUN ln -sf /usr/local/bin/pip3.9 /usr/local/bin/pip - RUN pip install --default-timeout=1000 $TF_PACKAGE==$TF_VERSION COPY tools/install_deps /install_deps From b206b1eb244ad277850f682c60b81223ad70f4fb Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Wed, 22 Dec 2021 23:02:21 -0800 Subject: [PATCH 10/12] * Standardize the dev container --- .github/workflows/release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5bf71ca429..6551a572bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,11 +124,12 @@ jobs: name: Upload dev container to DockerHub needs: [release-wheel, test-with-bazel] runs-on: ubuntu-18.04 - env: - PY_VERSION: ${{ env.MAX_PY_VERSION }} + if: (github.event_name == 'push' && github.ref == 'refs/heads/master') steps: - uses: actions/checkout@v2 + - env: + PY_VERSION: ${{ env.MAX_PY_VERSION }} - run: | set -e -x echo ${{ secrets.DOCKER_PW }} | docker login --username ${{ secrets.DOCKER_USER }} --password-stdin From 49c8d98e79e07341bc18368e2cfa3abab623b184 Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Wed, 22 Dec 2021 23:07:17 -0800 Subject: [PATCH 11/12] * Standardize the dev container --- .github/workflows/release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6551a572bd..7a0650fd54 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,12 +124,11 @@ jobs: name: Upload dev container to DockerHub needs: [release-wheel, test-with-bazel] runs-on: ubuntu-18.04 - + env: + PY_VERSION: '3.9' if: (github.event_name == 'push' && github.ref == 'refs/heads/master') steps: - uses: actions/checkout@v2 - - env: - PY_VERSION: ${{ env.MAX_PY_VERSION }} - run: | set -e -x echo ${{ secrets.DOCKER_PW }} | docker login --username ${{ secrets.DOCKER_USER }} --password-stdin From 89063e166e1306f49dee599b9e1c703f325086f2 Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Mon, 27 Dec 2021 17:28:43 -0800 Subject: [PATCH 12/12] * Update dev container to be GPU --- .github/workflows/release.yml | 2 +- tools/build_dev_container.sh | 6 +++--- tools/docker/dev_container.Dockerfile | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a0650fd54..706eae805d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -133,4 +133,4 @@ jobs: set -e -x echo ${{ secrets.DOCKER_PW }} | docker login --username ${{ secrets.DOCKER_USER }} --password-stdin bash .github/workflows/github_build_dev_container.sh - docker push tfaddons/dev_container:latest-cpu + docker push tfaddons/dev_container:latest-gpu diff --git a/tools/build_dev_container.sh b/tools/build_dev_container.sh index 542e26c3f8..4d57eeab0d 100755 --- a/tools/build_dev_container.sh +++ b/tools/build_dev_container.sh @@ -5,8 +5,8 @@ set -x -e docker build \ -f tools/docker/dev_container.Dockerfile \ --build-arg TF_VERSION=2.7.0 \ - --build-arg TF_PACKAGE=tensorflow-cpu \ + --build-arg TF_PACKAGE=tensorflow \ --build-arg PY_VERSION=$PY_VERSION \ --no-cache \ - --target dev_container_cpu \ - -t tfaddons/dev_container:latest-cpu ./ + --target dev_container \ + -t tfaddons/dev_container:latest-gpu ./ diff --git a/tools/docker/dev_container.Dockerfile b/tools/docker/dev_container.Dockerfile index 2a4902b030..68ba49ee86 100644 --- a/tools/docker/dev_container.Dockerfile +++ b/tools/docker/dev_container.Dockerfile @@ -1,6 +1,10 @@ #syntax=docker/dockerfile:1.1.5-experimental ARG PY_VERSION -FROM tensorflow/build:latest-python$PY_VERSION as dev_container_cpu +ARG IMAGE_TYPE + +# Currenly all of our dev images are GPU capable but at a cost of being quite large. +# See https://github.com/tensorflow/build/pull/47 +FROM tensorflow/build:latest-python$PY_VERSION as dev_container ARG TF_PACKAGE ARG TF_VERSION