M1 fixes cherry (#6167)

* Add M1 wheels binary builds (#5948) * [M1] Set build version and delocate wheels (#6110) This would package libpng and libjpeg.dylib into wheel files Add a very simple test step, copied from https://github.com/pytorch/pytorch.github.io/blob/1eaa33a3d3f1b83b64c5031c6dd04dbb238d6105/scripts/test_install.py#L78 Cherry-picked from https://github.com/pytorch/builder/blob/d0bc74cc363a9da5a8b6a40e883d40d25d050036/build_m1_domains.sh#L22 * [BE] Unify version computation (#6117) * [BE] Unify version computation Instead of hardcoding dev version in various script, use one from `version.txt` if `setup_build_version` is called without arguments Also, pass `--pre` option to M1 build/test pip install commands to build TorchVision against nightly pytorch * Pin torchvision dependency to a specific pytorch version * [M1] Install "jpeg<=9b" rather than OpenJpeg (#6122) Explicitly set PATH to point to `conda` binary, otherwise libjpeg detection logic does not work Pin libjpeg to the same version on x86 and m1 Add simple tests that jpeg can be decoded by a generated wheel * Add unit-tests for M1 (#6132) * Add M1 testing job * libjpeg -> jpeg<=9b in test-m1.yml * Added export PATH=~/miniconda3/bin... from 6122 * Tests were OK, let's see if we can remove the pinning * GH: Add M1 conda build workflows (#6135) Clean up Conda build folder before every run Enable artifact upload to GitHub for every workflow run, but upload to Conda/S3 only on nightly pushes Test plan: `conda install -c pytorch-nightly torchvision; python -c "import torchvision;print(torchvision.io.read_image('hummingbird.jpg').shape)"` * Fix `Test M1` workflow By passing `--pre` option to `pip install`, otherwise torchvision were always tested against last PyTorch release * Adding tagged builds for M1 (#6140) * Adding tagged builds * Testing * Testing * Testing * Testing * Adding conda builds * Fix `if` condition for s3/conda uploads (#6146) Replace `steps.extract_branch.outputs.branch` (which were probably taken from https://stackoverflow.com/questions/58033366/how-to-get-the-current-branch-within-github-actions ) with straightforward `github.event.ref` Tested in https://github.com/malfet/deleteme/runs/6822647720?check_suite_focus=true and https://github.com/malfet/deleteme/runs/6822691158?check_suite_focus=true * Fix typo in GHA nightly build condition (#6158) s#ref/heads/#refs/heads/# I should have noticed it while copy-n-pasting the condition. Unfortunately there are no way to test is other than in prod, but nightly builds are still not getting pushed, see https://github.com/pytorch/vision/runs/6860407007?check_suite_focus=true for example * Making sure we are building against release * Testing Testing Testing Testing testing Testing Testing Testing Testing Testing Testing Testing * Testing * Testing * Cleanup * Refactoring logic Co-authored-by: Nikita Shulga <[email protected]> Co-authored-by: Nicolas Hug <[email protected]>
pytorch · Jun 15, 2022 · 94c81c2 · 94c81c2
1 parent 9dd7cfd
commit 94c81c2
Show file tree

Hide file tree

Showing 6 changed files with 243 additions and 6 deletions.
diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml
@@ -0,0 +1,159 @@
+name: Build on M1
+on:
+  pull_request:
+    paths:
+      - .github/workflows/build-m1-binaries.yml
+  push:
+    branches:
+      - nightly
+      - main
+      - release/*
+    tags:
+        # NOTE: Binary build pipelines should only get triggered on release candidate builds
+        # Release candidate tags look like: v1.11.0-rc1
+        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+  workflow_dispatch:
+env:
+  CHANNEL: "nightly"
+jobs:
+  build_wheels:
+    name: "Build TorchVision M1 wheels"
+    runs-on: macos-m1
+    strategy:
+      matrix:
+        py_vers: [ "3.8", "3.9", "3.10" ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set CHANNEL (only for tagged pushes)
+        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }}
+        run: |
+          # reference ends with an RC suffix
+          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
+            echo "CHANNEL=test" >> "$GITHUB_ENV"
+          fi
+      - name: Set Release CHANNEL (for release)
+        if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'release') }}
+        run: |
+          echo "CHANNEL=test" >> "$GITHUB_ENV"
+      - name: Build TorchVision M1 wheel
+        shell: arch -arch arm64 bash {0}
+        env:
+          ENV_NAME: conda-env-${{ github.run_id }}
+          PY_VERS: ${{ matrix.py_vers }}
+        run: |
+          . ~/miniconda3/etc/profile.d/conda.sh
+          # Needed for JPEG library detection as setup.py detects conda presence by running `shlex.which('conda')`
+          export PATH=~/miniconda3/bin:$PATH
+          set -ex
+          . packaging/pkg_helpers.bash
+          # if we are uploading to test channell, our version consist only of the base: 0.x.x - no date string or suffix added
+          if [[ $CHANNEL == "test" ]]; then
+            setup_base_build_version
+          else
+            setup_build_version
+          fi
+
+          WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl
+          conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config
+          conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL}
+          conda run -p ${ENV_NAME} python3 -mpip install delocate
+          conda run -p ${ENV_NAME} python3 setup.py bdist_wheel
+          export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version:  *//')"
+          conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME}
+          conda env remove -p ${ENV_NAME}
+      - name: Test wheel
+        shell: arch -arch arm64 bash {0}
+        env:
+          ENV_NAME: conda-test-env-${{ github.run_id }}
+          PY_VERS: ${{ matrix.py_vers }}
+        run: |
+          . ~/miniconda3/etc/profile.d/conda.sh
+          set -ex
+          conda create -yp ${ENV_NAME} python=${PY_VERS} numpy
+          conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL}
+          conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl
+          # Test torch is importable, by changing cwd and running import commands
+          conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)"
+          conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchvision;print('Is torchvision useable?', all(x is not None for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]))"
+          conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)"
+          conda env remove -p ${ENV_NAME}
+      - name: Upload wheel to GitHub
+        uses: actions/upload-artifact@v3
+        with:
+          name: torchvision-py${{ matrix.py_vers }}-macos11-m1
+          path: dist/
+      - name: Upload wheel to S3
+        if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }}
+        shell: arch -arch arm64 bash {0}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
+        run: |
+          for pkg in dist/*; do
+            aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read
+          done
+  build_conda:
+    name: "Build TorchVision M1 conda packages"
+    runs-on: macos-m1
+    strategy:
+      matrix:
+        py_vers: [ "3.8", "3.9", "3.10" ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set CHANNEL (only for tagged pushes)
+        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }}
+        run: |
+          # reference ends with an RC suffix
+          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
+            echo "CHANNEL=test" >> "$GITHUB_ENV"
+          fi
+      - name: Set CHANNEL Release (for release)
+        if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'release') }}
+        run: |
+          echo "CHANNEL=test" >> "$GITHUB_ENV"
+      - name: Install conda-build and purge previous artifacts
+        shell: arch -arch arm64 bash {0}
+        run: |
+          . ~/miniconda3/etc/profile.d/conda.sh
+          conda install -yq conda-build
+          conda build purge-all
+
+      - name: Build TorchVision M1 conda package
+        shell: arch -arch arm64 bash {0}
+        env:
+          ENV_NAME: conda-env-${{ github.run_id }}
+          PYTHON_VERSION: ${{ matrix.py_vers }}
+          CU_VERSION: cpu
+        run: |
+          . ~/miniconda3/etc/profile.d/conda.sh
+          set -ex
+          . packaging/pkg_helpers.bash
+
+          if [[ $CHANNEL == "test" ]]; then
+            setup_base_build_version
+          else
+            setup_build_version
+          fi
+
+          setup_conda_pytorch_constraint
+          export SOURCE_ROOT_DIR=$(pwd)
+          conda build -c defaults $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision
+          mkdir -p dist
+          cp ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 dist/
+      - name: Upload package to GitHub
+        uses: actions/upload-artifact@v3
+        with:
+          name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda
+          path: dist/
+      - name: Upload package to conda
+        if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }}
+        shell: arch -arch arm64 bash {0}
+        env:
+          CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
+        run: |
+          . ~/miniconda3/etc/profile.d/conda.sh
+          conda install -yq anaconda-client
+          set -x
+          anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 -u "pytorch-${CHANNEL}" --label main --no-progress --force
diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml
@@ -0,0 +1,63 @@
+name: Unit-tests on M1
+on:
+  pull_request:
+    paths:
+      - .github/workflows/test-m1.yml
+  push:
+    branches:
+      - nightly
+      - main
+      - release/*
+    tags:
+        # NOTE: Binary build pipelines should only get triggered on release candidate builds
+        # Release candidate tags look like: v1.11.0-rc1
+        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+  workflow_dispatch:
+env:
+  CHANNEL: "nightly"
+jobs:
+  tests:
+    name: "Unit-tests on M1"
+    runs-on: macos-m1
+    strategy:
+      matrix:
+        py_vers: [ "3.8"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set CHANNEL (only for tagged pushes)
+        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }}
+        run: |
+          # reference ends with an RC suffix
+          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
+            echo "CHANNEL=test" >> "$GITHUB_ENV"
+          fi
+      - name: Set Release CHANNEL (for release)
+        if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release') }}
+        run: |
+          echo "CHANNEL=test" >> "$GITHUB_ENV"
+      - name: Install TorchVision
+        shell: arch -arch arm64 bash {0}
+        env:
+          ENV_NAME: conda-env-${{ github.run_id }}
+          PY_VERS: ${{ matrix.py_vers }}
+        run: |
+          . ~/miniconda3/etc/profile.d/conda.sh
+          # Needed for JPEG library detection as setup.py detects conda presence by running `shlex.which('conda')`
+          export PATH=~/miniconda3/bin:$PATH
+          set -ex
+          conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy
+          conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL}
+          conda run -p ${ENV_NAME} python3 setup.py develop
+          conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock av
+      - name: Run tests
+        shell: arch -arch arm64 bash {0}
+        env:
+          ENV_NAME: conda-env-${{ github.run_id }}
+          PY_VERS: ${{ matrix.py_vers }}
+        run: |
+          . ~/miniconda3/etc/profile.d/conda.sh
+          set -ex
+          conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20
+          conda env remove -p ${ENV_NAME}
diff --git a/packaging/build_cmake.sh b/packaging/build_cmake.sh
@@ -15,7 +15,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 . "$script_dir/pkg_helpers.bash"
 
 export BUILD_TYPE=conda
-setup_env 0.13.0
+setup_env
 export SOURCE_ROOT_DIR="$PWD"
 setup_conda_pytorch_constraint
 setup_conda_cudatoolkit_plain_constraint

diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh
@@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 . "$script_dir/pkg_helpers.bash"
 
 export BUILD_TYPE=conda
-setup_env 0.13.0
+setup_env
 export SOURCE_ROOT_DIR="$PWD"
 setup_conda_pytorch_constraint
 setup_conda_cudatoolkit_constraint

diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh
@@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 . "$script_dir/pkg_helpers.bash"
 
 export BUILD_TYPE=wheel
-setup_env 0.13.0
+setup_env
 setup_wheel_python
 pip_install numpy pyyaml future ninja
 pip_install --upgrade setuptools

diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
@@ -100,16 +100,31 @@ setup_cuda() {
 # Usage: setup_build_version 0.2.0
 setup_build_version() {
   if [[ -z "$BUILD_VERSION" ]]; then
-    export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX"
+    if [[ -z "$1" ]]; then
+      setup_base_build_version
+    else
+      BUILD_VERSION="$1"
+    fi
+    BUILD_VERSION="$BUILD_VERSION.dev$(date "+%Y%m%d")$VERSION_SUFFIX"
   else
-    export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
+    BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
   fi
 
   # Set build version based on tag if on tag
   if [[ -n "${CIRCLE_TAG}" ]]; then
     # Strip tag
-    export BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}"
+    BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}"
   fi
+
+  export BUILD_VERSION
+}
+
+setup_base_build_version() {
+  SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  # version.txt for some reason has `a` character after major.minor.rev
+  # command below yields 0.10.0 from version.txt containing 0.10.0a0
+  BUILD_VERSION=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" )
+  export BUILD_VERSION
 }
 
 # Set some useful variables for OS X, if applicable