Update torch requirement from <1.10.0,>=1.6.0 to >=1.6.0,<1.11.0

.github/workflows/ci.yml

@@ -17,16 +17,23 @@ on:
   - cron: '37 11 * * 1,2,3,4,5'  # early morning (11:37 UTC / 4:37 AM PDT) Monday - Friday
 
 env:
-  # NOTE: Need to update `TORCH_VERSION`, `TORCH_CPU_INSTALL` and `TORCH_GPU_INSTALL` for new torch releases.
-  TORCH_VERSION: 1.9.0
-  TORCH_CPU_INSTALL: pip install torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
-  TORCH_GPU_INSTALL: pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html # Our self-hosted GPU runners currently support CUDA 11.*
+  # NOTE: Need to update `TORCH_VERSION`, and `TORCH_*_INSTALL` for new torch releases.
+  TORCH_VERSION: 1.10.0
+  TORCH_CPU_INSTALL: conda install pytorch torchvision torchaudio cpuonly -c pytorch
+  TORCH_GPU_INSTALL: conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
   # Change this to invalidate existing cache.
-  CACHE_PREFIX: v2
+  CACHE_PREFIX: v4
   # Disable tokenizers parallelism because this doesn't help, and can cause issues in distributed tests.
   TOKENIZERS_PARALLELISM: 'false'
   # Disable multithreading with OMP because this can lead to dead-locks in distributed tests.
   OMP_NUM_THREADS: '1'
+  # See https://github.com/pytorch/pytorch/issues/37377#issuecomment-677851112.
+  MKL_THREADING_LAYER: 'GNU'
+  DEFAULT_PYTHON_VERSION: 3.8
+
+defaults:
+  run:
+    shell: bash -l {0}
 
 jobs:
   changelog:
@@ -58,7 +65,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Install requirements
       run: |
@@ -112,20 +119,12 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      env:
-        # Log useful debugging information.
-        ACTIONS_STEP_DEBUG: 'true'
+    - uses: conda-incubator/setup-miniconda@v2
       with:
-        python-version: 3.8
-
-    - name: Install prerequisites
-      run: |
-        pip install --upgrade pip setuptools wheel virtualenv
+        miniconda-version: "latest"
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Set build variables
-      shell: bash
       run: |
         # Get the exact Python version to use in the cache key.
         echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
@@ -148,20 +147,18 @@ jobs:
       with:
         path: .venv
         key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ matrix.task.torch_platform }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ matrix.task.torch_platform }}-
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
       run: |
-        test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
-        . .venv/bin/activate
+        conda create -y -p .venv python=${{ env.DEFAULT_PYTHON_VERSION }}
+        conda activate ./.venv
         make install TORCH_INSTALL="$TORCH_INSTALL"
 
     - name: Setup virtual environment (cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit == 'true'
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip install --no-deps -e .
         make download-extras
 
@@ -170,25 +167,25 @@ jobs:
       env:
         ALLENNLP_VERSION_OVERRIDE: ""  # Don't replace the core library.
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         git clone https://github.com/allenai/allennlp-models.git
         cd allennlp-models
-        pip install --upgrade --upgrade-strategy eager -e . -r dev-requirements.txt
+        git checkout dependabot/pip/torch-gte-1.7.0-and-lt-1.11.0
+        pip install -e . -r dev-requirements.txt
 
     - name: Debug info
       run: |
-        . .venv/bin/activate
-        which python
-        pip freeze
+        conda activate ./.venv
+        conda list
 
     - name: Ensure torch up-to-date
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         python scripts/check_torch_version.py
 
     - name: ${{ matrix.task.name }}
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         ${{ matrix.task.run }}
 
     - name: Prepare coverage report
@@ -209,7 +206,7 @@ jobs:
       run: |
         # Could run into issues with the cache if we don't uninstall the editable.
         # See https://github.com/pypa/pip/issues/4537.
-        . .venv/bin/activate
+        conda activate ./.venv
         pip uninstall --yes allennlp allennlp-models
 
   upload_coverage:
@@ -259,58 +256,48 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      env:
-        # Log useful debugging information.
-        ACTIONS_STEP_DEBUG: 'true'
+    - uses: conda-incubator/setup-miniconda@v2
       with:
-        python-version: 3.8
-
-    - name: Install prerequisites
-      run: |
-        pip install --upgrade pip setuptools wheel virtualenv
+        miniconda-version: "latest"
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Set build variables
-      shell: bash
       run: |
         echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
         echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
         echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
+        echo "TORCH_INSTALL=$TORCH_CPU_INSTALL" >> $GITHUB_ENV
 
     - uses: actions/cache@v2
       id: virtualenv-cache
       with:
         path: .venv
         key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
       run: |
-        test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
-        . .venv/bin/activate
-        make install TORCH_INSTALL="$TORCH_CPU_INSTALL"
+        conda create -y -p .venv python=${{ env.DEFAULT_PYTHON_VERSION }}
+        conda activate ./.venv
+        make install TORCH_INSTALL="$TORCH_INSTALL"
 
     - name: Setup virtual environment (cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit == 'true'
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip install --no-deps -e .
         make download-extras
 
     - name: Debug info
       run: |
-        . .venv/bin/activate
-        which python
+        conda activate ./.venv
         pip freeze
 
     - name: Check and set nightly version
       if: github.event_name == 'schedule'
       run: |
         # Verify that current version is ahead of the last release.
-        . .venv/bin/activate
+        conda activate ./.venv
         LATEST=$(scripts/get_version.py latest)
         CURRENT=$(scripts/get_version.py current)
         if [ "$CURRENT" == "$LATEST" ]; then
@@ -326,7 +313,7 @@ jobs:
       if: github.event_name == 'release'
       run: |
         # Remove 'refs/tags/' to get the actual tag from the release.
-        . .venv/bin/activate
+        conda activate ./.venv
         TAG=${GITHUB_REF#refs/tags/};
         VERSION=$(scripts/get_version.py current)
         if [ "$TAG" != "$VERSION" ]; then
@@ -337,7 +324,7 @@ jobs:
     - name: Build core package
       run: |
         # Just print out the version for debugging.
-        . .venv/bin/activate
+        conda activate ./.venv
         make version
         python setup.py bdist_wheel sdist
 
@@ -350,7 +337,7 @@ jobs:
     - name: Clean up
       if: always()
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip uninstall --yes allennlp
 
   # Tests installing from the distribution files.
@@ -365,7 +352,7 @@ jobs:
 
     steps:
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python }}
 
@@ -399,7 +386,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        cuda: ['10.2', '11.1']
+        cuda: ['11.3']
 
     steps:
     - uses: actions/checkout@v2
@@ -434,15 +421,15 @@ jobs:
         docker push $DOCKER_IMAGE_NAME
 
     - name: Upload default commit image
-      # CUDA 10.2 is currently our default.
-      if: github.event_name == 'push' && matrix.cuda == '10.2'
+      # CUDA 11.3 is currently our default.
+      if: github.event_name == 'push' && matrix.cuda == '11.3'
       run: |
         docker tag $DOCKER_IMAGE_NAME allennlp/commit:${GITHUB_SHA}
         docker push allennlp/commit:${GITHUB_SHA}
 
     - name: Upload latest image
-      # CUDA 10.2 is currently our default.
-      if: github.event_name == 'release' && matrix.cuda == '10.2'
+      # CUDA 11.3 is currently our default.
+      if: github.event_name == 'release' && matrix.cuda == '11.3'
       run: |
         docker tag $DOCKER_IMAGE_NAME allennlp/allennlp:latest
         docker push allennlp/allennlp:latest
@@ -464,52 +451,42 @@ jobs:
       with:
         ssh-private-key: ${{ secrets.DOCS_DEPLOY_KEY }}
 
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      env:
-        # Log useful debugging information.
-        ACTIONS_STEP_DEBUG: 'true'
+    - uses: conda-incubator/setup-miniconda@v2
       with:
-        python-version: 3.8
-
-    - name: Install prerequisites
-      run: |
-        pip install --upgrade pip setuptools wheel virtualenv
+        miniconda-version: "latest"
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Set build variables
-      shell: bash
       run: |
         echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
         echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
         echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
+        echo "TORCH_INSTALL=$TORCH_CPU_INSTALL" >> $GITHUB_ENV
 
     - uses: actions/cache@v2
       id: virtualenv-cache
       with:
         path: .venv
         key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
       run: |
-        test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
-        . .venv/bin/activate
-        make install TORCH_INSTALL="$TORCH_CPU_INSTALL"
+        conda create -y -p .venv python=${{ env.DEFAULT_PYTHON_VERSION }}
+        conda activate ./.venv
+        make install TORCH_INSTALL="$TORCH_INSTALL"
 
     - name: Setup virtual environment (cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit == 'true'
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip install --no-deps -e .
         make download-extras
 
     - name: Debug info
       run: |
-        . .venv/bin/activate
-        which python
-        pip freeze
+        conda activate ./.venv
+        conda list
 
     - name: Prepare environment
       run: |
@@ -523,7 +500,7 @@ jobs:
 
     - name: Build docs
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         ./scripts/build_docs.sh
 
     - name: Print the ref
@@ -555,7 +532,7 @@ jobs:
       run: |
         # Fail immediately if any step fails.
         set -e
-        . .venv/bin/activate
+        conda activate ./.venv
 
         LATEST=$(./scripts/get_version.py latest)
         STABLE=$(./scripts/get_version.py stable)
@@ -613,7 +590,7 @@ jobs:
     - name: Clean up
       if: always()
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip uninstall --yes allennlp
 
   # Publish the core distribution files to PyPI.
@@ -629,9 +606,9 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Install requirements
       run: |

Dockerfile

@@ -2,9 +2,13 @@
 # It's built from a wheel installation of allennlp using the base images from
 # https://github.com/allenai/docker-images/pkgs/container/pytorch
 
-ARG TORCH=1.9.0-cuda10.2
-ARG PYTHON=3.9
-FROM ghcr.io/allenai/pytorch:${TORCH}-python${PYTHON}-v0.0.1
+ARG TORCH=1.10.0-cuda11.3
+FROM ghcr.io/allenai/pytorch:${TORCH}
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /stage/allennlp
 

Dockerfile.test

@@ -1,8 +1,12 @@
 # Used to build an image for running tests.
 
-ARG TORCH=1.9.0-cuda10.2
-ARG PYTHON=3.9
-FROM ghcr.io/allenai/pytorch:${TORCH}-python${PYTHON}-v0.0.1
+ARG TORCH=1.10.0-cuda11.3
+FROM ghcr.io/allenai/pytorch:${TORCH}
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # These environment variables are helpful for debugging.
 # See https://pytorch.org/docs/stable/distributed.html#common-environment-variables for more info.