diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 719e14a1001..389e758f281 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,16 +17,23 @@ on:
   - cron: '37 11 * * 1,2,3,4,5'  # early morning (11:37 UTC / 4:37 AM PDT) Monday - Friday
 
 env:
-  # NOTE: Need to update `TORCH_VERSION`, `TORCH_CPU_INSTALL` and `TORCH_GPU_INSTALL` for new torch releases.
-  TORCH_VERSION: 1.9.0
-  TORCH_CPU_INSTALL: pip install torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
-  TORCH_GPU_INSTALL: pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html # Our self-hosted GPU runners currently support CUDA 11.*
+  # NOTE: Need to update `TORCH_VERSION`, and `TORCH_*_INSTALL` for new torch releases.
+  TORCH_VERSION: 1.10.0
+  TORCH_CPU_INSTALL: conda install pytorch torchvision torchaudio cpuonly -c pytorch
+  TORCH_GPU_INSTALL: conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
   # Change this to invalidate existing cache.
-  CACHE_PREFIX: v2
+  CACHE_PREFIX: v4
   # Disable tokenizers parallelism because this doesn't help, and can cause issues in distributed tests.
   TOKENIZERS_PARALLELISM: 'false'
   # Disable multithreading with OMP because this can lead to dead-locks in distributed tests.
   OMP_NUM_THREADS: '1'
+  # See https://github.com/pytorch/pytorch/issues/37377#issuecomment-677851112.
+  MKL_THREADING_LAYER: 'GNU'
+  DEFAULT_PYTHON_VERSION: 3.8
+
+defaults:
+  run:
+    shell: bash -l {0}
 
 jobs:
   changelog:
@@ -58,7 +65,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Install requirements
       run: |
@@ -112,20 +119,12 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      env:
-        # Log useful debugging information.
-        ACTIONS_STEP_DEBUG: 'true'
+    - uses: conda-incubator/setup-miniconda@v2
       with:
-        python-version: 3.8
-
-    - name: Install prerequisites
-      run: |
-        pip install --upgrade pip setuptools wheel virtualenv
+        miniconda-version: "latest"
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Set build variables
-      shell: bash
       run: |
         # Get the exact Python version to use in the cache key.
         echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
@@ -148,20 +147,18 @@ jobs:
       with:
         path: .venv
         key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ matrix.task.torch_platform }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ matrix.task.torch_platform }}-
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
       run: |
-        test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
-        . .venv/bin/activate
+        conda create -y -p .venv python=${{ env.DEFAULT_PYTHON_VERSION }}
+        conda activate ./.venv
         make install TORCH_INSTALL="$TORCH_INSTALL"
 
     - name: Setup virtual environment (cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit == 'true'
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip install --no-deps -e .
         make download-extras
 
@@ -170,25 +167,24 @@ jobs:
       env:
         ALLENNLP_VERSION_OVERRIDE: ""  # Don't replace the core library.
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         git clone https://github.com/allenai/allennlp-models.git
         cd allennlp-models
-        pip install --upgrade --upgrade-strategy eager -e . -r dev-requirements.txt
+        pip install -e . -r dev-requirements.txt
 
     - name: Debug info
       run: |
-        . .venv/bin/activate
-        which python
-        pip freeze
+        conda activate ./.venv
+        conda list
 
     - name: Ensure torch up-to-date
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         python scripts/check_torch_version.py
 
     - name: ${{ matrix.task.name }}
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         ${{ matrix.task.run }}
 
     - name: Prepare coverage report
@@ -209,7 +205,7 @@ jobs:
       run: |
         # Could run into issues with the cache if we don't uninstall the editable.
         # See https://github.com/pypa/pip/issues/4537.
-        . .venv/bin/activate
+        conda activate ./.venv
         pip uninstall --yes allennlp allennlp-models
 
   upload_coverage:
@@ -259,58 +255,48 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      env:
-        # Log useful debugging information.
-        ACTIONS_STEP_DEBUG: 'true'
+    - uses: conda-incubator/setup-miniconda@v2
       with:
-        python-version: 3.8
-
-    - name: Install prerequisites
-      run: |
-        pip install --upgrade pip setuptools wheel virtualenv
+        miniconda-version: "latest"
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Set build variables
-      shell: bash
       run: |
         echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
         echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
         echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
+        echo "TORCH_INSTALL=$TORCH_CPU_INSTALL" >> $GITHUB_ENV
 
     - uses: actions/cache@v2
       id: virtualenv-cache
       with:
         path: .venv
         key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
       run: |
-        test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
-        . .venv/bin/activate
-        make install TORCH_INSTALL="$TORCH_CPU_INSTALL"
+        conda create -y -p .venv python=${{ env.DEFAULT_PYTHON_VERSION }}
+        conda activate ./.venv
+        make install TORCH_INSTALL="$TORCH_INSTALL"
 
     - name: Setup virtual environment (cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit == 'true'
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip install --no-deps -e .
         make download-extras
 
     - name: Debug info
       run: |
-        . .venv/bin/activate
-        which python
+        conda activate ./.venv
         pip freeze
 
     - name: Check and set nightly version
       if: github.event_name == 'schedule'
       run: |
         # Verify that current version is ahead of the last release.
-        . .venv/bin/activate
+        conda activate ./.venv
         LATEST=$(scripts/get_version.py latest)
         CURRENT=$(scripts/get_version.py current)
         if [ "$CURRENT" == "$LATEST" ]; then
@@ -326,7 +312,7 @@ jobs:
       if: github.event_name == 'release'
       run: |
         # Remove 'refs/tags/' to get the actual tag from the release.
-        . .venv/bin/activate
+        conda activate ./.venv
         TAG=${GITHUB_REF#refs/tags/};
         VERSION=$(scripts/get_version.py current)
         if [ "$TAG" != "$VERSION" ]; then
@@ -337,7 +323,7 @@ jobs:
     - name: Build core package
       run: |
         # Just print out the version for debugging.
-        . .venv/bin/activate
+        conda activate ./.venv
         make version
         python setup.py bdist_wheel sdist
 
@@ -350,7 +336,7 @@ jobs:
     - name: Clean up
       if: always()
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip uninstall --yes allennlp
 
   # Tests installing from the distribution files.
@@ -365,7 +351,7 @@ jobs:
 
     steps:
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python }}
 
@@ -399,7 +385,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        cuda: ['10.2', '11.1']
+        cuda: ['11.3']
 
     steps:
     - uses: actions/checkout@v2
@@ -434,15 +420,15 @@ jobs:
         docker push $DOCKER_IMAGE_NAME
 
     - name: Upload default commit image
-      # CUDA 10.2 is currently our default.
-      if: github.event_name == 'push' && matrix.cuda == '10.2'
+      # CUDA 11.3 is currently our default.
+      if: github.event_name == 'push' && matrix.cuda == '11.3'
       run: |
         docker tag $DOCKER_IMAGE_NAME allennlp/commit:${GITHUB_SHA}
         docker push allennlp/commit:${GITHUB_SHA}
 
     - name: Upload latest image
-      # CUDA 10.2 is currently our default.
-      if: github.event_name == 'release' && matrix.cuda == '10.2'
+      # CUDA 11.3 is currently our default.
+      if: github.event_name == 'release' && matrix.cuda == '11.3'
       run: |
         docker tag $DOCKER_IMAGE_NAME allennlp/allennlp:latest
         docker push allennlp/allennlp:latest
@@ -464,52 +450,42 @@ jobs:
       with:
         ssh-private-key: ${{ secrets.DOCS_DEPLOY_KEY }}
 
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      env:
-        # Log useful debugging information.
-        ACTIONS_STEP_DEBUG: 'true'
+    - uses: conda-incubator/setup-miniconda@v2
       with:
-        python-version: 3.8
-
-    - name: Install prerequisites
-      run: |
-        pip install --upgrade pip setuptools wheel virtualenv
+        miniconda-version: "latest"
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Set build variables
-      shell: bash
       run: |
         echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
         echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
         echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
+        echo "TORCH_INSTALL=$TORCH_CPU_INSTALL" >> $GITHUB_ENV
 
     - uses: actions/cache@v2
       id: virtualenv-cache
       with:
         path: .venv
         key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
       run: |
-        test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
-        . .venv/bin/activate
-        make install TORCH_INSTALL="$TORCH_CPU_INSTALL"
+        conda create -y -p .venv python=${{ env.DEFAULT_PYTHON_VERSION }}
+        conda activate ./.venv
+        make install TORCH_INSTALL="$TORCH_INSTALL"
 
     - name: Setup virtual environment (cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit == 'true'
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip install --no-deps -e .
         make download-extras
 
     - name: Debug info
       run: |
-        . .venv/bin/activate
-        which python
-        pip freeze
+        conda activate ./.venv
+        conda list
 
     - name: Prepare environment
       run: |
@@ -523,7 +499,7 @@ jobs:
 
     - name: Build docs
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         ./scripts/build_docs.sh
 
     - name: Print the ref
@@ -555,7 +531,7 @@ jobs:
       run: |
         # Fail immediately if any step fails.
         set -e
-        . .venv/bin/activate
+        conda activate ./.venv
 
         LATEST=$(./scripts/get_version.py latest)
         STABLE=$(./scripts/get_version.py stable)
@@ -613,7 +589,7 @@ jobs:
     - name: Clean up
       if: always()
       run: |
-        . .venv/bin/activate
+        conda activate ./.venv
         pip uninstall --yes allennlp
 
   # Publish the core distribution files to PyPI.
@@ -629,9 +605,9 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Install requirements
       run: |
diff --git a/Dockerfile b/Dockerfile
index 1d87d409024..0e298eebc14 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,9 +2,13 @@
 # It's built from a wheel installation of allennlp using the base images from
 # https://github.com/allenai/docker-images/pkgs/container/pytorch
 
-ARG TORCH=1.9.0-cuda10.2
-ARG PYTHON=3.9
-FROM ghcr.io/allenai/pytorch:${TORCH}-python${PYTHON}-v0.0.1
+ARG TORCH=1.10.0-cuda11.3
+FROM ghcr.io/allenai/pytorch:${TORCH}
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /stage/allennlp
 
diff --git a/Dockerfile.test b/Dockerfile.test
index 81b97d32a41..d5a5661170d 100644
--- a/Dockerfile.test
+++ b/Dockerfile.test
@@ -1,8 +1,12 @@
 # Used to build an image for running tests.
 
-ARG TORCH=1.9.0-cuda10.2
-ARG PYTHON=3.9
-FROM ghcr.io/allenai/pytorch:${TORCH}-python${PYTHON}-v0.0.1
+ARG TORCH=1.10.0-cuda11.3
+FROM ghcr.io/allenai/pytorch:${TORCH}
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # These environment variables are helpful for debugging.
 # See https://pytorch.org/docs/stable/distributed.html#common-environment-variables for more info.
diff --git a/Makefile b/Makefile
index 3569b53f8c0..aa51648a6a2 100644
--- a/Makefile
+++ b/Makefile
@@ -11,9 +11,8 @@ MD_DOCS_TGT = site/
 MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md CONTRIBUTING.md)
 
 TORCH_INSTALL = pip install torch torchvision
-DOCKER_TORCH_VERSION = 1.9.0-cuda10.2
-DOCKER_TEST_TORCH_VERSION = 1.9.0-cuda10.2
-DOCKER_PYTHON_VERSION = 3.9
+DOCKER_TORCH_VERSION = 1.10.0-cuda11.3
+DOCKER_TEST_TORCH_VERSION = 1.10.0-cuda11.3
 
 DOCKER_TAG = latest
 DOCKER_IMAGE_NAME = allennlp/allennlp:$(DOCKER_TAG)
@@ -91,14 +90,12 @@ download-extras :
 
 .PHONY : install
 install :
-	# Ensure pip, setuptools, and wheel are up-to-date.
-	pip install --upgrade pip setuptools wheel
 	# Due to a weird thing with pip, we may need egg-info before running `pip install -e`.
 	# See https://github.com/pypa/pip/issues/4537.
 	# python setup.py install_egg_info
 	# Install torch ecosystem first.
 	$(TORCH_INSTALL)
-	pip install --upgrade --upgrade-strategy eager -e . -r dev-requirements.txt
+	pip install -e . -r dev-requirements.txt
 	# These nltk packages are used by the 'checklist' module.
 	$(NLTK_DOWNLOAD_CMD)
 
@@ -159,7 +156,6 @@ docker-image :
 		--pull \
 		-f Dockerfile \
 		--build-arg TORCH=$(DOCKER_TORCH_VERSION) \
-		--build-arg PYTHON=$(DOCKER_PYTHON_VERSION) \
 		-t $(DOCKER_IMAGE_NAME) .
 
 DOCKER_GPUS = --gpus all
diff --git a/setup.py b/setup.py
index e55a0939d30..d28b47ab0a5 100644
--- a/setup.py
+++ b/setup.py
@@ -50,9 +50,9 @@
         ]
     ),
     install_requires=[
+        "torch>=1.6.0,<1.11.0",
+        "torchvision>=0.8.1,<0.12.0",
         "cached-path>=0.3.1,<0.4.0",
-        "torch>=1.6.0,<1.10.0",
-        "torchvision>=0.8.1,<0.11.0",
         "fairscale==0.4.0",
         "jsonnet>=0.10.0 ; sys.platform != 'win32'",
         "overrides==3.1.0",
diff --git a/tests/data/image_loader_test.py b/tests/data/image_loader_test.py
index 096f1770639..22ae32b1c94 100644
--- a/tests/data/image_loader_test.py
+++ b/tests/data/image_loader_test.py
@@ -13,6 +13,8 @@ def setup_method(self):
             self.FIXTURES_ROOT / "data" / "images" / "COCO_train2014_000000458752.jpg"
         )
 
+        torchvision.set_image_backend("accimage")
+
         # Create a few small images of different sizes from the fixture.
         image = torchvision.io.read_image(self.image_fixture_path)
         assert image.shape == (3, 480, 640)