On-Board check-catalyst workflow to use GitHub Large Runners (#675)

**Context:** 
When high volume of jobs are being queued on CI. It is possible that it
will take a while before a job is picked up. To get around this, we have
provisioned an organization runner group of GitHub Large Runners. Since
these are paid runners, we retain the ability to scale this up to a
value suitable for us during periods of high usage.


**Description of the Change:**
A new label `urgent` has been introduced to this repo. Whenever a pull
request has this label attached, all the jobs within check-catalyst.yml
will swap over to use the new Large runner group described above.

This PR also bumps the version of actions/checkout and actions/cache
being used. This was done to deal with the deprecation of node16 within
github actions.

**Benefits:**
Lowered queue times for urgent CI workloads to get picked up.

**Possible Drawbacks:**
We can still see jobs being queued for a while if enough PRs have the
`urgent` label at once. To get around this, only add it to PRs that are
truly urgent. Also, we can increase the scaling factor of the group if
needed.

**Related GitHub Issues:**
None.
[sc-61791](https://app.shortcut.com/xanaduai/story/61791/update-catalyst-ci-to-use-large-runners)

[sc-61184]: fix for cache issues

Author: rashidnhm

.github/workflows/build-wheel-linux-x86_64.yaml

@@ -16,17 +16,25 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  determine_runner:
+    name: Determine runner type to use
+    uses: ./.github/workflows/determine-workflow-runner.yml
+    with:
+      default_runner: ubuntu-latest
+
   check_if_wheel_build_required:
     uses: ./.github/workflows/check-for-wheel-build.yml
 
   constants:
-    needs: [check_if_wheel_build_required]
+    needs: [check_if_wheel_build_required, determine_runner]
     if: needs.check_if_wheel_build_required.outputs.build-wheels == 'true'
     name: "Set build matrix"
     uses: ./.github/workflows/constants.yaml
+    with:
+      runs_on: ${{ needs.determine_runner.outputs.runner_group }}
 
   build-dependencies:
-    needs: [constants, check_if_wheel_build_required]
+    needs: [constants, check_if_wheel_build_required, determine_runner]
 
     strategy:
       fail-fast: false
@@ -37,12 +45,13 @@ jobs:
         container_img: ["quay.io/pypa/manylinux2014_x86_64"]
 
     name: Build Dependencies (Python ${{ matrix.python_version }})
-    runs-on: ubuntu-latest
+    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
     container: ${{ matrix.container_img }}
 
     if: needs.check_if_wheel_build_required.outputs.build-wheels == 'true'
 
     steps:
+    # TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
     - name: Checkout Catalyst repo
       uses: actions/checkout@v3
 
@@ -78,6 +87,7 @@ jobs:
         key: ${{ runner.os }}-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
         enableCrossOsArchive: True
 
+    # TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
     - name: Clone LLVM Submodule
       if: steps.cache-llvm-source.outputs.cache-hit != 'true'
       uses: actions/checkout@v3
@@ -86,6 +96,7 @@ jobs:
         ref: ${{ needs.constants.outputs.llvm_version }}
         path: mlir/llvm-project
 
+    # TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
     - name: Clone MHLO Submodule
       if: steps.cache-mhlo-source.outputs.cache-hit != 'true'
       uses: actions/checkout@v3
@@ -94,6 +105,7 @@ jobs:
         ref: ${{ needs.constants.outputs.mhlo_version }}
         path: mlir/mlir-hlo
 
+    # TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
     - name: Clone Enzyme Submodule
       if: steps.cache-enzyme-source.outputs.cache-hit != 'true'
       uses: actions/checkout@v3
@@ -206,20 +218,21 @@ jobs:
         cmake --build enzyme-build --target EnzymeStatic-18
 
   catalyst-linux-wheels-x86-64:
-    needs: [constants, build-dependencies]
+    needs: [constants, build-dependencies, determine_runner]
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: ${{ fromJSON(format('[{0}]', needs.determine_runner.outputs.runner_group)) }}
         arch: [x86_64]
-        python_version: ${{ fromJson(needs.constants.outputs.python_versions) }}
+        python_version: ${{ fromJSON(needs.constants.outputs.python_versions) }}
         container_img: ["quay.io/pypa/manylinux2014_x86_64"]
 
     name: Build Wheels (Python ${{ matrix.python_version }})
     runs-on: ${{ matrix.os }}
     container: ${{ matrix.container_img }}
 
     steps:
+    # TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
     - name: Checkout Catalyst repo
       uses: actions/checkout@v3
 
@@ -357,20 +370,20 @@ jobs:
         retention-days: 14
 
   test-wheels:
-    needs: [constants, catalyst-linux-wheels-x86-64]
+    needs: [constants, catalyst-linux-wheels-x86-64, determine_runner]
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, ubuntu-20.04]
-        python_version: ${{ fromJson(needs.constants.outputs.python_versions) }}
+        os: ${{ fromJSON(format('[{0}]', needs.determine_runner.outputs.runner_group)) }}
+        python_version: ${{ fromJSON(needs.constants.outputs.python_versions) }}
 
     # To check all wheels for supported python3 versions
     name: Test Wheels (Python ${{ matrix.python_version }}) on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
 
     steps:
     - name: Checkout Catalyst repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Download Wheel Artifact
       uses: actions/download-artifact@v3
@@ -379,7 +392,7 @@ jobs:
         path: dist
 
     - name: Set up Python ${{ matrix.python_version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python_version }}
 

.github/workflows/build-wheel-macos-arm64.yaml

@@ -44,7 +44,7 @@ jobs:
 
     steps:
     - name: Checkout Catalyst repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Setup Runner Environment
       id: setup_env
@@ -55,47 +55,47 @@ jobs:
     # Cache external project sources
     - name: Cache LLVM Source
       id: cache-llvm-source
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path:  ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project
         key: Linux-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source
         enableCrossOsArchive: True
 
     - name: Cache MHLO Source
       id: cache-mhlo-source
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo
         key: Linux-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source
         enableCrossOsArchive: True
 
     - name: Cache Enzyme Source
       id: cache-enzyme-source
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path:  ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme
         key: Linux-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
         enableCrossOsArchive: True
 
     - name: Clone LLVM Submodule
       if: steps.cache-llvm-source.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: llvm/llvm-project
         ref: ${{ needs.constants.outputs.llvm_version }}
         path: mlir/llvm-project
 
     - name: Clone MHLO Submodule
       if: steps.cache-mhlo-source.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: tensorflow/mlir-hlo
         ref: ${{ needs.constants.outputs.mhlo_version }}
         path: mlir/mlir-hlo
 
     - name: Clone Enzyme Submodule
       if: steps.cache-enzyme-source.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: EnzymeAD/Enzyme
         ref: ${{ needs.constants.outputs.enzyme_version }}
@@ -133,21 +133,21 @@ jobs:
     # Cache external project builds
     - name: Cache LLVM Build
       id: cache-llvm-build
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path:  ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build
         key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-generic-build
 
     - name: Cache MHLO Build
       id: cache-mhlo-build
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path:  ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build
         key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build
 
     - name: Cache Enzyme Build
       id: cache-enzyme-build
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build
         key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build
@@ -218,7 +218,7 @@ jobs:
 
     steps:
     - name: Checkout Catalyst repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Setup Runner Environment
       id: setup_env
@@ -232,7 +232,7 @@ jobs:
 
     - name: Get Cached LLVM Source
       id: cache-llvm-source
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project
         key: Linux-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source
@@ -241,15 +241,15 @@ jobs:
 
     - name: Get Cached LLVM Build
       id: cache-llvm-build
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path:  ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build
         key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-generic-build
         fail-on-cache-miss: True
 
     - name: Get Cached MHLO Source
       id: cache-mhlo-source
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo
         key: Linux-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source
@@ -258,23 +258,23 @@ jobs:
 
     - name: Get Cached MHLO Build
       id: cache-mhlo-build
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build
         key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build
         fail-on-cache-miss: True
 
     - name: Get Cached Enzyme Source
       id: cache-enzyme-source
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme
         key: Linux-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
         fail-on-cache-miss: True
 
     - name: Get Cached Enzyme Build
       id: cache-enzyme-build
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build
         key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build
@@ -356,7 +356,7 @@ jobs:
         delocate-wheel --require-archs=arm64 -w ./wheel -v dist/*.whl --ignore-missing-dependencies -vv
 
     - name: Upload Wheel Artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip
         path: wheel/
@@ -376,7 +376,7 @@ jobs:
 
     steps:
     - name: Checkout Catalyst repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Setup Runner Environment
       id: setup_env
@@ -385,7 +385,7 @@ jobs:
         python_version: ${{ matrix.python_version }}
 
     - name: Download Wheel Artifact
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip
         path: dist