Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

On-Board check-catalyst workflow to use GitHub Large Runners #675

Merged
merged 23 commits into from
Apr 25, 2024
Merged

On-Board check-catalyst workflow to use GitHub Large Runners #675

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c219290
Try large runners
rashidnhm Apr 23, 2024
319a71b
Move re-usable action to .github/workflows/
rashidnhm Apr 23, 2024
6d726f9
Ensure job is never skipped
rashidnhm Apr 23, 2024
c51cef4
Bump actions version. Swap `True` values to `true`
rashidnhm Apr 23, 2024
4b2563a
Change logic on what type of runner is used for workflow
rashidnhm Apr 23, 2024
ed2f5e4
Fix bug
rashidnhm Apr 23, 2024
ac0e0f1
Add startsWith constraint
rashidnhm Apr 24, 2024
14deb2f
Try actions/cache boolean
rashidnhm Apr 24, 2024
72d4051
Try downgrade actions/cache
rashidnhm Apr 24, 2024
6070595
Remove testing steps
rashidnhm Apr 24, 2024
77299b5
Merge branch 'main' into sc-61791-ci-use-large-runners
rashidnhm Apr 24, 2024
80b5180
On-board build-wheel-linux-x86_64 to use large runners
rashidnhm Apr 24, 2024
d168650
Merge branch 'sc-61791-ci-use-large-runners' of github.com:PennyLaneA…
rashidnhm Apr 24, 2024
702552a
Trigger CI
rashidnhm Apr 24, 2024
4074c33
Fix incorrect strategy matrix
rashidnhm Apr 24, 2024
412439d
Bump action versions across the board. Fix typo cmake call
rashidnhm Apr 24, 2024
5c2b7fb
Merge branch 'main' of github.com:PennyLaneAI/catalyst into sc-61791-…
rashidnhm Apr 24, 2024
7f36b5f
Merge branch 'main' of github.com:PennyLaneAI/catalyst into sc-61791-…
rashidnhm Apr 25, 2024
69a3193
Bump arm64 workflow to use upload-artifact@v4
rashidnhm Apr 25, 2024
6cd34e9
Downgrade manylinux build to checkout@v3
rashidnhm Apr 25, 2024
6b0a3fa
Merge branch 'main' of github.com:PennyLaneAI/catalyst into sc-61791-…
rashidnhm Apr 25, 2024
591ded0
Change other @v4 to v3 for actions/checkout
rashidnhm Apr 25, 2024
670798b
Downgrade cache@v4 to v3 for manylinux
rashidnhm Apr 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions .github/workflows/build-wheel-linux-x86_64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,25 @@ concurrency:
cancel-in-progress: true

jobs:
determine_runner:
name: Determine runner type to use
uses: ./.github/workflows/determine-workflow-runner.yml
with:
default_runner: ubuntu-latest

check_if_wheel_build_required:
uses: ./.github/workflows/check-for-wheel-build.yml

constants:
needs: [check_if_wheel_build_required]
needs: [check_if_wheel_build_required, determine_runner]
if: needs.check_if_wheel_build_required.outputs.build-wheels == 'true'
name: "Set build matrix"
uses: ./.github/workflows/constants.yaml
with:
runs_on: ${{ needs.determine_runner.outputs.runner_group }}

build-dependencies:
needs: [constants, check_if_wheel_build_required]
needs: [constants, check_if_wheel_build_required, determine_runner]

strategy:
fail-fast: false
Expand All @@ -37,12 +45,13 @@ jobs:
container_img: ["quay.io/pypa/manylinux2014_x86_64"]

name: Build Dependencies (Python ${{ matrix.python_version }})
runs-on: ubuntu-latest
runs-on: ${{ needs.determine_runner.outputs.runner_group }}
container: ${{ matrix.container_img }}

if: needs.check_if_wheel_build_required.outputs.build-wheels == 'true'

steps:
# TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
- name: Checkout Catalyst repo
uses: actions/checkout@v3

Expand Down Expand Up @@ -78,6 +87,7 @@ jobs:
key: ${{ runner.os }}-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
enableCrossOsArchive: True

# TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
- name: Clone LLVM Submodule
if: steps.cache-llvm-source.outputs.cache-hit != 'true'
uses: actions/checkout@v3
Expand All @@ -86,6 +96,7 @@ jobs:
ref: ${{ needs.constants.outputs.llvm_version }}
path: mlir/llvm-project

# TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
- name: Clone MHLO Submodule
if: steps.cache-mhlo-source.outputs.cache-hit != 'true'
uses: actions/checkout@v3
Expand All @@ -94,6 +105,7 @@ jobs:
ref: ${{ needs.constants.outputs.mhlo_version }}
path: mlir/mlir-hlo

# TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
- name: Clone Enzyme Submodule
if: steps.cache-enzyme-source.outputs.cache-hit != 'true'
uses: actions/checkout@v3
Expand Down Expand Up @@ -206,20 +218,21 @@ jobs:
cmake --build enzyme-build --target EnzymeStatic-18

catalyst-linux-wheels-x86-64:
needs: [constants, build-dependencies]
needs: [constants, build-dependencies, determine_runner]
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
os: ${{ fromJSON(format('[{0}]', needs.determine_runner.outputs.runner_group)) }}
arch: [x86_64]
python_version: ${{ fromJson(needs.constants.outputs.python_versions) }}
python_version: ${{ fromJSON(needs.constants.outputs.python_versions) }}
container_img: ["quay.io/pypa/manylinux2014_x86_64"]

name: Build Wheels (Python ${{ matrix.python_version }})
runs-on: ${{ matrix.os }}
container: ${{ matrix.container_img }}

steps:
# TODO: Cannot use actions/checkout@v4 as node20 is not supported in manylinux2014 container
- name: Checkout Catalyst repo
uses: actions/checkout@v3

Expand Down Expand Up @@ -357,20 +370,20 @@ jobs:
retention-days: 14

test-wheels:
needs: [constants, catalyst-linux-wheels-x86-64]
needs: [constants, catalyst-linux-wheels-x86-64, determine_runner]
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, ubuntu-20.04]
python_version: ${{ fromJson(needs.constants.outputs.python_versions) }}
os: ${{ fromJSON(format('[{0}]', needs.determine_runner.outputs.runner_group)) }}
python_version: ${{ fromJSON(needs.constants.outputs.python_versions) }}

# To check all wheels for supported python3 versions
name: Test Wheels (Python ${{ matrix.python_version }}) on ${{ matrix.os }}
runs-on: ${{ matrix.os }}

steps:
- name: Checkout Catalyst repo
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Download Wheel Artifact
uses: actions/download-artifact@v3
Expand All @@ -379,7 +392,7 @@ jobs:
path: dist

- name: Set up Python ${{ matrix.python_version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}

Expand Down
40 changes: 20 additions & 20 deletions .github/workflows/build-wheel-macos-arm64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:

steps:
- name: Checkout Catalyst repo
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Setup Runner Environment
id: setup_env
Expand All @@ -55,47 +55,47 @@ jobs:
# Cache external project sources
- name: Cache LLVM Source
id: cache-llvm-source
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project
key: Linux-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source
enableCrossOsArchive: True

- name: Cache MHLO Source
id: cache-mhlo-source
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo
key: Linux-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source
enableCrossOsArchive: True

- name: Cache Enzyme Source
id: cache-enzyme-source
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme
key: Linux-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
enableCrossOsArchive: True

- name: Clone LLVM Submodule
if: steps.cache-llvm-source.outputs.cache-hit != 'true'
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
repository: llvm/llvm-project
ref: ${{ needs.constants.outputs.llvm_version }}
path: mlir/llvm-project

- name: Clone MHLO Submodule
if: steps.cache-mhlo-source.outputs.cache-hit != 'true'
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
repository: tensorflow/mlir-hlo
ref: ${{ needs.constants.outputs.mhlo_version }}
path: mlir/mlir-hlo

- name: Clone Enzyme Submodule
if: steps.cache-enzyme-source.outputs.cache-hit != 'true'
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
repository: EnzymeAD/Enzyme
ref: ${{ needs.constants.outputs.enzyme_version }}
Expand Down Expand Up @@ -133,21 +133,21 @@ jobs:
# Cache external project builds
- name: Cache LLVM Build
id: cache-llvm-build
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build
key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-generic-build

- name: Cache MHLO Build
id: cache-mhlo-build
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build
key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build

- name: Cache Enzyme Build
id: cache-enzyme-build
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build
key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build
Expand Down Expand Up @@ -218,7 +218,7 @@ jobs:

steps:
- name: Checkout Catalyst repo
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Setup Runner Environment
id: setup_env
Expand All @@ -232,7 +232,7 @@ jobs:

- name: Get Cached LLVM Source
id: cache-llvm-source
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project
key: Linux-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source
Expand All @@ -241,15 +241,15 @@ jobs:

- name: Get Cached LLVM Build
id: cache-llvm-build
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build
key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-generic-build
fail-on-cache-miss: True

- name: Get Cached MHLO Source
id: cache-mhlo-source
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo
key: Linux-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source
Expand All @@ -258,23 +258,23 @@ jobs:

- name: Get Cached MHLO Build
id: cache-mhlo-build
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build
key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build
fail-on-cache-miss: True

- name: Get Cached Enzyme Source
id: cache-enzyme-source
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme
key: Linux-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
fail-on-cache-miss: True

- name: Get Cached Enzyme Build
id: cache-enzyme-build
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build
key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build
Expand Down Expand Up @@ -356,7 +356,7 @@ jobs:
delocate-wheel --require-archs=arm64 -w ./wheel -v dist/*.whl --ignore-missing-dependencies -vv

- name: Upload Wheel Artifact
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip
path: wheel/
Expand All @@ -376,7 +376,7 @@ jobs:

steps:
- name: Checkout Catalyst repo
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Setup Runner Environment
id: setup_env
Expand All @@ -385,7 +385,7 @@ jobs:
python_version: ${{ matrix.python_version }}

- name: Download Wheel Artifact
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip
path: dist
Expand Down
Loading
Loading