From c71bba98f0fb9856b69ed30258b5fb8ae4eec610 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 4 Jul 2024 05:32:17 +0200 Subject: [PATCH 01/15] Remove failing `x86` build that is failing the pip release --- .github/workflows/pip-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index 70cbb21be..02d5d4ea2 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -64,7 +64,7 @@ jobs: runs-on: windows-latest strategy: matrix: - target: [x64, x86] + target: [x64] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 From 2f401ff8464aba4e074206d1a122ce4ee4d08e4f Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 4 Jul 2024 07:20:47 +0200 Subject: [PATCH 02/15] Fix `pip-release` by bumping actions version --- .github/workflows/pip-release.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index 02d5d4ea2..293679bcf 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -61,20 +61,24 @@ jobs: file_glob: true windows: - runs-on: windows-latest + runs-on: ${{ matrix.platform.runner }} strategy: matrix: - target: [x64] + platform: + - runner: windows-latest + target: x64 + - runner: windows-latest + target: x86 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.7" - architecture: ${{ matrix.target }} + architecture: ${{ matrix.platform.target }} - name: Build wheels uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.target }} + target: ${{ matrix.platform.target }} args: --release --out dist sccache: "true" working-directory: apis/python/node From e3d9686474278082f7b29971be51cd06f588d1dd Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sat, 6 Jul 2024 12:59:00 +0200 Subject: [PATCH 03/15] Refactor pip and cli CI/CD --- .github/workflows/pip-release.yml | 150 +++++++++++++++++---------- .github/workflows/release.yml | 162 ++++++++---------------------- 2 files changed, 141 insertions(+), 171 deletions(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index 293679bcf..2c9de7254 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -19,46 +19,106 @@ permissions: jobs: linux: - runs-on: ubuntu-latest + runs-on: ${{ matrix.platform.runner }} strategy: matrix: - target: [x86_64, x86, aarch64] + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + # - runner: ubuntu-latest + # target: s390x + # - runner: ubuntu-latest + # target: ppc64le steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "3.8" - - name: Install dependencies - run: | - pip install patchelf --upgrade + python-version: 3.8 - name: Build wheels uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.target }} + target: ${{ matrix.platform.target }} args: --release --out dist --zig sccache: "true" manylinux: auto working-directory: apis/python/node + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: apis/python/node/dist + + musllinux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.8 - name: Build wheels uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.target }} - args: --release --out dist --zig + target: ${{ matrix.platform.target }} + args: --release --out dist sccache: "true" + manylinux: musllinux_1_2 working-directory: apis/python/node + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.platform.target }} + path: apis/python/node/dist + + musleabi: + runs-on: ubuntu-latest + strategy: + matrix: + platform: + [ + { + target: "armv7-unknown-linux-musleabihf", + image_tag: "armv7-musleabihf", + }, + ] + container: + image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }} + env: + CFLAGS_armv7_unknown_linux_musleabihf: "-mfpu=vfpv3-d16" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.8 + - name: Build Wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} manylinux: musllinux_1_2 + container: off + args: --release -o dist + working-directory: apis/python/node - name: Upload wheels uses: actions/upload-artifact@v3 with: - name: wheels + name: wheels-musllinux-${{ matrix.platform.target }} path: apis/python/node/dist - - name: Upload to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: apis/python/node/dist/* - tag: ${{ github.ref }} - file_glob: true windows: runs-on: ${{ matrix.platform.runner }} @@ -66,14 +126,12 @@ jobs: matrix: platform: - runner: windows-latest - target: x64 - - runner: windows-latest - target: x86 + target: x86_64 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.7" + python-version: 3.8 architecture: ${{ matrix.platform.target }} - name: Build wheels uses: PyO3/maturin-action@v1 @@ -83,47 +141,37 @@ jobs: sccache: "true" working-directory: apis/python/node - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-windows-${{ matrix.platform.target }} path: apis/python/node/dist - - name: Upload to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: apis/python/node/dist/* - tag: ${{ github.ref }} - file_glob: true macos: - runs-on: macos-latest + runs-on: ${{ matrix.platform.runner }} strategy: matrix: - target: [x86_64, aarch64] + platform: + - runner: macos-12 + target: x86_64 + - runner: macos-14 + target: aarch64 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: 3.8 - name: Build wheels uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.target }} + target: ${{ matrix.platform.target }} args: --release --out dist sccache: "true" working-directory: apis/python/node - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-macos-${{ matrix.platform.target }} path: apis/python/node/dist - - name: Upload to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: apis/python/node/dist/* - tag: ${{ github.ref }} - file_glob: true sdist: runs-on: ubuntu-latest @@ -145,15 +193,13 @@ jobs: name: Release runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" - needs: [linux, windows, macos, sdist] + needs: [linux, musllinux, musleabi, windows, macos, sdist] steps: - - uses: actions/download-artifact@v3 - with: - name: wheels + - uses: actions/download-artifact@v4 - name: Publish to PyPI uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASS }} with: command: upload - args: --skip-existing * + args: --non-interactive --skip-existing * diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7d747d61..ce448bc75 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,48 +83,59 @@ jobs: cargo publish -p dora-record --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-rerun --token ${{ secrets.CARGO_REGISTRY_TOKEN }} - windows-release: - name: "Windows Release" - + unix: + runs-on: ${{ matrix.platform.runner }} strategy: matrix: - platform: [windows-2022] + platform: + - runner: ubuntu-20.04 + target: x86_64-unknown-linux-gnu + - runner: ubuntu-20.04 + target: i686-unknown-linux-gnu + - runner: ubuntu-20.04 + target: aarch64-unknown-linux-gnu + - runner: ubuntu-20.04 + target: aarch64-unknown-linux-musl + - runner: ubuntu-20.04 + target: armv7-unknown-linux-musleabihf + - runner: macos-12 + target: aarch64-apple-darwin + - runner: macos-12 + target: x86_64-apple-darwin fail-fast: false - runs-on: ${{ matrix.platform }} - steps: - uses: actions/checkout@v3 - - uses: r7kamura/rust-problem-matchers@v1.1.0 - - - name: "Build binaries" - timeout-minutes: 60 - run: "cargo build --release -p dora-cli" - - - name: Create Archive (Windows) - if: runner.os == 'Windows' - shell: powershell + - name: "Add toolchains" + run: rustup target add ${{ matrix.platform.target }} + - name: "Build" + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --release --target ${{ matrix.platform.target }} -p dora-cli + - name: "Archive Linux" run: | - New-Item -Path archive -ItemType Directory - Copy-Item target/release/dora.exe -Destination archive/dora.exe - Compress-Archive -Path archive\* -DestinationPath archive.zip - - - name: "Upload release asset" + zip -r ${{ matrix.platform.target }}.zip target/ ${{ matrix.platform.target }}/release + - name: "Upload asset" uses: actions/upload-release-asset@v1.0.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: archive.zip - asset_name: dora-${{ github.ref_name }}-x86_64-${{ runner.os }}.zip + asset_path: ${{ matrix.platform.target }}.zip + asset_name: dora-${{ github.ref_name }}-${{ matrix.platform.target }}.zip asset_content_type: application/zip - unix-release: - name: "Unix Release" + windows-release: + name: "Windows Release" strategy: matrix: - platform: [macos-12, ubuntu-20.04] + platform: + - runner: windows-2022 + target: x86_64-pc-windows-msvc + fail-fast: false runs-on: ${{ matrix.platform }} @@ -137,14 +148,13 @@ jobs: timeout-minutes: 60 run: "cargo build --release -p dora-cli" - - name: "Create Archive (Unix)" - if: runner.os == 'Linux' || runner.os == 'macOS' + - name: Create Archive (Windows) + if: runner.os == 'Windows' + shell: powershell run: | - mkdir archive - cp target/release/dora archive/dora - cd archive - zip -r ../archive.zip . - cd .. + New-Item -Path archive -ItemType Directory + Copy-Item target/release/dora.exe -Destination archive/dora.exe + Compress-Archive -Path archive\* -DestinationPath archive.zip - name: "Upload release asset" uses: actions/upload-release-asset@v1.0.1 @@ -153,91 +163,5 @@ jobs: with: upload_url: ${{ github.event.release.upload_url }} asset_path: archive.zip - asset_name: dora-${{ github.ref_name }}-x86_64-${{ runner.os }}.zip - asset_content_type: application/zip - - linux-arm-release: - name: "Linux ARM Release" - - strategy: - matrix: - platform: [ubuntu-20.04] - fail-fast: false - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/checkout@v3 - - - uses: r7kamura/rust-problem-matchers@v1.1.0 - - - name: "Build Linux ARM64" - if: runner.os == 'Linux' - uses: actions-rs/cargo@v1 - with: - use-cross: true - command: build - args: --release --target aarch64-unknown-linux-gnu -p dora-cli - - - name: "Archive Linux ARM64" - if: runner.os == 'Linux' - run: | - mkdir archive_aarch64 - cp target/aarch64-unknown-linux-gnu/release/dora archive_aarch64/dora - cd archive_aarch64 - zip -r ../archive_aarch64.zip . - cd .. - - - name: "Upload Linux ARM64 asset" - if: runner.os == 'Linux' - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: archive_aarch64.zip - asset_name: dora-${{ github.ref_name }}-aarch64-${{ runner.os }}.zip - asset_content_type: application/zip - - mac-arm-release: - name: "MacOS ARM Release" - - strategy: - matrix: - platform: [macos-12] - fail-fast: false - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/checkout@v3 - - - uses: r7kamura/rust-problem-matchers@v1.1.0 - - name: "Add macOS ARM64 toolchains" - if: runner.os == 'macOS' - run: rustup target add aarch64-apple-darwin - - - name: "Build macOS ARM64" - if: runner.os == 'macOS' - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora-cli -p dora-daemon - - - name: "Archive macOS ARM64" - if: runner.os == 'macOS' - run: | - mkdir archive_aarch64 - cp target/aarch64-apple-darwin/release/dora archive_aarch64/dora - cd archive_aarch64 - zip -r ../archive_aarch64.zip . - cd .. - - - name: "Upload macOS ARM64 asset" - if: runner.os == 'macOS' - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: archive_aarch64.zip - asset_name: dora-${{ github.ref_name }}-aarch64-${{ runner.os }}.zip + asset_name: dora-${{ github.ref_name }}-${{ matrix.platform.target }}.zip asset_content_type: application/zip From 0e58dd9617090f9c7fb043253f87eb216f135242 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sat, 6 Jul 2024 13:10:31 +0200 Subject: [PATCH 04/15] Fix windows release --- .github/workflows/pip-release.yml | 4 +++- .github/workflows/release.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index 2c9de7254..d23f922fb 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -126,7 +126,9 @@ jobs: matrix: platform: - runner: windows-latest - target: x86_64 + target: x86 + - runner: windows-latest + target: x64 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ce448bc75..86e574359 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -137,7 +137,7 @@ jobs: target: x86_64-pc-windows-msvc fail-fast: false - runs-on: ${{ matrix.platform }} + runs-on: ${{ matrix.platform.runner }} steps: - uses: actions/checkout@v3 From 76223ee9192f5566b8332e10d7d2fd07825c024a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sat, 6 Jul 2024 13:19:29 +0200 Subject: [PATCH 05/15] Zip only binary --- .github/workflows/release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86e574359..4fe7db800 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,7 +116,10 @@ jobs: args: --release --target ${{ matrix.platform.target }} -p dora-cli - name: "Archive Linux" run: | - zip -r ${{ matrix.platform.target }}.zip target/ ${{ matrix.platform.target }}/release + mkdir ${{ matrix.platform.target }} + cp target/${{ matrix.platform.target }}/release/dora ${{ matrix.platform.target }}/dora + zip -r ${{ matrix.platform.target }}.zip ${{ matrix.platform.target }} + - name: "Upload asset" uses: actions/upload-release-asset@v1.0.1 env: From fff866c425a424edc4398a7451e754f0000fd805 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sat, 6 Jul 2024 13:26:17 +0200 Subject: [PATCH 06/15] Upload wheel with the release --- .github/workflows/pip-release.yml | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index d23f922fb..a464763ba 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -53,6 +53,13 @@ jobs: with: name: wheels-linux-${{ matrix.platform.target }} path: apis/python/node/dist + - name: Upload to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: apis/python/node/dist/* + tag: ${{ github.ref }} + file_glob: true musllinux: runs-on: ${{ matrix.platform.runner }} @@ -85,6 +92,13 @@ jobs: with: name: wheels-musllinux-${{ matrix.platform.target }} path: apis/python/node/dist + - name: Upload to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: apis/python/node/dist/* + tag: ${{ github.ref }} + file_glob: true musleabi: runs-on: ubuntu-latest @@ -119,6 +133,13 @@ jobs: with: name: wheels-musllinux-${{ matrix.platform.target }} path: apis/python/node/dist + - name: Upload to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: apis/python/node/dist/* + tag: ${{ github.ref }} + file_glob: true windows: runs-on: ${{ matrix.platform.runner }} @@ -147,6 +168,13 @@ jobs: with: name: wheels-windows-${{ matrix.platform.target }} path: apis/python/node/dist + - name: Upload to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: apis/python/node/dist/* + tag: ${{ github.ref }} + file_glob: true macos: runs-on: ${{ matrix.platform.runner }} @@ -174,6 +202,13 @@ jobs: with: name: wheels-macos-${{ matrix.platform.target }} path: apis/python/node/dist + - name: Upload to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: apis/python/node/dist/* + tag: ${{ github.ref }} + file_glob: true sdist: runs-on: ubuntu-latest From 1264bd47ad8e0e12f99c94d3e4668194e077ab89 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sun, 7 Jul 2024 04:53:34 +0200 Subject: [PATCH 07/15] Remove compiling of x86 windows python --- .github/workflows/pip-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index a464763ba..cc993e291 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -146,8 +146,6 @@ jobs: strategy: matrix: platform: - - runner: windows-latest - target: x86 - runner: windows-latest target: x64 steps: From 5b526107c3d14268114260cb66f07aa5b884e9b9 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jul 2024 09:37:11 +0200 Subject: [PATCH 08/15] Use musleabihf for armv7 --- .github/workflows/pip-release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index cc993e291..8ff574313 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -72,8 +72,6 @@ jobs: target: x86 - runner: ubuntu-latest target: aarch64 - - runner: ubuntu-latest - target: armv7 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -124,7 +122,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - manylinux: musllinux_1_2 + manylinux: auto container: off args: --release -o dist working-directory: apis/python/node From 469347338a1abd793a591b34706d878b6c5a1638 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jul 2024 05:41:10 +0200 Subject: [PATCH 09/15] fix wheels path searching --- .github/workflows/pip-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index 8ff574313..2c514ac55 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -235,4 +235,4 @@ jobs: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASS }} with: command: upload - args: --non-interactive --skip-existing * + args: --non-interactive --skip-existing wheels-*/* From 46e3788ee9800911e9bd18ceea9e737a046ace98 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jul 2024 07:32:44 +0200 Subject: [PATCH 10/15] Remove archive folder within the zip archive --- .github/workflows/release.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4fe7db800..b35051b3c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -114,11 +114,8 @@ jobs: use-cross: true command: build args: --release --target ${{ matrix.platform.target }} -p dora-cli - - name: "Archive Linux" - run: | - mkdir ${{ matrix.platform.target }} - cp target/${{ matrix.platform.target }}/release/dora ${{ matrix.platform.target }}/dora - zip -r ${{ matrix.platform.target }}.zip ${{ matrix.platform.target }} + - name: "Archive" + run: zip -r ${{ matrix.platform.target }}.zip target/${{ matrix.platform.target }}/release/dora - name: "Upload asset" uses: actions/upload-release-asset@v1.0.1 @@ -154,10 +151,7 @@ jobs: - name: Create Archive (Windows) if: runner.os == 'Windows' shell: powershell - run: | - New-Item -Path archive -ItemType Directory - Copy-Item target/release/dora.exe -Destination archive/dora.exe - Compress-Archive -Path archive\* -DestinationPath archive.zip + run: Compress-Archive -Path target/release/dora.exe -DestinationPath archive.zip - name: "Upload release asset" uses: actions/upload-release-asset@v1.0.1 From 39d7d49de420b01092566cf431d1e81aeeb6c654 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jul 2024 08:06:32 +0200 Subject: [PATCH 11/15] Add installl shell --- .github/install.sh | 186 ++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 2 +- 2 files changed, 187 insertions(+), 1 deletion(-) create mode 100755 .github/install.sh diff --git a/.github/install.sh b/.github/install.sh new file mode 100755 index 000000000..b9d39cc72 --- /dev/null +++ b/.github/install.sh @@ -0,0 +1,186 @@ + +#!/usr/bin/env sh + +set -eu + +if [ -n "${GITHUB_ACTIONS-}" ]; then + set -x +fi + +# Check pipefail support in a subshell, ignore if unsupported +# shellcheck disable=SC3040 +(set -o pipefail 2> /dev/null) && set -o pipefail + +help() { + cat <<'EOF' +Install a binary released on GitHub + +USAGE: + install.sh [options] + +FLAGS: + -h, --help Display this message + -f, --force Force overwriting an existing binary + +OPTIONS: + --repo REPO Github Repository to install the binary from + --bin BIN Name of the binary to install + --tag TAG Tag (version) of the bin to install, defaults to latest release + --to LOCATION Where to install the binary [default: ~/.cargo/bin] + --target TARGET +EOF +} + + +say() { + echo "install: $*" >&2 +} + +err() { + if [ -n "${td-}" ]; then + rm -rf "$td" + fi + + say "error: $*" + exit 1 +} + +need() { + if ! command -v "$1" > /dev/null 2>&1; then + err "need $1 (command not found)" + fi +} + + + +download() { + url="$1" + output="$2" + + if command -v curl > /dev/null; then + curl --proto =https --tlsv1.2 -sSfL "$url" "-o$output" + else + wget --https-only --secure-protocol=TLSv1_2 --quiet "$url" "-O$output" + fi +} + +force=false +while test $# -gt 0; do + case $1 in + --help | -h) + help + exit 0 + ;; + --repo) + repo=$2 + shift + ;; + --bin) + bin=$2 + shift + ;; + --tag) + tag=$2 + shift + ;; + --target) + target=$2 + shift + ;; + --to) + dest=$2 + shift + ;; + *) + say "error: unrecognized argument '$1'. Usage:" + help + exit 1 + ;; + esac + shift +done + +url=https://github.com/$repo/$bin +releases=$url/releases + +command -v curl > /dev/null 2>&1 || + command -v wget > /dev/null 2>&1 || + err "need wget or curl (command not found)" + +need mkdir +need mktemp + +if [ -z "${tag-}" ]; then + need grep + need cut +fi + +if [ -z "${target-}" ]; then + need cut +fi + +if [ -z "${dest-}" ]; then + dest="$HOME/.cargo/bin" +fi + +if [ -z "${tag-}" ]; then + tag=$( + download https://api.github.com/repos/$repo/$bin/releases/latest - | + grep tag_name | + cut -d'"' -f4 + ) +fi + + +if [ -z "${target-}" ]; then + # bash compiled with MINGW (e.g. git-bash, used in github windows runners), + # unhelpfully includes a version suffix in `uname -s` output, so handle that. + # e.g. MINGW64_NT-10-0.19044 + kernel=$(uname -s | cut -d- -f1) + uname_target="$(uname -m)-$kernel" + + case $uname_target in + aarch64-Linux) target=aarch64-unknown-linux-musl;; + arm64-Darwin) target=aarch64-apple-darwin;; + armv7l-Linux) target=armv7-unknown-linux-musleabihf;; + x86_64-Darwin) target=x86_64-apple-darwin;; + x86_64-Linux) target=x86_64-unknown-linux-gnu;; + x86_64-MINGW64_NT) target=x86_64-pc-windows-msvc;; + x86_64-Windows_NT) target=x86_64-pc-windows-msvc;; + *) + # shellcheck disable=SC2016 + err 'Could not determine target from output of `uname -m`-`uname -s`, please use `--target`:' "$uname_target" + ;; + esac +fi + +case $target in + *) extension=zip; need unzip;; +esac + +archive="$releases/download/v$tag/$bin-v$tag-$target.$extension" +say "Repository: $url" +say "bin: $bin" +say "Tag: $tag" +say "Target: $target" +say "Destination: $dest" +say "Archive: $archive" + +td=$(mktemp -d || mktemp -d -t tmp) + +if [ "$extension" = "zip" ]; then + download "$archive" "$td/$bin.zip" + unzip -d "$td" "$td/$bin.zip" +else + download "$archive" - | tar -C "$td" -xz +fi + +if [ -e "$dest/$bin" ] && [ "$force" = false ]; then + err "\`$dest/$bin\` already exists" +else + mkdir -p "$dest" + cp "$td/$bin" "$dest/$bin" + chmod 755 "$dest/$bin" +fi + +rm -rf "$td" \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b35051b3c..74d3a4fb9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -115,7 +115,7 @@ jobs: command: build args: --release --target ${{ matrix.platform.target }} -p dora-cli - name: "Archive" - run: zip -r ${{ matrix.platform.target }}.zip target/${{ matrix.platform.target }}/release/dora + run: zip -j -r ${{ matrix.platform.target }}.zip target/${{ matrix.platform.target }}/release/dora - name: "Upload asset" uses: actions/upload-release-asset@v1.0.1 From 13fb05c4fa5b584b3ce98647790c7687c943de4e Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 19 Jul 2024 10:42:50 +0200 Subject: [PATCH 12/15] Test pip release creation as part of normal CI --- .github/workflows/pip-release.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index 2c514ac55..c1e6a7a58 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -13,6 +13,8 @@ on: types: - "published" workflow_dispatch: + push: + pull_request: permissions: contents: write @@ -49,11 +51,13 @@ jobs: manylinux: auto working-directory: apis/python/node - name: Upload wheels + if: github.event_name == 'release' uses: actions/upload-artifact@v4 with: name: wheels-linux-${{ matrix.platform.target }} path: apis/python/node/dist - name: Upload to release + if: github.event_name == 'release' uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} @@ -86,11 +90,13 @@ jobs: manylinux: musllinux_1_2 working-directory: apis/python/node - name: Upload wheels + if: github.event_name == 'release' uses: actions/upload-artifact@v4 with: name: wheels-musllinux-${{ matrix.platform.target }} path: apis/python/node/dist - name: Upload to release + if: github.event_name == 'release' uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} @@ -127,11 +133,13 @@ jobs: args: --release -o dist working-directory: apis/python/node - name: Upload wheels + if: github.event_name == 'release' uses: actions/upload-artifact@v3 with: name: wheels-musllinux-${{ matrix.platform.target }} path: apis/python/node/dist - name: Upload to release + if: github.event_name == 'release' uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} @@ -160,11 +168,13 @@ jobs: sccache: "true" working-directory: apis/python/node - name: Upload wheels + if: github.event_name == 'release' uses: actions/upload-artifact@v4 with: name: wheels-windows-${{ matrix.platform.target }} path: apis/python/node/dist - name: Upload to release + if: github.event_name == 'release' uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} @@ -194,11 +204,13 @@ jobs: sccache: "true" working-directory: apis/python/node - name: Upload wheels + if: github.event_name == 'release' uses: actions/upload-artifact@v4 with: name: wheels-macos-${{ matrix.platform.target }} path: apis/python/node/dist - name: Upload to release + if: github.event_name == 'release' uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} @@ -217,6 +229,7 @@ jobs: args: --out dist working-directory: apis/python/node - name: Upload sdist + if: github.event_name == 'release' uses: actions/upload-artifact@v3 with: name: wheels @@ -225,7 +238,7 @@ jobs: release: name: Release runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" + if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/') needs: [linux, musllinux, musleabi, windows, macos, sdist] steps: - uses: actions/download-artifact@v4 From 93f68b6b0aebe7070aa6f5df8f102faa3395192f Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 19 Jul 2024 11:15:03 +0200 Subject: [PATCH 13/15] Check build for cross-compiled targets on CI We release for these targets, so we should check them during CI runs too. --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8708ba964..47a1d6b09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -421,3 +421,35 @@ jobs: override: true - name: Typos check with custom config file uses: crate-ci/typos@master + + cross-check: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-20.04 + target: x86_64-unknown-linux-gnu + - runner: ubuntu-20.04 + target: i686-unknown-linux-gnu + - runner: ubuntu-20.04 + target: aarch64-unknown-linux-gnu + - runner: ubuntu-20.04 + target: aarch64-unknown-linux-musl + - runner: ubuntu-20.04 + target: armv7-unknown-linux-musleabihf + - runner: macos-12 + target: aarch64-apple-darwin + - runner: macos-12 + target: x86_64-apple-darwin + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: r7kamura/rust-problem-matchers@v1.1.0 + - name: "Add toolchains" + run: rustup target add ${{ matrix.platform.target }} + - name: "Build" + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: check + args: --target ${{ matrix.platform.target }} -p dora-cli From 99f054ccbf173eb2596f9bb16d4c498e2d5108c0 Mon Sep 17 00:00:00 2001 From: Haixuan Xavier Tao Date: Sun, 21 Jul 2024 08:35:30 +0200 Subject: [PATCH 14/15] Only check pip release on push to main --- .github/workflows/pip-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index c1e6a7a58..1916c0d7a 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -14,6 +14,8 @@ on: - "published" workflow_dispatch: push: + branches: + - main pull_request: permissions: From b55d4bf4c85136c98ae2a28895142fa8c9d86463 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sun, 21 Jul 2024 09:03:26 +0200 Subject: [PATCH 15/15] Remove install.sh --- .github/install.sh | 186 --------------------------------------------- 1 file changed, 186 deletions(-) delete mode 100755 .github/install.sh diff --git a/.github/install.sh b/.github/install.sh deleted file mode 100755 index b9d39cc72..000000000 --- a/.github/install.sh +++ /dev/null @@ -1,186 +0,0 @@ - -#!/usr/bin/env sh - -set -eu - -if [ -n "${GITHUB_ACTIONS-}" ]; then - set -x -fi - -# Check pipefail support in a subshell, ignore if unsupported -# shellcheck disable=SC3040 -(set -o pipefail 2> /dev/null) && set -o pipefail - -help() { - cat <<'EOF' -Install a binary released on GitHub - -USAGE: - install.sh [options] - -FLAGS: - -h, --help Display this message - -f, --force Force overwriting an existing binary - -OPTIONS: - --repo REPO Github Repository to install the binary from - --bin BIN Name of the binary to install - --tag TAG Tag (version) of the bin to install, defaults to latest release - --to LOCATION Where to install the binary [default: ~/.cargo/bin] - --target TARGET -EOF -} - - -say() { - echo "install: $*" >&2 -} - -err() { - if [ -n "${td-}" ]; then - rm -rf "$td" - fi - - say "error: $*" - exit 1 -} - -need() { - if ! command -v "$1" > /dev/null 2>&1; then - err "need $1 (command not found)" - fi -} - - - -download() { - url="$1" - output="$2" - - if command -v curl > /dev/null; then - curl --proto =https --tlsv1.2 -sSfL "$url" "-o$output" - else - wget --https-only --secure-protocol=TLSv1_2 --quiet "$url" "-O$output" - fi -} - -force=false -while test $# -gt 0; do - case $1 in - --help | -h) - help - exit 0 - ;; - --repo) - repo=$2 - shift - ;; - --bin) - bin=$2 - shift - ;; - --tag) - tag=$2 - shift - ;; - --target) - target=$2 - shift - ;; - --to) - dest=$2 - shift - ;; - *) - say "error: unrecognized argument '$1'. Usage:" - help - exit 1 - ;; - esac - shift -done - -url=https://github.com/$repo/$bin -releases=$url/releases - -command -v curl > /dev/null 2>&1 || - command -v wget > /dev/null 2>&1 || - err "need wget or curl (command not found)" - -need mkdir -need mktemp - -if [ -z "${tag-}" ]; then - need grep - need cut -fi - -if [ -z "${target-}" ]; then - need cut -fi - -if [ -z "${dest-}" ]; then - dest="$HOME/.cargo/bin" -fi - -if [ -z "${tag-}" ]; then - tag=$( - download https://api.github.com/repos/$repo/$bin/releases/latest - | - grep tag_name | - cut -d'"' -f4 - ) -fi - - -if [ -z "${target-}" ]; then - # bash compiled with MINGW (e.g. git-bash, used in github windows runners), - # unhelpfully includes a version suffix in `uname -s` output, so handle that. - # e.g. MINGW64_NT-10-0.19044 - kernel=$(uname -s | cut -d- -f1) - uname_target="$(uname -m)-$kernel" - - case $uname_target in - aarch64-Linux) target=aarch64-unknown-linux-musl;; - arm64-Darwin) target=aarch64-apple-darwin;; - armv7l-Linux) target=armv7-unknown-linux-musleabihf;; - x86_64-Darwin) target=x86_64-apple-darwin;; - x86_64-Linux) target=x86_64-unknown-linux-gnu;; - x86_64-MINGW64_NT) target=x86_64-pc-windows-msvc;; - x86_64-Windows_NT) target=x86_64-pc-windows-msvc;; - *) - # shellcheck disable=SC2016 - err 'Could not determine target from output of `uname -m`-`uname -s`, please use `--target`:' "$uname_target" - ;; - esac -fi - -case $target in - *) extension=zip; need unzip;; -esac - -archive="$releases/download/v$tag/$bin-v$tag-$target.$extension" -say "Repository: $url" -say "bin: $bin" -say "Tag: $tag" -say "Target: $target" -say "Destination: $dest" -say "Archive: $archive" - -td=$(mktemp -d || mktemp -d -t tmp) - -if [ "$extension" = "zip" ]; then - download "$archive" "$td/$bin.zip" - unzip -d "$td" "$td/$bin.zip" -else - download "$archive" - | tar -C "$td" -xz -fi - -if [ -e "$dest/$bin" ] && [ "$force" = false ]; then - err "\`$dest/$bin\` already exists" -else - mkdir -p "$dest" - cp "$td/$bin" "$dest/$bin" - chmod 755 "$dest/$bin" -fi - -rm -rf "$td" \ No newline at end of file