ci.yml

name: General CI

# This file is a joint CI of parachain and tee-worker, it contains:
# - build (of docker images)
# - format check
# - unit tests
# - integration tests

# Some notes:
#
# [1] the tee-worker part is a modified version of tee-worker/.github/workflows/build_and_test.yml
# with extra triggering control.
#
# [2] the original file (`tee-worker/.github/workflows/build_and_test.yml`) is kept to sync
# upstream changes, therefore we need to manually apply the changes to this file.

# [3] please beware that if a job in `needs` is skipped, its dependent job will also be skipped,
# see https://github.com/actions/runner/issues/491
#
# jobs that will always be executed:
# - fmt
# - set-condition
# - parachain-build-dev
# - identity-build
#
# [4] please note that job-level if `env` is not supported:
# https://github.com/actions/runner/issues/1189
# as a workaround, we need to put it in a step-level or use `needs.outputs`
#

on:
  push:
    branches:
      - dev
    paths-ignore:
      - "**/dependabot.yml"
      - "**/README.md"
  pull_request:
    branches:
      - dev
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
  workflow_dispatch:
    inputs:
      rebuild-parachain:
        type: boolean
        description: rebuild-parachain
        required: true
        default: true
      rebuild-identity:
        type: boolean
        description: rebuild-identity
        required: true
        default: true
      rebuild-bitacross:
        type: boolean
        description: rebuild-bitacross
        required: true
        default: true
      run-multi-worker-test:
        type: boolean
        description: run-multi-worker-test
        required: true
        default: false
      push-docker:
        type: boolean
        description: push-docker
        required: true
        default: false

env:
  CARGO_TERM_COLOR: always
  DOCKER_BUILDKIT: 1
  # the branch or tag on which this workflow is triggered
  # `head_ref` will only be set if the triggering event is `pull_request`
  REF_VERSION: ${{ github.head_ref || github.ref_name }}

concurrency:
  # see https://stackoverflow.com/questions/74117321/if-condition-in-concurrency-in-gha
  # along with the `sequentialise` job below, it guarantees:
  # - for push in dev: all triggered CIs will be run sequentially, none is cancelled
  # - for PR: later triggered CIs will cancel previous runs, maximum one CI will be run
  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
  cancel-in-progress: true

jobs:
  set-condition:
    runs-on: ubuntu-latest
    # see https://github.com/orgs/community/discussions/25722
    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
    outputs:
      rebuild_parachain: ${{ steps.env.outputs.rebuild_parachain }}
      rebuild_identity: ${{ steps.env.outputs.rebuild_identity }}
      rebuild_bitacross: ${{ steps.env.outputs.rebuild_bitacross }}
      push_docker: ${{ steps.env.outputs.push_docker }}
      run_parachain_test: ${{ steps.env.outputs.run_parachain_test }}
      run_identity_test: ${{ steps.env.outputs.run_identity_test }}
      run_bitacross_test: ${{ steps.env.outputs.run_bitacross_test }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # Checks to see if any files in the PR/commit match one of the listed file types.
      # We can use this filter to decide whether or not to build docker images
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: .github/file-filter.yml
          list-files: shell

      - name: Set condition
        id: env
        run: |
          rebuild_parachain=false
          rebuild_identity=false
          rebuild_bitacross=false
          push_docker=false
          run_parachain_test=false
          run_identity_test=false
          if [ "${{ github.event.inputs.rebuild-parachain }}" = "true" ] || [ "${{ steps.filter.outputs.parachain_src }}" = "true" ]; then
            rebuild_parachain=true
          fi
          if [ "${{ github.event.inputs.rebuild-identity }}" = "true" ] || [ "${{ steps.filter.outputs.identity_src }}" = "true" ]; then
            rebuild_identity=true
          fi
          if [ "${{ github.event.inputs.rebuild-bitacross }}" = "true" ] || [ "${{ steps.filter.outputs.bitacross_src }}" = "true" ]; then
            rebuild_bitacross=true
          fi
          if [ "${{ github.event.inputs.push-docker }}" = "true" ]; then
            push_docker=true
          elif [ "${{ github.event_name }}" = 'push' ] && [ "${{ github.ref }}" = 'refs/heads/dev' ]; then
            push_docker=true
          fi
          if [ "${{ steps.filter.outputs.parachain_test }}" = "true" ] || [ "$rebuild_parachain" = "true" ]; then
            run_parachain_test=true
          fi
          if [ "${{ steps.filter.outputs.identity_test }}" = "true" ] || [ "$rebuild_parachain" = "true" ] || [ "$rebuild_identity" = "true" ]; then
            run_identity_test=true
          fi
          if [ "${{ steps.filter.outputs.bitacross_test }}" = "true" ] || [ "$rebuild_parachain" = "true" ] || [ "$rebuild_bitacross" = "true" ]; then
            run_bitacross_test=true
          fi
          echo "rebuild_parachain=$rebuild_parachain" | tee -a $GITHUB_OUTPUT
          echo "rebuild_identity=$rebuild_identity" | tee -a $GITHUB_OUTPUT
          echo "rebuild_bitacross=$rebuild_bitacross" | tee -a $GITHUB_OUTPUT
          echo "push_docker=$push_docker" | tee -a $GITHUB_OUTPUT
          echo "run_parachain_test=$run_parachain_test" | tee -a $GITHUB_OUTPUT
          echo "run_identity_test=$run_identity_test" | tee -a $GITHUB_OUTPUT
          echo "run_bitacross_test=$run_bitacross_test" | tee -a $GITHUB_OUTPUT

  fmt:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install rust toolchain
        run: rustup show

      - name: Install pre-built taplo
        run: |
          mkdir -p $HOME/.local/bin
          wget -q https://github.com/tamasfe/taplo/releases/download/0.8.1/taplo-linux-x86_64.gz
          gzip -d taplo-linux-x86_64.gz
          cp taplo-linux-x86_64 $HOME/.local/bin/taplo
          chmod a+x $HOME/.local/bin/taplo
          echo "$HOME/.local/bin" >> $GITHUB_PATH

      - name: Parachain fmt check
        working-directory: ./parachain
        run: |
          cargo fmt --all -- --check
          taplo fmt --check

      - name: tee-worker fmt check
        working-directory: ./tee-worker
        run: |
          cargo fmt --all -- --check
          taplo fmt --check

      - name: identity-worker enclave-runtime fmt check
        working-directory: ./tee-worker/identity/enclave-runtime
        run: |
          cargo fmt --all -- --check

      - name: bitacross-worker enclave-runtime fmt check
        working-directory: ./tee-worker/bitacross/enclave-runtime
        run: |
          cargo fmt --all -- --check

      - name: Enable corepack and pnpm
        run: corepack enable && corepack enable pnpm

      - name: identity-worker ts-tests install npm deps
        working-directory: ./tee-worker/identity/ts-tests
        # We can't update the lockfile here because the client-api hasn't been generated yet.
        run: pnpm install --frozen-lockfile

      - name: identity-worker check ts code format
        working-directory: ./tee-worker/identity/ts-tests
        run: pnpm run check-format

      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  # sequentialise the workflow runs on `dev` branch
  # the if condition is applied in step level to make this job always `successful`
  sequentialise:
    runs-on: ubuntu-latest
    steps:
      - name: Wait for previous run
        if: ${{ !failure() && (github.event_name == 'push') && (github.ref == 'refs/heads/dev') }}
        uses: litentry/consecutive-workflow-action@main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          interval: 300
          branch: dev

  parachain-clippy:
    runs-on: ubuntu-latest
    needs:
      - fmt
      - set-condition
      - sequentialise
    if: needs.set-condition.outputs.rebuild_parachain == 'true'
    steps:
      - uses: actions/checkout@v4

      - name: Install rust toolchain
        run: rustup show

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -yq openssl clang libclang-dev cmake protobuf-compiler

      - name: Run cargo clippy check
        run: make clippy

      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  tee-check:
    runs-on: ubuntu-latest
    needs:
      - fmt
      - set-condition
      - sequentialise
    container: "litentry/litentry-tee-dev:latest"
    steps:
      - uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          sudo apt-get update && \
          sudo apt-get install -yq openssl clang libclang-dev cmake protobuf-compiler

      - name: Cargo test
        working-directory: ./tee-worker
        run: |
          cargo test --release

      - name: Check clippy
        working-directory: ./tee-worker
        shell: bash
        run: |
          for d in . identity/enclave-runtime bitacross/enclave-runtime; do
            pushd "$d"
            echo "::group::cargo clippy all"
            cargo clippy --release -- -D warnings
            echo "::endgroup::"
            echo "::group::cargo clippy development"
            cargo clippy --release --features development -- -D warnings
            echo "::endgroup::"
            echo "::group::cargo clippy offchain-worker"
            cargo clippy --release --features offchain-worker -- -D warnings
            echo "::group::cargo clippy offchain-worker,development"
            cargo clippy --release --features offchain-worker,development -- -D warnings
            echo "::endgroup::"
            popd
          done

      - name: Identity-worker specific clippy
        working-directory: ./tee-worker/identity/enclave-runtime
        shell: bash
        run: |
          echo "::group::cargo clippy sidechain"
          cargo clippy --release --features sidechain -- -D warnings
          echo "::endgroup::"
          echo "::group::cargo clippy evm"
          cargo clippy --release --features evm -- -D warnings
          echo "::endgroup::"
          echo "::group::cargo clippy sidechain development"
          cargo clippy --release --features sidechain,development -- -D warnings
          echo "::endgroup::"
          echo "::group::cargo clippy evm development"
          cargo clippy --release --features evm,development -- -D warnings
          echo "::endgroup::"

      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  parachain-build-dev:
    runs-on: ubuntu-latest
    needs:
      - fmt
      - set-condition
      - sequentialise
    steps:
      - uses: actions/checkout@v4

      - name: Free up disk space
        if: startsWith(runner.name, 'GitHub Actions')
        uses: ./.github/actions/disk-cleanup

      - name: Build docker image
        working-directory: ./parachain
        if: needs.set-condition.outputs.rebuild_parachain == 'true'
        run: |
          echo "::group::build docker image"
          ./scripts/build-docker.sh release latest --features=fast-runtime
          echo "::endgroup::"
          echo "::group::docker images"
          docker images --all
          echo "::endgroup::"

      - name: Pull docker image optionally
        if: needs.set-condition.outputs.rebuild_parachain == 'false'
        run: |
          docker pull litentry/litentry-parachain:latest
          docker pull litentry/litentry-chain-aio:latest

      - name: Save docker image
        run: |
          docker save litentry/litentry-parachain:latest litentry/litentry-chain-aio:latest | gzip > litentry-parachain-dev.tar.gz

      - name: Upload parachain docker image
        uses: actions/upload-artifact@v4
        with:
          name: litentry-parachain-dev
          path: litentry-parachain-dev.tar.gz
          if-no-files-found: error

      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  identity-build:
    runs-on: ubuntu-latest
    needs:
      - fmt
      - set-condition
      - sequentialise
    steps:
      - uses: actions/checkout@v4

      - name: Free up disk space
        if: startsWith(runner.name, 'GitHub Actions')
        uses: ./.github/actions/disk-cleanup

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          # use the docker driver to access the local image
          # we don't need external caches or multi platforms here
          # see https://docs.docker.com/build/drivers/
          driver: docker

      - name: Cache worker-cache
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        uses: actions/cache@v4
        with:
          path: |
            worker-cache
          key: worker-cache-${{ env.REF_VERSION }}-${{ hashFiles('tee-worker/identity/**/Cargo.lock', 'tee-worker/identity/**/Cargo.toml') }}
          restore-keys: |
            worker-cache-${{ env.REF_VERSION }}-
            worker-cache-

      - name: Create cache folder if not exist
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        run: |
          for i in 'git/db' 'registry/cache' 'registry/index' 'sccache'; do
            [ ! -d "worker-cache/$i" ] && mkdir -p "worker-cache/$i" || true
            echo "hello" > worker-cache/$i/nix
          done
          echo "::group::List worker-cache size"
          du -sh worker-cache/*
          echo "::endgroup::"
          echo "::group::Show disk usage"
          df -h .
          echo "::endgroup::"

      # cache mount in buildkit won't be exported as image layers, so it doesn't work well with GHA cache, see
      # https://github.com/moby/buildkit/issues/1512
      # https://www.reddit.com/r/rust/comments/126xeyx/exploring_the_problem_of_faster_cargo_docker/
      # https://hackmd.io/@kobzol/S17NS71bh (a great summary)
      #
      # the `reproducible-containers/buildkit-cache-dance` seems to be unstable too in my test
      # sometimes it works, sometimes it results in empty cache, root cause unclear
      #
      # thus the persistence of rust cache (w/ sccache) is maintained manually
      #
      # there's no cache-from/to as the docker image layer is too large and it takes too long to sync
      # moreoever, we are not so eager to have layer caches, as the most critical operation(cargo build)
      # is "atomic" and can't be broken into layers.
      - name: Build local builder
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: tee-worker/identity/build.Dockerfile
          tags: local-builder:latest
          target: builder
          build-args: |
            WORKER_MODE_ARG=sidechain
            WORKER_ENV_DATA_PROVIDERS_CONFIG=1
            WORKER_MOCK_SERVER=1
            ADDITIONAL_FEATURES_ARG=

      - name: Copy caches from the built image
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        run: |
          echo "::group::Show disk usage"
          df -h .
          echo "::endgroup::"
          echo "::group::docker images"
          docker images --all
          echo "::endgroup::"
          echo "::group::copy cache out"
          for i in 'git/db' 'registry/cache' 'registry/index'; do
            b="${i%/*}"
            rm -rf worker-cache/$i
            docker cp "$(docker create --rm local-builder:latest):/opt/rust/$i" worker-cache/$b
          done
          rm -rf worker-cache/sccache
          docker cp "$(docker create --rm local-builder:latest):/opt/rust/sccache" worker-cache
          du -sh worker-cache/*
          echo "::endgroup::"
          echo "::group::df -h ."
          df -h .
          echo "::endgroup::"

      - name: Build worker
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: tee-worker/identity/build.Dockerfile
          tags: litentry/identity-worker:latest
          target: deployed-worker

      - name: Build cli
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: tee-worker/identity/build.Dockerfile
          tags: litentry/identity-cli:latest
          target: deployed-client

      - name: Pull and tag worker and cli image optionally
        if: needs.set-condition.outputs.rebuild_identity == 'false'
        run: |
          docker pull litentry/identity-worker:latest
          docker pull litentry/identity-cli:latest

      - run: docker images --all

      - name: Test enclave
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        # cargo test is not supported in the enclave
        # see https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
        run: docker run litentry/identity-worker:latest test --all

      - name: Save docker images
        run: docker save litentry/identity-worker:latest litentry/identity-cli:latest | gzip > litentry-identity.tar.gz

      - name: Upload docker images
        uses: actions/upload-artifact@v4
        with:
          name: litentry-identity
          path: litentry-identity.tar.gz
          if-no-files-found: error
      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  bitacross-build:
    runs-on: ubuntu-latest
    needs:
      - fmt
      - set-condition
      - sequentialise
    steps:
      - uses: actions/checkout@v4

      - name: Free up disk space
        if: startsWith(runner.name, 'GitHub Actions')
        uses: ./.github/actions/disk-cleanup

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          # use the docker driver to access the local image
          # we don't need external caches or multi platforms here
          # see https://docs.docker.com/build/drivers/
          driver: docker

      - name: Build local builder
        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: tee-worker/bitacross/build.Dockerfile
          tags: local-builder:latest
          target: builder
          build-args: |
            WORKER_MODE_ARG=offchain-worker
            ADDITIONAL_FEATURES_ARG=

      - name: Build worker
        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: tee-worker/bitacross/build.Dockerfile
          tags: litentry/bitacross-worker:latest
          target: deployed-worker

      - name: Build cli
        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: tee-worker/bitacross/build.Dockerfile
          tags: litentry/bitacross-cli:latest
          target: deployed-client

      - name: Pull and tag worker and cli image optionally
        if: needs.set-condition.outputs.rebuild_bitacross == 'false'
        run: |
          docker pull litentry/bitacross-worker:latest
          docker pull litentry/bitacross-cli:latest

      - run: docker images --all

      - name: Test enclave
        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
        # cargo test is not supported in the enclave
        # see https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
        run: docker run litentry/bitacross-worker:latest test --all

      - name: Save docker images
        run: docker save litentry/bitacross-worker:latest litentry/bitacross-cli:latest | gzip > litentry-bitacross.tar.gz

      - name: Upload docker images
        uses: actions/upload-artifact@v4
        with:
          name: litentry-bitacross
          path: litentry-bitacross.tar.gz
          if-no-files-found: error
      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  parachain-ts-test:
    runs-on: ubuntu-latest
    needs:
      - set-condition
      - parachain-build-dev
    strategy:
      matrix:
        chain:
          - litentry
          - rococo
          - paseo
    name: ${{ matrix.chain }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: litentry-parachain-dev

      - name: Load docker image
        run: |
          docker load < litentry-parachain-dev.tar.gz

      - name: Enable corepack and pnpm
        run: corepack enable && corepack enable pnpm

      - name: Run ts tests for ${{ matrix.chain }}
        if: needs.set-condition.outputs.run_parachain_test == 'true'
        timeout-minutes: 35
        run: |
          make test-ts-${{ matrix.chain }}

      - name: Collect docker logs if test fails
        continue-on-error: true
        uses: jwalton/gh-docker-logs@v2
        if: failure()
        with:
          tail: all
          dest: docker-logs

      - name: Upload docker logs if test fails
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.chain }}-ts-tests-docker-logs
          path: docker-logs
          if-no-files-found: ignore
          retention-days: 3

      - name: Archive logs if test fails
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.chain }}-ts-tests-artifact
          path: /tmp/parachain_dev/
          if-no-files-found: ignore
          retention-days: 3

      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  parachain-unit-test:
    runs-on: ubuntu-latest
    needs:
      - fmt
      - set-condition
      - sequentialise
    # run_parachain_test is related to ts-tests only
    if: needs.set-condition.outputs.rebuild_parachain == 'true'
    steps:
      - uses: actions/checkout@v4

      - name: Run all unittests
        working-directory: ./parachain
        run: |
          echo "::group::core-primitives unittest"
          cargo test --locked -p core-primitives --lib
          echo "::endgroup::"
          echo "::group::all pallets unittest"
          cargo test --locked -p pallet-* --lib
          echo "::endgroup::"
          echo "::group::all pallets unittest with runtime-benchmarks feature"
          cargo test --locked -p pallet-* --lib --features=runtime-benchmarks
          echo "::endgroup::"

      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  parachain-runtime-test:
    runs-on: ubuntu-latest
    needs:
      - fmt
      - set-condition
      - sequentialise
    if: needs.set-condition.outputs.rebuild_parachain == 'true'
    steps:
      - uses: actions/checkout@v4

      - name: Free up disk space
        if: startsWith(runner.name, 'GitHub Actions')
        uses: ./.github/actions/disk-cleanup

      - name: Install toolchain
        working-directory: ./parachain
        run: rustup show

      - name: Install dependencies
        run: |
          sudo apt-get update && \
          sudo apt-get install -yq openssl clang libclang-dev cmake protobuf-compiler

      # We could have used matrix but the runtime tests are executed sequentially for a cleaner GHA visualisation graph.
      # It won't take much longer as we run them back to back.
      - name: Run runtime tests
        working-directory: ./parachain
        run: |
          echo "::group::rococo runtime test"
          cargo test --locked -p rococo-parachain-runtime --lib
          echo "::endgroup::"
          echo "::group::litentry runtime test"
          cargo test --locked -p litentry-parachain-runtime --lib
          echo "::endgroup::"

      - name: Fail early
        if: failure()
        uses: andymckay/cancel-action@0.5

  identity-single-worker-test:
    runs-on: ubuntu-latest
    needs:
      - set-condition
      - parachain-build-dev
      - identity-build
    strategy:
      fail-fast: false
      matrix:
        include:
          - test_name: lit-di-substrate-identity-test
          - test_name: lit-di-evm-identity-test
          - test_name: lit-di-bitcoin-identity-test
          - test_name: lit-di-solana-identity-test
          - test_name: lit-dr-vc-test
          - test_name: lit-parentchain-nonce
          - test_name: lit-test-failed-parentchain-extrinsic
          - test_name: lit-twitter-identity-test
          - test_name: lit-discord-identity-test
    name: ${{ matrix.test_name }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: litentry-parachain-dev

      - uses: actions/download-artifact@v4
        with:
          name: litentry-identity

      - name: Load docker image
        run: |
          docker load < litentry-parachain-dev.tar.gz
          docker load < litentry-identity.tar.gz
          docker images

      - name: Enable corepack and pnpm
        run: corepack enable && corepack enable pnpm

      - name: Launch parachain network
        run: |
          make launch-network-litentry

      - name: Integration single worker test ${{ matrix.test_name }}
        working-directory: ./tee-worker/identity/docker
        if: needs.set-condition.outputs.run_identity_test == 'true'
        timeout-minutes: 40
        run: |
          docker compose -f docker-compose.yml -f ${{ matrix.test_name }}.yml up --no-build --exit-code-from ${{ matrix.test_name }} ${{ matrix.test_name }}

      - name: Stop integration single worker docker containers
        working-directory: ./tee-worker/identity/docker
        if: needs.set-condition.outputs.run_identity_test == 'true'
        run: |
          docker compose -f docker-compose.yml -f ${{ matrix.test_name }}.yml stop

      - name: Collect docker logs if test fails
        continue-on-error: true
        uses: jwalton/gh-docker-logs@v2
        if: failure()
        with:
          tail: all
          dest: docker-logs

      - name: Upload docker logs if test fails
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.test_name }}-docker-logs
          path: docker-logs
          if-no-files-found: ignore
          retention-days: 3

  identity-multi-worker-test:
    runs-on: ubuntu-latest
    continue-on-error: true
    if: ${{ github.event.inputs.run-multi-worker-test == 'true' }} # only if triggered manually
    needs:
      - set-condition
      - parachain-build-dev
      - identity-build
    strategy:
      fail-fast: false
      matrix:
        include:
          - test_name: lit-di-bitcoin-identity-multiworker-test
          - test_name: lit-di-evm-identity-multiworker-test
          - test_name: lit-di-solana-identity-multiworker-test
          - test_name: lit-di-substrate-identity-multiworker-test
          - test_name: lit-dr-vc-multiworker-test
          - test_name: lit-resume-worker
    name: ${{ matrix.test_name }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: litentry-parachain-dev

      - uses: actions/download-artifact@v4
        with:
          name: litentry-identity

      - name: Load docker image
        run: |
          docker load < litentry-parachain-dev.tar.gz
          docker load < litentry-identity.tar.gz
          docker images

      - name: Enable corepack and pnpm
        run: corepack enable && corepack enable pnpm

      - name: Launch parachain network
        run: |
          make launch-network-litentry

      - name: Integration multi worker test ${{ matrix.test_name }}
        working-directory: ./tee-worker/identity/docker
        if: needs.set-condition.outputs.run_identity_test == 'true'
        timeout-minutes: 40
        run: |
          docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml up --no-build --exit-code-from ${{ matrix.test_name }} ${{ matrix.test_name }}

      - name: Stop integration multi worker docker containers
        working-directory: ./tee-worker/identity/docker
        if: needs.set-condition.outputs.run_identity_test == 'true'
        run: |
          docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml stop

      - name: Collect docker logs if test fails
        continue-on-error: true
        uses: jwalton/gh-docker-logs@v2
        if: failure()
        with:
          tail: all
          dest: docker-logs

      - name: Upload docker logs if test fails
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.test_name }}-docker-logs
          path: docker-logs
          if-no-files-found: ignore
          retention-days: 3

  bitacross-worker-test:
    runs-on: ubuntu-latest
    needs:
      - set-condition
      - parachain-build-dev
      - bitacross-build
    strategy:
      fail-fast: false
      matrix:
        include:
          - test_name: lit-sign-bitcoin
    name: ${{ matrix.test_name }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: litentry-parachain-dev

      - uses: actions/download-artifact@v4
        with:
          name: litentry-bitacross

      - name: Load docker image
        run: |
          docker load < litentry-parachain-dev.tar.gz
          docker load < litentry-bitacross.tar.gz
          docker images

      - name: Enable corepack and pnpm
        run: corepack enable && corepack enable pnpm

      - name: Launch parachain network
        run: |
          make launch-network-litentry

      - name: Integration bitacross worker test ${{ matrix.test_name }}
        working-directory: ./tee-worker/bitacross/docker
        if: needs.set-condition.outputs.run_bitacross_test == 'true'
        timeout-minutes: 40
        run: |
          docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml up --no-build --exit-code-from ${{ matrix.test_name }} ${{ matrix.test_name }}

      - name: Stop integration multi worker docker containers
        working-directory: ./tee-worker/bitacross/docker
        if: needs.set-condition.outputs.run_bitacross_test == 'true'
        run: |
          docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml stop

      - name: Collect docker logs if test fails
        continue-on-error: true
        uses: jwalton/gh-docker-logs@v2
        if: failure()
        with:
          tail: all
          dest: docker-logs

      - name: Upload docker logs if test fails
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.test_name }}-docker-logs
          path: docker-logs
          if-no-files-found: ignore
          retention-days: 3

  # Secrets are not passed to the runner when a workflow is triggered from a forked repository,
  # see https://docs.github.com/en/actions/security-guides/encrypted-secrets#using-encrypted-secrets-in-a-workflow
  #
  # Only try to push docker image when
  #   - parachain-ts-test passes
  #   - identity-single-worker-test passes
  #   - set-condition.outputs.push_docker is `true`
  # Whether the parachain or tee-worker image will actually be pushed still depends on if a new image was built/rebuilt.
  # This is important not to overwrite any other jobs where a rebuild **was** triggered.
  #
  # We don't have to depend on jobs like `parachain-unit-test` as they have the same trigger condition `rebuild_parachain`,
  # so there must be no new image if `parachain-unit-test` is skipped.
  #
  # `!failure()` needs to be used to cover skipped jobs
  push-docker:
    runs-on: ubuntu-latest
    needs:
      - set-condition
      - parachain-ts-test
      - identity-single-worker-test
      - bitacross-worker-test
    if: ${{ !failure() && needs.set-condition.outputs.push_docker == 'true' }}
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: litentry-parachain-dev

      - uses: actions/download-artifact@v4
        with:
          name: litentry-identity

      - uses: actions/download-artifact@v4
        with:
          name: litentry-bitacross

      - name: Load docker image
        run: |
          docker load < litentry-parachain-dev.tar.gz
          docker load < litentry-identity.tar.gz
          docker load < litentry-bitacross.tar.gz

      - name: Dockerhub login
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      - name: Push parachain image
        if: needs.set-condition.outputs.rebuild_parachain == 'true'
        run: |
          docker push litentry/litentry-parachain
          docker push litentry/litentry-chain-aio

      - name: Push tee-worker image
        if: needs.set-condition.outputs.rebuild_identity == 'true'
        run: |
          docker push litentry/identity-worker
          docker push litentry/identity-cli

      - name: Push bitacross-worker image
        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
        run: |
          docker push litentry/bitacross-worker
          docker push litentry/bitacross-cli