From cd5052eee2da0a40a634da4390edf22543d3723a Mon Sep 17 00:00:00 2001 From: Kirill Azovtsev Date: Mon, 27 Jan 2025 00:35:57 +0300 Subject: [PATCH] update zombie builder actions --- .github/workflows/zombie-builder.yml | 269 ++++++++---------- zombienet/docker/Makefile.include/polkadot.mk | 22 ++ 2 files changed, 146 insertions(+), 145 deletions(-) diff --git a/.github/workflows/zombie-builder.yml b/.github/workflows/zombie-builder.yml index fba130e2e8..afc015862f 100644 --- a/.github/workflows/zombie-builder.yml +++ b/.github/workflows/zombie-builder.yml @@ -11,45 +11,59 @@ on: # - cron: '0 2 * * 0' # Update zombie builder image each Sunday night workflow_dispatch: inputs: - polkadot_image_rebuild: - description: 'Rebuild the polkadot builder image' + platform: + description: 'Platform selection' + type: choice + options: + - linux/amd64 + - linux/arm64 + default: linux/amd64 + polkadot_builder: + description: 'Build the polkadot builder image' required: false default: false type: 'boolean' - polkadot_binaries_rebuild: - description: 'Rebuild the polkadot binaries' + polkadot_builder_tag: + description: 'Custom Polkadot Builder tag' + default: 'latest' required: false - default: true - type: 'boolean' - zombie_tester_image_rebuild: - description: 'Rebuild the zombie tester image' + type: 'string' + polkadot_binaries: + description: 'Build the polkadot binaries' required: false - default: false + default: true type: 'boolean' polkadot_sdk_tag: description: 'Custom Polkadot SDK tag' required: false type: 'string' - builder_latest_tag: - description: 'Custom Polkadot Builder tag' - default: 'latest' - required: false - type: 'string' - zombie_tester_latest_tag: - description: 'Custom Zombie Tester tag' - default: 'latest' - required: false - type: 'string' + + # zombie_tester_image_rebuild: + # description: 'Rebuild the zombie tester image' + # required: false + # default: false + # type: 'boolean' + # polkadot_sdk_tag: + # description: 'Custom Polkadot SDK tag' + # required: false + # type: 'string' + # zombie_tester_latest_tag: + # description: 'Custom Zombie Tester tag' + # default: 'latest' + # required: false + # type: 'string' env: DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ GITHUB_HUNTER_USERNAME: ${{ secrets.HUNTER_USERNAME }} GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }} - CACHE_VERSION: v001 - CACHE_PATHS: ./zombienet/docker/cargo + PLATFORM: ${{ github.event.inputs.platform || 'linux/amd64' }} POLKADOT_SDK_TAG: ${{ github.event.inputs.polkadot_sdk_tag }} - BUILDER_LATEST_TAG: ${{ github.event.inputs.builder_latest_tag || 'latest' }} + BUILDER_LATEST_TAG: ${{ github.event.inputs.polkadot_builder_tag || 'latest' }} TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_latest_tag || 'latest' }} + SCCACHE_GCS_BUCKET: ${{ secrets.SCCACHE_GCS_BUCKET }} + CACHE_VERSION: v001 + CACHE_PATHS: ./zombienet/docker/build_docker/cargo jobs: build_polkadot_builder: @@ -72,41 +86,36 @@ jobs: gcloud auth configure-docker --quiet gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet - - name: "Get Polkadot and Zombienet versions" - working-directory: ./zombienet/docker - run: make get_versions - - - name: "Check if image tag exists" + - name: "Check if image tag exists using Makefile" id: check-tag working-directory: ./zombienet/docker run: | - POLKADOT_VERSION=$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2) - POLKADOT_BUILDER_IMAGE=${DOCKER_REGISTRY_PATH}polkadot_builder:${POLKADOT_VERSION} - TAG_EXISTS=$(docker manifest inspect ${POLKADOT_BUILDER_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false") - echo "exists=$TAG_EXISTS" >> $GITHUB_ENV - echo "polkadot_builder_image=$POLKADOT_BUILDER_IMAGE" >> $GITHUB_OUTPUT + make set_versions + exists=$(make polkadot_builder_check_tag) + echo "exists=$exists" >> $GITHUB_ENV - name: "Build Polkadot Builder" working-directory: ./zombienet/docker if: env.exists == 'false' || github.event.inputs.polkadot_image_rebuild == 'true' - run: make polkadot_builder \ - DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} - BUILDER_LATEST_TAG=${BUILDER_LATEST_TAG} - - - name: "Push Polkadot Builder" - working-directory: ./zombienet/docker - if: env.exists == 'false' || github.event.inputs.polkadot_image_rebuild == 'true' - run: make polkadot_builder_push \ - DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} + run: make polkadot_builder_all_arch - name: "Versions Info" if: always() + working-directory: ./zombienet/docker run: | - echo "Polkadot Builder Image: ${{ steps.check-tag.outputs.polkadot_builder_image }}" + make polkadot_builder_image_info building_binaries: runs-on: [ actions-runner-controller ] + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # include: + # - platform: linux/amd64 + # os: ubuntu-24.04 + # - platform: linux/arm64 + # os: ubuntu-24.04-arm needs: [build_polkadot_builder] timeout-minutes: 120 @@ -132,118 +141,88 @@ jobs: gcloud auth configure-docker --quiet gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet - - name: "Get Polkadot and Zombienet versions" - working-directory: ./zombienet/docker - run: make get_versions - - - name: "Setup package version if not rebuilding" - if: github.event.inputs.polkadot_binaries_rebuild == 'false' - working-directory: ./zombienet/docker - run: | - ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') - REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') - gcloud config set artifacts/repository $ARTIFACTS_REPO - gcloud config set artifacts/location $REGION - POLKADOT_BINARY_PACKAGE_VERSION=$(gcloud artifacts versions list --package=polkadot-binary --format=json | \ - jq -r 'sort_by(.createTime) | .[-1] | {version: (.name | split("/") | last), createTime: .createTime} | .version') - echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" | tee -a variables.env - - name: "Build target" if: github.event.inputs.polkadot_binaries_rebuild == 'true' working-directory: ./zombienet/docker - run: make polkadot_binary \ - DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ - SCCACHE_GCS_BUCKET=${{ secrets.SCCACHE_GCS_BUCKET }} \ + run: make polkadot_binary - name: "Push Polkadot APT Package" if: github.event.inputs.polkadot_binaries_rebuild == 'true' working-directory: ./zombienet/docker - id: push_apt_package - run: | - SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) - POLKADOT_RELEASE_GLOBAL_NUMERIC=$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2) - POLKADOT_BINARY_PACKAGE_VERSION="${POLKADOT_RELEASE_GLOBAL_NUMERIC}-${SHORT_COMMIT_HASH}" - echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" | tee -a variables.env - make upload_apt_package - - - name: "Upload Variables Artifact" - uses: actions/upload-artifact@v4 - with: - name: variables-artifact - path: ./zombienet/docker/variables.env - -# - name: Archive build outputs -# uses: actions/upload-artifact@v4 -# with: -# name: build-output -# path: ./zombienet/docker/polkadot_binary/ - - - build_zombie_tester: - runs-on: ubuntu-latest - needs: [building_binaries] - timeout-minutes: 60 - steps: - - name: "Checkout repository" - uses: actions/checkout@v4 - - - name : "Authenticate with Google Cloud" - uses: 'google-github-actions/auth@v2' - with: - credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} - - - name: "Set up Cloud SDK" - uses: 'google-github-actions/setup-gcloud@v2' + run: make upload_apt_package - - name: "Configure Docker for GCR" - run: | - gcloud auth configure-docker --quiet - gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet - - - name: "Get Polkadot and Zombienet versions" + - name: "Debian package Info" + if: always() working-directory: ./zombienet/docker - run: make get_versions - - - name: "Download Variables Artifact" - uses: actions/download-artifact@v4 - with: - name: variables-artifact - path: . - - - name: "Load Variables" - id: load-variables run: | - source variables.env - echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" >> $GITHUB_ENV + make polkadot_deb_package_info - - name: "Check if image tag exists" - id: check-tag - working-directory: ./zombienet/docker - run: | - ZOMBIENET_RELEASE=$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2) - POLKADOT_SDK_RELEASE=$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2) - ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:${POLKADOT_SDK_RELEASE}_${ZOMBIENET_RELEASE} - TAG_EXISTS=$(docker manifest inspect ${ZOMBIE_TESTER_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false") - echo "exists=$TAG_EXISTS" >> $GITHUB_ENV - echo "zombie_tester_image=$ZOMBIE_TESTER_IMAGE" >> $GITHUB_OUTPUT - - - name: "Build Zombie Tester" - working-directory: ./zombienet/docker - if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true' - run: make zombie_tester \ - DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS} \ - PROJECT_ID=${{ secrets.GCP_PROJECT }} \ - TESTER_LATEST_TAG=${{ env.TESTER_LATEST_TAG }} \ - POLKADOT_BINARY_PACKAGE_VERSION="${POLKADOT_BINARY_PACKAGE_VERSION}" - - - name: "Push Zombie Tester" - working-directory: ./zombienet/docker - if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true' - run: make zombie_tester_push \ - DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ - - name: "Versions Info" - if: always() - run: | - echo "Zombie Tester Image: ${{ steps.check-tag.outputs.zombie_tester_image }}" + # build_zombie_tester: + # runs-on: ubuntu-latest + # needs: [building_binaries] + # timeout-minutes: 60 + # steps: + # - name: "Checkout repository" + # uses: actions/checkout@v4 + + # - name : "Authenticate with Google Cloud" + # uses: 'google-github-actions/auth@v2' + # with: + # credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + # - name: "Set up Cloud SDK" + # uses: 'google-github-actions/setup-gcloud@v2' + + # - name: "Configure Docker for GCR" + # run: | + # gcloud auth configure-docker --quiet + # gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + # - name: "Get Polkadot and Zombienet versions" + # working-directory: ./zombienet/docker + # run: make get_versions + + # - name: "Download Variables Artifact" + # uses: actions/download-artifact@v4 + # with: + # name: variables-artifact + # path: . + + # - name: "Load Variables" + # id: load-variables + # run: | + # source variables.env + # echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" >> $GITHUB_ENV + + # - name: "Check if image tag exists" + # id: check-tag + # working-directory: ./zombienet/docker + # run: | + # ZOMBIENET_RELEASE=$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2) + # POLKADOT_SDK_RELEASE=$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2) + # ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:${POLKADOT_SDK_RELEASE}_${ZOMBIENET_RELEASE} + # TAG_EXISTS=$(docker manifest inspect ${ZOMBIE_TESTER_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false") + # echo "exists=$TAG_EXISTS" >> $GITHUB_ENV + # echo "zombie_tester_image=$ZOMBIE_TESTER_IMAGE" >> $GITHUB_OUTPUT + + # - name: "Build Zombie Tester" + # working-directory: ./zombienet/docker + # if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true' + # run: make zombie_tester \ + # DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + # GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS} \ + # PROJECT_ID=${{ secrets.GCP_PROJECT }} \ + # TESTER_LATEST_TAG=${{ env.TESTER_LATEST_TAG }} \ + # POLKADOT_BINARY_PACKAGE_VERSION="${POLKADOT_BINARY_PACKAGE_VERSION}" + + # - name: "Push Zombie Tester" + # working-directory: ./zombienet/docker + # if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true' + # run: make zombie_tester_push \ + # DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + + # - name: "Versions Info" + # if: always() + # run: | + # echo "Zombie Tester Image: ${{ steps.check-tag.outputs.zombie_tester_image }}" diff --git a/zombienet/docker/Makefile.include/polkadot.mk b/zombienet/docker/Makefile.include/polkadot.mk index fe3233f8c5..10cb138b6f 100644 --- a/zombienet/docker/Makefile.include/polkadot.mk +++ b/zombienet/docker/Makefile.include/polkadot.mk @@ -20,6 +20,22 @@ polkadot_builder_push_manifest: docker manifest push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) && \ docker manifest push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) +polkadot_builder_all_arch: set_versions + $(MAKE) polkadot_builder PLATFORM=linux/amd64 ARCHITECTURE=amd64 ; \ + $(MAKE) polkadot_builder_push PLATFORM=linux/amd64 ARCHITECTURE=amd64 ; \ + $(MAKE) polkadot_builder PLATFORM=linux/arm64 ARCHITECTURE=arm64 ; \ + $(MAKE) polkadot_builder_push PLATFORM=linux/arm64 ARCHITECTURE=arm64 ; \ + $(MAKE) polkadot_builder_push_manifest + +polkadot_builder_image_info: set_versions + @echo "---------------------------------" + @echo POLKADOT_BUILDER_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) + @echo POLKADOT_BUILDER_AMD64_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-amd64 + @echo POLKADOT_BUILDER_ARM64_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-arm64 + +polkadot_builder_check_tag: + @docker manifest inspect $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) > /dev/null 2>&1 && echo "true" || echo "false" + polkadot_binary: $(MAKE) docker_run; \ $(MAKE) docker_exec || $(MAKE) docker_stop_clean; \ @@ -96,3 +112,9 @@ upload_apt_package: set_versions gcloud config set artifacts/location $(REGION); \ gcloud artifacts versions delete $(POLKADOT_DEB_PACKAGE_VERSION) --package=polkadot-binary --quiet || true ; \ gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./$(DOCKER_BUILD_DIR_NAME)/home/pkg/$(POLKADOT_DEB_PACKAGE_NAME) + +polkadot_deb_package_info: set_versions + @echo "---------------------------------" + @echo "POLKADOT_SDK_RELEASE: $(POLKADOT_SDK_RELEASE)" + @echo "POLKADOT_DEB_PACKAGE_NAME: $(POLKADOT_DEB_PACKAGE_NAME)" + @echo "POLKADOT_DEB_PACKAGE_VERSION: $(POLKADOT_DEB_PACKAGE_VERSION)" \ No newline at end of file