From e785829c46f06d36f3b191e2fafae0844509a999 Mon Sep 17 00:00:00 2001 From: zerg-su <65297783+zerg-su@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:13:26 +0300 Subject: [PATCH] CI - version fix and cache disable option (#2212) CI - version fix and cache disable option --------- Co-authored-by: Kirill Azovtsev --- .github/workflows/test.yml | 42 ++++++++++++++++++++++--- .github/workflows/zombie-tests.yaml | 16 +++++++++- housekeeping/docker/kagome-dev/Makefile | 2 ++ zombienet/docker/Makefile | 5 +-- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index efca05ab0c..b403ac6b6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,14 @@ on: options: - 'true' - 'false' + use_cache: + description: 'Use cache for build' + required: true + default: 'true' + type: 'choice' + options: + - 'true' + - 'false' env: BUILD_DIR: build @@ -40,6 +48,7 @@ env: DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ PROJECT_ID: ${{ secrets.GCP_PROJECT }} CACHE_PATH: ./build/cache + USE_CACHE: ${{ github.event.inputs.use_cache || 'true' }} DEFAULT_BUILD_TYPE: Release GIT_REF_NAME: ${{ github.ref_name }} IS_MAIN_OR_TAG: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) }} @@ -52,6 +61,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 + if: ${{ env.USE_CACHE == 'true' }} with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ env.CACHE_VERSION }} @@ -84,6 +94,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 + if: ${{ env.USE_CACHE == 'true' }} with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} @@ -123,6 +134,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 + if: ${{ env.USE_CACHE == 'true' }} with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} @@ -140,6 +152,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 + if: ${{ env.USE_CACHE == 'true' }} with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ env.CACHE_VERSION }} @@ -186,6 +199,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 + if: ${{ env.USE_CACHE == 'true' }} with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} @@ -202,6 +216,11 @@ jobs: steps: - name: "Checkout repository" uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Get master branch" + run: git fetch origin master:master - name: "Authenticate with Google Cloud" uses: 'google-github-actions/auth@v2' @@ -216,14 +235,19 @@ jobs: gcloud auth configure-docker --quiet gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + - name: "Get commit version" + working-directory: ./housekeeping/docker/kagome-dev + run: make get_versions + - name: "Check version" working-directory: ./housekeeping/docker/kagome-dev run: | SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) - echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV + echo "short_commit_hash=${SHORT_COMMIT_HASH}" | tee $GITHUB_ENV - name: "Cache dependencies" id: cache-restore + if: ${{ env.USE_CACHE == 'true' }} uses: actions/cache/restore@v4 with: path: ${{ env.CACHE_PATH }} @@ -275,6 +299,11 @@ jobs: steps: - name: "Checkout repository" uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Get master branch" + run: git fetch origin master:master - name: "Authenticate with Google Cloud" uses: 'google-github-actions/auth@v2' @@ -289,7 +318,7 @@ jobs: gcloud auth configure-docker --quiet gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet - - name: "Get Polkadot and Zombienet versions" + - name: "Get commit version" working-directory: ./housekeeping/docker/kagome-dev run: make get_versions @@ -309,7 +338,7 @@ jobs: echo "package_exist=${PACKAGE_EXIST}" >> $GITHUB_ENV - name: "Cache dependencies" - if: env.package_exist == 'False' + if: ${{ env.package_exist == 'False' && env.USE_CACHE == 'true' }} id: cache-restore uses: actions/cache/restore@v4 with: @@ -439,6 +468,11 @@ jobs: steps: - name: "Checkout repository" uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Get master branch" + run: git fetch origin master:master # - name: Create and enable swap # run: | @@ -465,7 +499,7 @@ jobs: gcloud auth configure-docker --quiet gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet - - name: "Get Polkadot and Zombienet versions" + - name: "Get commit version" working-directory: ./housekeeping/docker/kagome-dev run: make get_versions diff --git a/.github/workflows/zombie-tests.yaml b/.github/workflows/zombie-tests.yaml index 6792a5b25c..6910f94803 100644 --- a/.github/workflows/zombie-tests.yaml +++ b/.github/workflows/zombie-tests.yaml @@ -26,6 +26,14 @@ on: options: - 'ON' - 'OFF' + use_cache: + description: 'Use cache for build' + required: true + default: 'true' + type: 'choice' + options: + - 'true' + - 'false' env: DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ @@ -36,6 +44,7 @@ env: CACHE_VERSION: v01 CACHE_PATH: ./build/cache + USE_CACHE: ${{ github.event.inputs.use_cache || 'true' }} jobs: kagome_dev_docker_build: @@ -45,6 +54,11 @@ jobs: steps: - name: "Checkout repository" uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Get master branch" + run: git fetch origin master:master - name: "Authenticate with Google Cloud" uses: 'google-github-actions/auth@v2' @@ -79,7 +93,7 @@ jobs: echo "package_exist=${PACKAGE_EXIST}" >> $GITHUB_ENV - name: "Cache dependencies" - if: env.package_exist == 'False' + if: ${{ env.package_exist == 'False' && env.USE_CACHE == 'true' }} id: cache-restore uses: actions/cache/restore@v4 with: diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile index dda73c914b..7dfd71bb8b 100644 --- a/housekeeping/docker/kagome-dev/Makefile +++ b/housekeeping/docker/kagome-dev/Makefile @@ -23,6 +23,7 @@ RUST_VERSION ?= 1.79.0 # kagome_dev_docker_build Variables BUILD_DIR ?= build CACHE_DIR := $(shell pwd)/../../../../kagome/$(BUILD_DIR)/cache +WORKING_DIR := $(shell pwd)/../../../../kagome DEPENDENCIES ?= libstdc++6, zlib1g, libgcc-s1, libc6, libtinfo6, libseccomp2, libatomic1 GITHUB_HUNTER_USERNAME ?= GITHUB_HUNTER_TOKEN ?= @@ -50,6 +51,7 @@ export BUILDKIT_PROGRESS=auto get_versions: @echo "full_commit_hash: `git rev-parse HEAD`" | tee commit_hash.txt @echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" | tee -a commit_hash.txt + @echo "kagome_version: `cd $(WORKING_DIR) && ./get_version.sh`" | tee kagome_version.txt kagome_builder_deb: docker build --platform $(PLATFORM) \ diff --git a/zombienet/docker/Makefile b/zombienet/docker/Makefile index 656676ccac..6171731a3d 100644 --- a/zombienet/docker/Makefile +++ b/zombienet/docker/Makefile @@ -90,8 +90,9 @@ endef get_versions: python3 version.py https://github.com/paritytech/polkadot-sdk && \ python3 version.py https://github.com/paritytech/zombienet - echo "full_commit_hash: `git rev-parse HEAD`" > commit_hash.txt - echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" >> commit_hash.txt + @echo "full_commit_hash: `git rev-parse HEAD`" | tee commit_hash.txt + @echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" | tee -a commit_hash.txt + @echo "kagome_version: `cd $(WORKING_DIR) && ./get_version.sh`" | tee kagome_version.txt polkadot_builder: $(MAKE) get_versions