Clear Kagome docker Build makefile

qdrvm · Jul 21, 2024 · 2df8391 · 2df8391
1 parent 60a8e13
commit 2df8391
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 167 deletions.
diff --git a/.github/workflows/zombie-tests.yaml b/.github/workflows/zombie-tests.yaml
@@ -22,7 +22,6 @@ on:
 env:
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
   BUILD_DIR: build
-  KAGOME_IN_DOCKER: 1
   CTEST_OUTPUT_ON_FAILURE: 1
 
   CACHE_VERSION: v01
@@ -138,18 +137,18 @@ jobs:
     steps:
       - name: "Checkout repository"
         uses: actions/checkout@v4
-        
-#      - name: Create and enable swap
-#        run: |
-#          free -h
-#          df -h
-#          sudo fallocate -l 4G /swapfile
-#          sudo chmod 600 /swapfile
-#          sudo mkswap /swapfile
-#          sudo swapon /swapfile
-#          sudo swapon --show
-#          free -h
-#          df -h
+
+      - name: Create and enable swap
+        run: |
+          free -h
+          df -h
+          sudo fallocate -l 4G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          sudo swapon --show
+          free -h
+          df -h
         
       - name: "Authenticate with Google Cloud"
         uses: 'google-github-actions/auth@v2'
@@ -205,136 +204,3 @@ jobs:
         run: |
           free -h
           df -h
-        
-#  Prepare-Kagome-Docker:
-#    strategy:
-#      fail-fast: false
-#      matrix:
-#        options:
-##          - name: "Self-hosted: Debug Docker image"
-##            build-type: "Debug"
-#          - name: "Self-hosted: Release Docker image"
-#            build-type: "Release"
-##          - name: "Self-hosted: RelWithDebInfo Docker image"
-##            build-type: "RelWithDebInfo"
-#    runs-on: [ actions-runner-controller ]
-#    timeout-minutes: 150
-#    container: qdrvm/kagome-dev:testing
-#    steps:
-#      - uses: actions/checkout@v4
-#      - uses: actions/cache@v4
-#        with:
-#          path: ${{ env.CACHE_PATHS }}
-#          key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }}
-#      - run: git config --global --add safe.directory /__w/kagome/kagome
-#      - run: git fetch --prune --unshallow
-#      - name: Build Kagome
-#        env:
-#          BUILD_TYPE: "${{ matrix.options.build-type }}"
-#          KAGOME_ROOT: /__w/kagome/kagome
-#        run: bash -c "cd $KAGOME_ROOT && source /venv/bin/activate && ./housekeeping/docker/kagome-dev/make.sh"
-#      - name: Copy Kagome
-#        env:
-#          KAGOME_BUILD_DIR: /__w/kagome/kagome/build
-#        run: bash -c "mkdir -p ${KAGOME_BUILD_DIR}/docker_context && cp -a ${KAGOME_BUILD_DIR}/node/kagome ${KAGOME_BUILD_DIR}/docker_context"
-#      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v3
-#      - name: Build Docker Image
-#        uses: docker/build-push-action@v5
-#        with:
-#          tags: kagome-dev:${{ github.run_number }}
-#          file: ./housekeeping/docker/kagome/minideb-release.Dockerfile
-#          context: /__w/kagome/kagome/build/docker_context
-#          outputs: type=docker,dest=/tmp/kagome_dev.tar
-#      - name: upload artifact
-#        uses: actions/upload-artifact@v4
-#        with:
-#          name: kagome-dev
-#          path: /tmp/kagome_dev.tar
-
-#  Prepare-Zombie-Tests-Docker:
-#    runs-on: [ actions-runner-controller ]
-#    timeout-minutes: 60
-#    needs: [Prepare-Kagome-Docker]
-#    steps:
-#      - name: Download artifact
-#        uses: actions/download-artifact@v4
-#        with:
-#          name: kagome-dev
-#          path: /tmp
-#      - name: Load image
-#        run: |
-#          docker load --input /tmp/kagome_dev.tar
-#          docker image ls -a          
-#      - name: Set owner of working dir recursively
-#        run: sudo chown -R $(whoami) .
-#      - uses: actions/checkout@v4
-#      - name: Extract branch name
-#        shell: bash
-#        run: |
-#          branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
-#          echo "branch=${branch}" >> $GITHUB_OUTPUT
-#          echo "Branch name: ${branch}"
-#        id: extract_branch
-#      - name: Build zombie-tester
-#        working-directory: ./zombienet/docker
-#        run: make tester BRANCH_NAME=${{ steps.extract_branch.outputs.branch }} KAGOME_IMAGE=kagome-dev:${{ github.run_number }}
-#      - name: Archive zombie-tester
-#        shell: bash
-#        run: docker save -o /tmp/zombie-tester.tar zombie-tester:latest
-#      - name: Upload artifact
-#        uses: actions/upload-artifact@v4
-#        with:
-#          name: zombie-tester
-#          path: /tmp/zombie-tester.tar
-#
-#  Zombie-Tests:
-#    runs-on: [ actions-runner-controller ]
-#    timeout-minutes: 60
-#    needs: [Prepare-Zombie-Tests-Docker]
-#    strategy:
-#      fail-fast: false
-#      matrix:
-#        options:
-#          - name: "PVF preparation & execution time"
-#            test: "test-polkadot-functional-0001-parachains-pvf"
-#          - name: "Disputes initiation, conclusion and lag"
-#            test: "test-polkadot-functional-0002-parachains-disputes"
-#          - name: "BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute"
-#            test: "test-polkadot-functional-0003-beefy-and-mmr"
-#          - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block"
-#            test: "test-polkadot-functional-0004-parachains-garbage-candidate"
-#          - name: "Past-session dispute slashing"
-#            test: "test-polkadot-functional-0005-parachains-disputes-past-session"
-#          - name: "Test if parachains make progress with most of approvals being tranch0"
-#            test: "test-polkadot-functional-0006-parachains-max-tranche0"
-#          - name: "Test if disputes triggered on finalized blocks within scope always end as valid"
-#            test: "test-polkadot-functional-0007-dispute-freshly-finalized"
-#          - name: "Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded"
-#            test: "test-polkadot-functional-0008-dispute-old-finalized"
-#          - name: "Approval voting coalescing does not lag finality"
-#            test: "test-polkadot-functional-0009-approval-voting-coalescing"
-#          - name: "Test validator disabling effects"
-#            test: "test-polkadot-functional-0010-validator-disabling"
-#          - name: "Test we are producing blocks at 6 seconds clip"
-#            test: "test-polkadot-functional-0011-async-backing-6-seconds-rate"
-#    steps:
-#      - name: Download artifact
-#        uses: actions/download-artifact@v4
-#        with:
-#          name: zombie-tester
-#          path: /tmp
-#      - name: Load image
-#        run: |
-#          docker load --input /tmp/zombie-tester.tar
-#          docker image ls -a
-#      - uses: actions/checkout@v4
-#      - name: Run test
-#        working-directory: ./zombienet/docker
-#        run: make ${{ matrix.options.test }} ZOMBIE_TESTER_IMAGE=zombie-tester:latest
-#      - name: Upload test logs as artifact
-#        if: always()
-#        uses: actions/upload-artifact@v4
-#        with:
-#          name: logs-${{ matrix.options.test }}
-#          path: /tmp/test_logs
diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile
@@ -1,34 +1,39 @@
-#RUST_IMAGE=rust:1.79-slim-bookworm
-
-RUST_VERSION ?= 1.79.0
+# Shared Environment Variables
 # bookworm (Debian 12)
 MINIDEB_IMAGE ?= bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2b5b6de63d6542bff10c16
-KAGOME_PACKAGE_VERSION ?= 20.07.24-0fa54d1-Release
-
+## DOCKER_REGISTRY_PATH empty for local usage
+DOCKER_REGISTRY_PATH ?=
+GOOGLE_APPLICATION_CREDENTIALS ?=
 PLATFORM ?= linux/amd64
 ARCHITECTURE ?= x86_64
 PACKAGE_ARCHITECTURE ?= amd64
 
-DOCKER_REGISTRY_PATH ?=
-ARTIFACTS_REPO ?= kagome-apt
-REGION ?= europe-north1
+# Debug, Release, RelWithDebInfo
+BUILD_TYPE ?= Release
 
+# Generated versions
 MINIDEB_HASH := $(subst bitnami/minideb@sha256:,,$(MINIDEB_IMAGE))
 MINIDEB_SHORT_HASH := $(shell echo $(MINIDEB_HASH) | cut -c1-7)
 MINIDEB_TAG = $(MINIDEB_SHORT_HASH)_rust-$(RUST_VERSION)
 
-DEPENDENCIES ?= libstdc++6, zlib1g, libgcc-s1, libc6, libtinfo6, libseccomp2, libatomic1
+# kagome_builder_deb Variables
+RUST_VERSION ?= 1.79.0
 
+# kagome_dev_docker_build Variables
+BUILD_DIR ?= build
+CACHE_DIR := $(shell pwd)/../../../../kagome/$(BUILD_DIR)/cache
+DEPENDENCIES ?= libstdc++6, zlib1g, libgcc-s1, libc6, libtinfo6, libseccomp2, libatomic1
 GITHUB_HUNTER_USERNAME ?=
 GITHUB_HUNTER_TOKEN ?=
 CTEST_OUTPUT_ON_FAILURE ?= 1
 
-# Debug, Release, RelWithDebInfo
-BUILD_TYPE ?= Release
+# kagome_runtime_cache Variables
+KAGOME_PACKAGE_VERSION ?=
+
+# upload_apt_package Variables
+ARTIFACTS_REPO ?= kagome-apt
+REGION ?= europe-north1
 
-BUILD_DIR ?= build
-CACHE_DIR := $(shell pwd)/../../../../kagome/$(BUILD_DIR)/cache
-WORKING_DIR := $(shell pwd)/../../../../kagome
 
 export DOCKER_BUILDKIT=1
 # BUILDKIT_PROGRESS - auto, plain, tty, rawjson
@@ -53,6 +58,7 @@ kagome_builder_deb_push:
 	docker push $(DOCKER_REGISTRY_PATH)kagome_builder_deb:latest
 
 kagome_dev_docker_build:
+	$(MAKE) get_versions
 	mkdir -p $(CACHE_DIR)/.cargo $(CACHE_DIR)/.hunter $(CACHE_DIR)/.cache/ccache $(CACHE_DIR)/.rustup ; \
 	CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \
 	SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \
@@ -109,7 +115,6 @@ upload_apt_package:
 	gcloud config set artifacts/repository $(ARTIFACTS_REPO); \
 	gcloud config set artifacts/location $(REGION); \
 	gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev_$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb
-
 
 runtime_cache:
 	CONTAINER_NAME=kagome_dev_runtime_cache_$$(openssl rand -hex 6); \
@@ -150,7 +155,6 @@ runtime_cache:
 	fi; \
 	docker stop $$CONTAINER_NAME
 
-
 upload_apt_package_runtime:
 	RUNTIME_VERSION=$$(python3 get_wasmedge_version.py); \
 	echo "Runtime version: $$RUNTIME_VERSION"; \
@@ -161,4 +165,4 @@ upload_apt_package_runtime:
 		echo "Package with version $$RUNTIME_VERSION already exists"; \
 		gcloud artifacts versions delete $$RUNTIME_VERSION --package=kagome-dev-runtime --quiet; \
 	fi; \
-	gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev-runtime_$${RUNTIME_VERSION}_$(PACKAGE_ARCHITECTURE).deb
+	gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev-runtime_$${RUNTIME_VERSION}_$(PACKAGE_ARCHITECTURE).deb
diff --git a/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile b/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile
@@ -6,9 +6,12 @@ ARG ARCHITECTURE=x86_64
 
 FROM ${BASE_IMAGE}
 
-SHELL ["/bin/bash", "-c"]
+ARG AUTHOR
+ENV AUTHOR=${AUTHOR}
+LABEL org.opencontainers.image.authors="${AUTHOR}"
+LABEL org.opencontainers.image.description="Kagome builder image"
 
-ENV KAGOME_IN_DOCKER=1
+SHELL ["/bin/bash", "-c"]
 
 RUN install_packages \
         build-essential \

diff --git a/zombienet/docker/Makefile b/zombienet/docker/Makefile
@@ -2,8 +2,8 @@
 # bookworm (Debian 12)
 MINIDEB_IMAGE=bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2b5b6de63d6542bff10c16
 ## DOCKER_REGISTRY_PATH empty for local usage
-GOOGLE_APPLICATION_CREDENTIALS ?=
 DOCKER_REGISTRY_PATH ?=
+GOOGLE_APPLICATION_CREDENTIALS ?=
 PLATFORM ?= linux/amd64
 ARCHITECTURE ?= amd64
 
@@ -182,6 +182,7 @@ zombie_tester:
 			-t $(DOCKER_REGISTRY_PATH)zombie_tester:$${POLKADOT_SDK_RELEASE}_$${ZOMBIENET_RELEASE} \
 			--secret id=google_creds,src=$(GOOGLE_APPLICATION_CREDENTIALS) \
 			-f zombie_tester.Dockerfile \
+			--build-arg BASE_IMAGE=$(MINIDEB_IMAGE) \
 			--build-arg PROJECT_ID=$$PROJECT_ID \
 			--build-arg POLKADOT_BINARY_PACKAGE_VERSION="$(POLKADOT_BINARY_PACKAGE_VERSION)" \
 			--build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \

diff --git a/zombienet/docker/zombie_tester.Dockerfile b/zombienet/docker/zombie_tester.Dockerfile
@@ -1,6 +1,6 @@
 ARG AUTHOR="[email protected] <Kirill Azovtsev>"
 
-ARG BASE_IMAGE=bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2b5b6de63d6542bff10c16
+ARG BASE_IMAGE
 ARG RUST_VERSION=1.79.0
 
 ARG PROJECT_ID
@@ -14,6 +14,8 @@ ARG ARCHITECTURE=x86_64
 
 FROM ${BASE_IMAGE} as zombie-tester
 
+ARG AUTHOR
+ENV AUTHOR=${AUTHOR}
 LABEL org.opencontainers.image.authors="${AUTHOR}"
 LABEL org.opencontainers.image.description="Zombienet image"