Skip to content

Commit

Permalink
build apt package pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
qdrvm-ci committed Jun 22, 2024
1 parent a8226c0 commit 947063f
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 17 deletions.
18 changes: 10 additions & 8 deletions .github/workflows/zombie-builder.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,7 @@ env:
# Change all container: values to this variable once this feature is available
# https://github.jparrowsec.cnmunity/t/how-to-set-and-access-a-workflow-variable/17335/6
CACHE_VERSION: v001
CACHE_PATHS: |
./cargo/registry
./cargo/git
CACHE_PATHS: ./zombienet/docker/cargo

jobs:
# get_polkadot_binary_image:
Expand Down Expand Up @@ -156,12 +154,16 @@ jobs:
run: make polkadot_binary \
DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
SCCACHE_GCS_BUCKET=${{ secrets.SCCACHE_GCS_BUCKET }} \

- name: "Push Polkadot APT Package"
working-directory: ./zombienet/docker
run: make upload_apt_package

- name: Archive build outputs
uses: actions/upload-artifact@v4
with:
name: build-output
path: ./zombienet/docker/polkadot_binary/
# - name: Archive build outputs
# uses: actions/upload-artifact@v4
# with:
# name: build-output
# path: ./zombienet/docker/polkadot_binary/


# building_binary:
Expand Down
8 changes: 7 additions & 1 deletion zombienet/docker/.gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
*-versions.txt
commit_hash.txt

cumulus
polkadot
polkadot

cargo/
pkg/
polkadot_binary/
49 changes: 41 additions & 8 deletions zombienet/docker/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ MINIDEB_IMAGE=bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2

DOCKER_REGISTRY_PATH ?=
PLATFORM ?= linux/amd64
ARCH ?= amd64
SCCACHE_VERSION ?= 0.7.4
ARTIFACTS_REPO ?= kagome-apt
REGION ?= europe-north1

#USE_GCP_SCCACHE ?= true
SCCACHE_GCS_BUCKET ?=
Expand Down Expand Up @@ -52,6 +55,8 @@ endef
get_versions:
python3 version.py https://github.com/paritytech/polkadot-sdk && \
python3 version.py https://github.com/paritytech/zombienet
echo "full_commit_hash: `git rev-parse HEAD`" > commit_hash.txt
echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" >> commit_hash.txt

polkadot_builder:
if [ -f polkadot-sdk-versions.txt ]; then \
Expand Down Expand Up @@ -80,8 +85,8 @@ polkadot_binary:
if [ -f polkadot-sdk-versions.txt ]; then \
df -h ; \
CONTAINER_NAME=polkadot_build_$$(openssl rand -hex 6); \
POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
mkdir -p ./polkadot_binary ./cargo/registry ./cargo/git; \
POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
mkdir -p ./polkadot_binary ./cargo/registry ./cargo/git ./pkg; \
docker run -d --name $$CONTAINER_NAME \
--platform $(PLATFORM) \
--entrypoint "/bin/bash" \
Expand All @@ -91,27 +96,55 @@ polkadot_binary:
-e SCCACHE_GCS_KEY_PREFIX=polkadot_builder \
-e SCCACHE_GCS_RW_MODE=READ_WRITE \
-e SCCACHE_LOG=info \
-e SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) \
-e POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC \
-e ARCH=$(ARCH) \
-v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \
-v ./polkadot_binary:/tmp/polkadot_binary \
-v ./cargo/registry:/usr/local/cargo/registry/ \
-v ./cargo/git:/usr/local/cargo/git/ \
-v ./pkg:/home/nonroot/polkadot-sdk/pkg \
-v ./build_apt_package.sh:/home/nonroot/polkadot-sdk/build_apt_package.sh \
$(DOCKER_REGISTRY_PATH)polkadot_builder:$$POLKADOT_SDK_RELEASE \
-c "tail -f /dev/null"; \
DOCKER_EXEC_RESULT=0; \
docker exec -t $$CONTAINER_NAME /bin/bash -c \
"cargo update $(CARGO_PACKETS) && \
cargo build --profile testnet $(CARGO_PACKETS); \
la -la ./target/testnet/ ; \
cp ./target/testnet/malus \
cargo build --profile testnet $(CARGO_PACKETS) && \
ls -la ./target/testnet/ && \
mv -f ./target/testnet/malus \
./target/testnet/undying-collator \
./target/testnet/adder-collator \
/tmp/polkadot_binary/ ; \
ls -la /tmp/polkadot_binary /usr/local/cargo/registry /usr/local/cargo/git" ; \
ls -la ./polkadot_binary ./cargo/registry ./cargo/git ; df -h ; \
/tmp/polkadot_binary/ && \
ls -la /tmp/polkadot_binary /usr/local/cargo/registry /usr/local/cargo/git && \
./build_apt_package.sh \
$$POLKADOT_RELEASE_GLOBAL_NUMERIC-$$SHORT_COMMIT_HASH \
$ARCH \
polkadot-binary \
/tmp/polkadot_binary \
'Polkadot binaries: malus undying-collator adder-collator' \
'libstdc++6, zlib1g, libgcc-s1, libc6'" \
|| DOCKER_EXEC_RESULT=$$? ; \
if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \
echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \
docker stop $$CONTAINER_NAME; \
exit $$DOCKER_EXEC_RESULT; \
fi; \
ls -la ./polkadot_binary ./cargo/registry ./cargo/git ; \
df -h ; \
docker stop $$CONTAINER_NAME; \
else \
echo "One or more files are missing."; \
fi

upload_apt_package:
SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \
POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \
ARCH=$(ARCH); \
gcloud config set artifacts/repository $(ARTIFACTS_REPO); \
gcloud config set artifacts/location $(REGION); \
gcloud artifacts versions delete $$POLKADOT_RELEASE_GLOBAL_NUMERIC-$$SHORT_COMMIT_HASH --package=polkadot-binary --quiet || true ; \
gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/polkadot-binary_$$POLKADOT_RELEASE_GLOBAL_NUMERIC-$${SHORT_COMMIT_HASH}_$$ARCH.deb


polkadot_binary_temp:
Expand Down
93 changes: 93 additions & 0 deletions zombienet/docker/build_apt_package.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/bin/bash

set -euo pipefail

# Function to display usage
usage() {
echo "Usage: $0 <version> <architecture> <package_name> <artifacts_dir> <description> <dependencies>"
echo "Example: $0 1.0 amd64 mypackage /path/to/artifacts 'libc6, libssl1.1'"
exit 1
}

# Function to log messages
log() {
local MESSAGE=$1
echo "[INFO] $MESSAGE"
}

# Check for the correct number of arguments
if [ "$#" -ne 6 ]; then
usage
fi

# Assign variables from arguments
VERSION=$1
ARCHITECTURE=$2
PACKAGE_NAME=$3
ARTIFACTS_DIR=$4
DESCRIPTION=$5
DEPENDENCIES=$6

# Define maintainer and homepage variables
MAINTAINER="Zak Fein <[email protected]>"
HOMEPAGE="https://qdrvm.io"
DIR_NAME=${PACKAGE_NAME}_${VERSION}_${ARCHITECTURE}

# Validate parameters
log "Validating parameters..."

if [ -z "$VERSION" ]; then
echo "Error: Version is required." >&2
exit 1
fi

if [ -z "$ARCHITECTURE" ]; then
echo "Error: Architecture is required." >&2
exit 1
fi

if [ -z "$PACKAGE_NAME" ]; then
echo "Error: Package name is required." >&2
exit 1
fi

if [ -z "$ARTIFACTS_DIR" ] || [ ! -d "$ARTIFACTS_DIR" ]; then
echo "Error: A valid artifacts directory is required." >&2
exit 1
fi

if [ -z "$DESCRIPTION" ]; then
echo "Error: Description is required." >&2
exit 1
fi

if [ -z "$DEPENDENCIES" ]; then
echo "Error: Dependencies are required." >&2
exit 1
fi

log "Creating package directory structure..."
mkdir -p ./pkg/${DIR_NAME}/DEBIAN
mkdir -p ./pkg/${DIR_NAME}/usr/local/bin

log "Copying artifacts..."
mv -f ${ARTIFACTS_DIR}/* ./pkg/${DIR_NAME}/usr/local/bin/

log "Creating control file..."
cat <<EOF > ./pkg/${DIR_NAME}/DEBIAN/control
Package: ${PACKAGE_NAME}
Version: ${VERSION}
Maintainer: ${MAINTAINER}
Depends: ${DEPENDENCIES}
Architecture: ${ARCHITECTURE}
Homepage: ${HOMEPAGE}
Description: ${DESCRIPTION}
EOF

log "Building the package..."
dpkg --build ./pkg/${DIR_NAME}

log "Displaying package info..."
dpkg-deb --info ./pkg/${DIR_NAME}.deb

log "Package created successfully."

0 comments on commit 947063f

Please sign in to comment.