Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RUM-4079 chore: Migrate release automation to GitLab #1945

Merged
merged 4 commits into from
Jul 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ Carthage/Build
Carthage/Checkouts

xcuserdata/
instrumented-tests/DatadogSDKTesting.*
instrumented-tests/LICENSE

*.local.xcconfig

Expand All @@ -20,3 +18,7 @@ __pycache__
*.swp
.venv
.vscode
*.pytest_cache

# CI job artifacts
artifacts/
176 changes: 145 additions & 31 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,19 @@ stages:
- test
- ui-test
- smoke-test
- release-build
- release-publish

variables:
MAIN_BRANCH: "master"
DEVELOP_BRANCH: "develop"
# Prefilled variables for running a pipeline manually:
# Ref.: https://docs.gitlab.com/ee/ci/pipelines/index.html#prefill-variables-in-manual-pipelines
RELEASE_GIT_TAG:
description: "The Git tag for the release pipeline. If set, release pipeline will be triggered for the given tag."
RELEASE_DRY_RUN:
value: "1"
description: "Controls the dry run mode for the release pipeline. If set to '1', the pipeline will execute all steps but will not publish artifacts. If set to '0', the pipeline will run fully."

default:
tags:
Expand All @@ -18,35 +27,39 @@ default:
# β”‚ Utility jobs: β”‚
# β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜

# Trigger jobs on 'develop' and 'master' branches
.run:when-develop-or-master:
rules:
- if: '$CI_COMMIT_BRANCH == $DEVELOP_BRANCH || $CI_COMMIT_BRANCH == $MAIN_BRANCH'
when: always
# Utility jobs define rules for including or excluding dependent jobs from the pipeline.
#
# Ref.: https://docs.gitlab.com/ee/ci/jobs/job_rules.html
# > Rules are evaluated in order until the first match. When a match is found, the job is either included or excluded
# > from the pipeline, depending on the configuration.

# Trigger jobs on SDK code changes, comparing against 'develop' branch
.run:if-sdk-modified:
.test-pipeline-job:
rules:
- changes:
- if: '$CI_COMMIT_BRANCH == $DEVELOP_BRANCH || $CI_COMMIT_BRANCH == $MAIN_BRANCH' # always on main branches
- if: '$CI_COMMIT_BRANCH' # when on other branch with following changes compared to develop
changes:
paths:
- "Datadog*/**/*"
- "IntegrationTests/**/*"
- "TestUtilities/**/*"
- "*" # match any file in the root directory
compare_to: 'develop' # cannot use variable due to: https://gitlab.com/gitlab-org/gitlab/-/issues/369916
compare_to: 'develop' # cannot use $DEVELOP_BRANCH var due to: https://gitlab.com/gitlab-org/gitlab/-/issues/369916

# Trigger jobs on changes in `tools/*`, comparing against 'develop' branch
.run:if-tools-modified:
rules:
- changes:
paths:
- "tools/**/*"
- "Makefile"
- ".gitlab-ci.yml"
compare_to: 'develop'
.release-pipeline-job:
rules:
- if: '$CI_COMMIT_TAG || $RELEASE_GIT_TAG'

.release-pipeline-delayed-job:
rules:
- if: '$CI_COMMIT_TAG || $RELEASE_GIT_TAG'
when: delayed
start_in: 20 minutes

ENV check:
stage: pre
rules:
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
script:
- make env-check

Expand All @@ -57,8 +70,7 @@ ENV check:
Lint:
stage: lint
rules:
- !reference [.run:when-develop-or-master, rules]
- !reference [.run:if-sdk-modified, rules]
- !reference [.test-pipeline-job, rules]
script:
- make clean repo-setup ENV=ci
- make lint license-check
Expand All @@ -67,8 +79,8 @@ Lint:
Unit Tests (iOS):
stage: test
rules:
- !reference [.run:when-develop-or-master, rules]
- !reference [.run:if-sdk-modified, rules]
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
variables:
XCODE: "15.3.0"
OS: "17.4"
Expand All @@ -82,8 +94,8 @@ Unit Tests (iOS):
Unit Tests (tvOS):
stage: test
rules:
- !reference [.run:when-develop-or-master, rules]
- !reference [.run:if-sdk-modified, rules]
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
variables:
XCODE: "15.3.0"
OS: "17.4"
Expand All @@ -97,8 +109,8 @@ Unit Tests (tvOS):
UI Tests:
stage: ui-test
rules:
- !reference [.run:when-develop-or-master, rules]
- !reference [.run:if-sdk-modified, rules]
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
variables:
XCODE: "15.3.0"
OS: "17.4"
Expand All @@ -118,15 +130,23 @@ UI Tests:

Tools Tests:
stage: test
rules:
- !reference [.run:when-develop-or-master, rules]
- !reference [.run:if-tools-modified, rules]
rules:
- if: '$CI_COMMIT_BRANCH' # when on branch with following changes compared to develop
changes:
paths:
- "tools/**/*"
- "Makefile"
- ".gitlab-ci.yml"
compare_to: 'develop'
script:
- make clean repo-setup ENV=ci
- make tools-test

Smoke Tests (iOS):
stage: smoke-test
rules:
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
tags:
- macos:ventura
- specific:true
Expand All @@ -136,13 +156,16 @@ Smoke Tests (iOS):
PLATFORM: "iOS Simulator"
DEVICE: "iPhone 15 Pro"
script:
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "$OS" # temporary, waiting for AMI
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "$OS" --ssh # temporary, waiting for AMI
- make clean repo-setup ENV=ci
- make spm-build-ios
- make smoke-test-ios-all OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"

Smoke Tests (tvOS):
stage: smoke-test
rules:
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
tags:
- macos:ventura
- specific:true
Expand All @@ -152,13 +175,16 @@ Smoke Tests (tvOS):
PLATFORM: "tvOS Simulator"
DEVICE: "Apple TV"
script:
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "$OS" # temporary, waiting for AMI
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "$OS" --ssh # temporary, waiting for AMI
- make clean repo-setup ENV=ci
- make spm-build-tvos
- make smoke-test-tvos-all OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"

Smoke Tests (visionOS):
stage: smoke-test
rules:
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
tags:
- macos:ventura
- specific:true
Expand All @@ -172,6 +198,9 @@ Smoke Tests (visionOS):

Smoke Tests (macOS):
stage: smoke-test
rules:
- !reference [.test-pipeline-job, rules]
- !reference [.release-pipeline-job, rules]
tags:
- macos:ventura
- specific:true
Expand All @@ -181,3 +210,88 @@ Smoke Tests (macOS):
- ./tools/runner-setup.sh --xcode "$XCODE" # temporary, waiting for AMI
- make clean repo-setup ENV=ci
- make spm-build-macos

# β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
# β”‚ SDK release: β”‚
# β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜

.release-before-script: &export_MAKE_release_params
- export GIT_TAG=${RELEASE_GIT_TAG:-$CI_COMMIT_TAG} # CI_COMMIT_TAG if set, otherwise default to RELEASE_GIT_TAG
- if [ -z "$GIT_TAG" ]; then echo "GIT_TAG is not set"; exit 1; fi # sanity check
- export ARTIFACTS_PATH="artifacts/$GIT_TAG"
- export DRY_RUN=${CI_COMMIT_TAG:+0} # 0 if CI_COMMIT_TAG is set
- export DRY_RUN=${DRY_RUN:-$RELEASE_DRY_RUN} # otherwise default to RELEASE_DRY_RUN

Build Artifacts:
stage: release-build
rules:
- !reference [.release-pipeline-job, rules]
variables:
XCODE: "15.3.0"
artifacts:
paths:
- artifacts
expire_in: 4 weeks
before_script:
- *export_MAKE_release_params
script:
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "17.4" --ssh # temporary, waiting for AMI
- make env-check
- make clean
- make release-build release-validate

Publish GH Asset:
stage: release-publish
rules:
- !reference [.release-pipeline-job, rules]
variables:
XCODE: "15.3.0"
before_script:
- *export_MAKE_release_params
script:
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "17.4" # temporary, waiting for AMI
- make env-check
- make clean
- make release-publish-github

Publish CP podspecs (internal):
stage: release-publish
rules:
- !reference [.release-pipeline-job, rules]
variables:
XCODE: "15.3.0"
before_script:
- *export_MAKE_release_params
script:
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "17.4" # temporary, waiting for AMI
- make env-check
- make clean
- make release-publish-internal-podspecs

Publish CP podspecs (dependent):
stage: release-publish
rules:
- !reference [.release-pipeline-delayed-job, rules]
variables:
XCODE: "15.3.0"
before_script:
- *export_MAKE_release_params
script:
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "17.4" # temporary, waiting for AMI
- make env-check
- make clean
- make release-publish-dependent-podspecs

Publish CP podspecs (legacy):
stage: release-publish
rules:
- !reference [.release-pipeline-delayed-job, rules]
variables:
XCODE: "15.3.0"
before_script:
- *export_MAKE_release_params
script:
- ./tools/runner-setup.sh --xcode "$XCODE" --iOS --tvOS --os "17.4" # temporary, waiting for AMI
- make env-check
- make clean
- make release-publish-legacy-podspecs
67 changes: 67 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ all: env-check repo-setup templates
smoke-test smoke-test-ios smoke-test-ios-all smoke-test-tvos smoke-test-tvos-all \
spm-build spm-build-ios spm-build-tvos spm-build-visionos spm-build-macos \
models-generate rum-models-generate sr-models-generate models-verify rum-models-verify sr-models-verify \
release-build release-validate release-publish-github \
release-publish-podspec release-publish-internal-podspecs release-publish-dependent-podspecs release-publish-legacy-podspecs \
set-ci-secret

REPO_ROOT := $(PWD)
include tools/utils/common.mk
Expand Down Expand Up @@ -335,6 +338,70 @@ e2e-monitors-generate:
@./tools/nightly-e2e-tests/nightly_e2e.py generate-tf --tests-dir ../../Datadog/E2ETests
@echo "⚠️ Remember to delete all iOS monitors manually from Mobile-Integration org before running 'terraform apply'."

# Builds release artifacts for given tag
release-build:
@$(call require_param,GIT_TAG)
@$(call require_param,ARTIFACTS_PATH)
@$(ECHO_TITLE) "make release-build GIT_TAG='$(GIT_TAG)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'"
./tools/release/build.sh --tag "$(GIT_TAG)" --artifacts-path "$(ARTIFACTS_PATH)"

# Validate release artifacts for given tag
release-validate:
@$(call require_param,GIT_TAG)
@$(call require_param,ARTIFACTS_PATH)
@$(ECHO_TITLE) "make release-validate GIT_TAG='$(GIT_TAG)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'"
./tools/release/validate-version.sh --artifacts-path "$(ARTIFACTS_PATH)" --tag "$(GIT_TAG)"
./tools/release/validate-xcframeworks.sh --artifacts-path "$(ARTIFACTS_PATH)"

# Publish GitHub asset to GH release
release-publish-github:
@$(call require_param,GIT_TAG)
@$(call require_param,ARTIFACTS_PATH)
@:$(eval DRY_RUN ?= 1)
@:$(eval OVERWRITE_EXISTING ?= 0)
@$(ECHO_TITLE) "make release-publish-github GIT_TAG='$(GIT_TAG)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)' DRY_RUN='$(DRY_RUN)' OVERWRITE_EXISTING='$(OVERWRITE_EXISTING)'"
DRY_RUN=$(DRY_RUN) OVERWRITE_EXISTING=$(OVERWRITE_EXISTING) ./tools/release/publish-github.sh \
--artifacts-path "$(ARTIFACTS_PATH)" \
--tag "$(GIT_TAG)"

# Publish Cocoapods podspec to trunk
release-publish-podspec:
@$(call require_param,PODSPEC_NAME)
@$(call require_param,ARTIFACTS_PATH)
@:$(eval DRY_RUN ?= 1)
@$(ECHO_TITLE) "make release-publish-podspec PODSPEC_NAME='$(PODSPEC_NAME)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)' DRY_RUN='$(DRY_RUN)'"
DRY_RUN=$(DRY_RUN) ./tools/release/publish-podspec.sh \
--artifacts-path "$(ARTIFACTS_PATH)" \
--podspec-name "$(PODSPEC_NAME)"

# Publish DatadogInternal podspec
release-publish-internal-podspecs:
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogInternal.podspec"

# Publish podspecs that depend on DatadogInternal
release-publish-dependent-podspecs:
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogCore.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogLogs.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogTrace.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogRUM.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogSessionReplay.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogCrashReporting.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogWebViewTracking.podspec"

# Publish legacy podspecs
release-publish-legacy-podspecs:
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogObjc.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogAlamofireExtension.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogSDK.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogSDKObjc.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogSDKCrashReporting.podspec"
@$(MAKE) release-publish-podspec PODSPEC_NAME="DatadogSDKAlamofireExtension.podspec"

# Set ot update CI secrets
set-ci-secret:
@$(ECHO_TITLE) "make set-ci-secret"
@./tools/secrets/set-secret.sh

bump:
@read -p "Enter version number: " version; \
echo "// GENERATED FILE: Do not edit directly\n\ninternal let __sdkVersion = \"$$version\"" > DatadogCore/Sources/Versioning.swift; \
Expand Down
Loading