Merge pull request #1921 from DataDog/ncreated/RUM-4079/migrate-smoke…

…-and-tool-tests RUM-4079 chore: Migrate tools and smoke tests to GitLab
DataDog · Jun 25, 2024 · 85dcd0f · 85dcd0f
2 parents 1098905 + ef7ead5
commit 85dcd0f
Show file tree

Hide file tree

Showing 110 changed files with 736 additions and 979 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -13,5 +13,3 @@ A brief description of implementation details of this PR.
 
 ### Custom CI job configuration (optional)
 - [ ] Run unit tests for Session Replay
-- [ ] Run smoke tests
-- [ ] Run tests for `tools/`
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -3,6 +3,7 @@ stages:
   - lint
   - test
   - ui-test
+  - smoke-test
 
 variables:
   MAIN_BRANCH: "master"
@@ -34,6 +35,16 @@ default:
           - "*" # match any file in the root directory
         compare_to: 'develop' # cannot use variable due to: https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
+# Trigger jobs on changes in `tools/*`, comparing against 'develop' branch
+.run:if-tools-modified:
+  rules:
+    - changes:
+        paths:
+          - "tools/**/*"
+          - "Makefile"
+          - ".gitlab-ci.yml"
+        compare_to: 'develop'
+
 ENV check:
   stage: pre
   script:
@@ -59,11 +70,12 @@ Unit Tests (iOS):
     - !reference [.run:when-develop-or-master, rules]
     - !reference [.run:if-sdk-modified, rules]
   variables:
+    XCODE: "15.3.0"
     OS: "17.4"
     PLATFORM: "iOS Simulator"
     DEVICE: "iPhone 15 Pro"
   script:
-    - ./tools/runner-setup.sh --ios --os "$OS" # temporary, waiting for AMI
+    - ./tools/runner-setup.sh --xcode "$XCODE" --iOS --os "$OS" # temporary, waiting for AMI
     - make clean repo-setup ENV=ci
     - make test-ios-all OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
 
@@ -73,11 +85,12 @@ Unit Tests (tvOS):
     - !reference [.run:when-develop-or-master, rules]
     - !reference [.run:if-sdk-modified, rules]
   variables:
+    XCODE: "15.3.0"
     OS: "17.4"
     PLATFORM: "tvOS Simulator"
     DEVICE: "Apple TV"
   script:
-    - ./tools/runner-setup.sh --tvos --os "$OS" # temporary, waiting for AMI
+    - ./tools/runner-setup.sh --xcode "$XCODE" --tvOS --os "$OS" # temporary, waiting for AMI
     - make clean repo-setup ENV=ci
     - make test-tvos-all OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
 
@@ -87,6 +100,7 @@ UI Tests:
     - !reference [.run:when-develop-or-master, rules]
     - !reference [.run:if-sdk-modified, rules]
   variables:
+    XCODE: "15.3.0"
     OS: "17.4"
     PLATFORM: "iOS Simulator"
     DEVICE: "iPhone 15 Pro"
@@ -98,6 +112,72 @@ UI Tests:
           - CrashReporting
           - NetworkInstrumentation
   script:
-    - ./tools/runner-setup.sh --ios --os "$OS" # temporary, waiting for AMI
+    - ./tools/runner-setup.sh --xcode "$XCODE" --iOS --os "$OS" # temporary, waiting for AMI
     - make clean repo-setup ENV=ci
     - make ui-test TEST_PLAN="$TEST_PLAN" OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
+
+Tools Tests:
+  stage: test
+  rules: 
+    - !reference [.run:when-develop-or-master, rules]
+    - !reference [.run:if-tools-modified, rules]
+  script:
+    - make clean repo-setup ENV=ci
+    - make tools-test
+
+Smoke Tests (iOS):
+  stage: smoke-test
+  tags:
+    - macos:ventura
+    - specific:true
+  variables:
+    XCODE: "15.2.0"
+    OS: "17.2"
+    PLATFORM: "iOS Simulator"
+    DEVICE: "iPhone 15 Pro"
+  script:
+    - ./tools/runner-setup.sh --xcode "$XCODE" --iOS --os "$OS" # temporary, waiting for AMI
+    - make clean repo-setup ENV=ci
+    - make spm-build-ios
+    - make smoke-test-ios-all OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
+
+Smoke Tests (tvOS):
+  stage: smoke-test
+  tags:
+    - macos:ventura
+    - specific:true
+  variables:
+    XCODE: "15.2.0"
+    OS: "17.2"
+    PLATFORM: "tvOS Simulator"
+    DEVICE: "Apple TV"
+  script:
+    - ./tools/runner-setup.sh --xcode "$XCODE" --tvOS --os "$OS" # temporary, waiting for AMI
+    - make clean repo-setup ENV=ci
+    - make spm-build-tvos
+    - make smoke-test-tvos-all OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
+
+Smoke Tests (visionOS):
+  stage: smoke-test
+  tags:
+    - macos:ventura
+    - specific:true
+  variables:
+    XCODE: "15.2.0"
+    OS: "1.0"
+  script:
+    - ./tools/runner-setup.sh --xcode "$XCODE" --visionOS --os "$OS" # temporary, waiting for AMI
+    - make clean repo-setup ENV=ci
+    - make spm-build-visionos
+
+Smoke Tests (macOS):
+  stage: smoke-test
+  tags:
+    - macos:ventura
+    - specific:true
+  variables:
+    XCODE: "15.2.0"
+  script:
+    - ./tools/runner-setup.sh --xcode "$XCODE" # temporary, waiting for AMI
+    - make clean repo-setup ENV=ci
+    - make spm-build-macos
diff --git a/Datadog.xcworkspace/contents.xcworkspacedata b/Datadog.xcworkspace/contents.xcworkspacedata
diff --git a/IntegrationTests/IntegrationTests.xcodeproj/project.pbxproj b/IntegrationTests/IntegrationTests.xcodeproj/project.pbxproj
@@ -259,7 +259,7 @@
 		D2774EE2299E2E90004EC36A /* LogMatcher.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LogMatcher.swift; sourceTree = "<group>"; };
 		D2774EE3299E2E90004EC36A /* JSONDataMatcher.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = JSONDataMatcher.swift; sourceTree = "<group>"; };
 		D2791EF827170A760046E07A /* RUMSwiftUIScenarioTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMSwiftUIScenarioTests.swift; sourceTree = "<group>"; };
-		D29495F429967780003518CD /* http-server-mock */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "http-server-mock"; path = "../instrumented-tests/http-server-mock"; sourceTree = "<group>"; };
+		D29495F429967780003518CD /* http-server-mock */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "http-server-mock"; path = "../tools/http-server-mock"; sourceTree = "<group>"; };
 		D2A401D029925A2D00B230A3 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		D2AEFF1A29925BEC00A28997 /* CrashReporting.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = CrashReporting.xctestplan; sourceTree = "<group>"; };
 		D2AEFF1B29925BEC00A28997 /* Default.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = Default.xctestplan; sourceTree = "<group>"; };

diff --git a/...tionTests/IntegrationTests.xcodeproj/xcshareddata/xcschemes/IntegrationScenarios.xcscheme b/...tionTests/IntegrationTests.xcodeproj/xcshareddata/xcschemes/IntegrationScenarios.xcscheme
@@ -66,7 +66,7 @@
             ActionType = "Xcode.IDEStandardExecutionActionsCore.ExecutionActionType.ShellScriptAction">
             <ActionContent
                title = "Run Script"
-               scriptText = "REPOSITORY_ROOT=&quot;${SRCROOT}/..&quot;&#10;cd &quot;${REPOSITORY_ROOT}&quot; &amp;&amp; ./instrumented-tests/http-server-mock/python/start_mock_server.py&amp;&#10;">
+               scriptText = "REPOSITORY_ROOT=&quot;${SRCROOT}/..&quot;&#10;cd &quot;${REPOSITORY_ROOT}&quot; &amp;&amp; ./tools/http-server-mock/python/start_mock_server.py&amp;&#10;">
                <EnvironmentBuildable>
                   <BuildableReference
                      BuildableIdentifier = "primary"

diff --git a/IntegrationTests/IntegrationTests.xcodeproj/xcshareddata/xcschemes/Runner iOS.xcscheme b/IntegrationTests/IntegrationTests.xcodeproj/xcshareddata/xcschemes/Runner iOS.xcscheme
@@ -32,7 +32,7 @@
             ActionType = "Xcode.IDEStandardExecutionActionsCore.ExecutionActionType.ShellScriptAction">
             <ActionContent
                title = "Run Script"
-               scriptText = "REPOSITORY_ROOT=&quot;${SRCROOT}/..&quot;&#10;cd &quot;${REPOSITORY_ROOT}&quot; &amp;&amp; ./instrumented-tests/http-server-mock/python/start_mock_server.py&amp;&#10;">
+               scriptText = "REPOSITORY_ROOT=&quot;${SRCROOT}/..&quot;&#10;cd &quot;${REPOSITORY_ROOT}&quot; &amp;&amp; ./tools/http-server-mock/python/start_mock_server.py&amp;&#10;">
                <EnvironmentBuildable>
                   <BuildableReference
                      BuildableIdentifier = "primary"

diff --git a/Makefile b/Makefile
@@ -3,8 +3,13 @@ all: env-check repo-setup templates
 		lint license-check \
 		test test-ios test-ios-all test-tvos test-tvos-all \
 		ui-test ui-test-all ui-test-podinstall \
+		tools-test \
+		smoke-test smoke-test-ios smoke-test-ios-all smoke-test-tvos smoke-test-tvos-all \
+		spm-build spm-build-ios spm-build-tvos spm-build-visionos spm-build-macos \
 		models-generate rum-models-generate sr-models-generate models-verify rum-models-verify sr-models-verify \
 
+REPO_ROOT := $(PWD)
+include tools/utils/common.mk
 
 define DD_SDK_TESTING_XCCONFIG_CI
 DD_TEST_RUNNER=1\n
@@ -67,7 +72,7 @@ DEFAULT_ENV := dev
 
 env-check:
 	@$(ECHO_TITLE) "make env-check"
-	./tools/env_check.sh
+	./tools/env-check.sh
 
 repo-setup:
 	@:$(eval ENV ?= $(DEFAULT_ENV))
@@ -96,25 +101,12 @@ DEFAULT_TVOS_OS := latest
 DEFAULT_TVOS_PLATFORM := tvOS Simulator
 DEFAULT_TVOS_DEVICE := Apple TV
 
-build-spm:
-	@$(call require_param,PLATFORM)
-	@:$(eval PLATFORM ?= iOS)
-	./tools/spm.sh --platform $(PLATFORM)
-
-build-spm-all:
-	./tools/spm.sh --platform iOS
-	./tools/spm.sh --platform tvOS
-	./tools/spm.sh --platform visionOS
-
 # Run unit tests for specified SCHEME
 test:
 	@$(call require_param,SCHEME)
 	@$(call require_param,OS)
 	@$(call require_param,PLATFORM)
 	@$(call require_param,DEVICE)
-	@:$(eval OS ?= $(DEFAULT_IOS_OS))
-	@:$(eval PLATFORM ?= $(DEFAULT_IOS_PLATFORM))
-	@:$(eval DEVICE ?= $(DEFAULT_IOS_DEVICE))
 	@$(ECHO_TITLE) "make test SCHEME='$(SCHEME)' OS='$(OS)' PLATFORM='$(PLATFORM)' DEVICE='$(DEVICE)'"
 	./tools/test.sh --scheme "$(SCHEME)" --os "$(OS)" --platform "$(PLATFORM)" --device "$(DEVICE)"
 
@@ -175,6 +167,79 @@ ui-test-podinstall:
 	@$(ECHO_TITLE) "make ui-test-podinstall"
 	cd IntegrationTests/ && bundle exec pod install
 
+# Run tests for repo tools
+tools-test:
+	@$(ECHO_TITLE) "make tools-test"
+	./tools/tools-test.sh
+
+# Run smoke tests
+smoke-test:
+	@$(call require_param,TEST_DIRECTORY)
+	@$(call require_param,OS)
+	@$(call require_param,PLATFORM)
+	@$(call require_param,DEVICE)
+	@$(ECHO_TITLE) "make smoke-test TEST_DIRECTORY='$(TEST_DIRECTORY)' OS='$(OS)' PLATFORM='$(PLATFORM)' DEVICE='$(DEVICE)'"
+	./tools/smoke-test.sh --test-directory "$(TEST_DIRECTORY)" --os "$(OS)" --platform "$(PLATFORM)" --device "$(DEVICE)"
+
+# Run smoke tests for specified TEST_DIRECTORY using iOS Simulator
+smoke-test-ios:
+	@$(call require_param,TEST_DIRECTORY)
+	@:$(eval OS ?= $(DEFAULT_IOS_OS))
+	@:$(eval PLATFORM ?= $(DEFAULT_IOS_PLATFORM))
+	@:$(eval DEVICE ?= $(DEFAULT_IOS_DEVICE))
+	@$(MAKE) smoke-test TEST_DIRECTORY="$(TEST_DIRECTORY)" OS="$(OS)" PLATFORM="$(PLATFORM)" DEVICE="$(DEVICE)"
+
+# Run all smoke tests using iOS Simulator
+smoke-test-ios-all:
+	@$(MAKE) smoke-test-ios TEST_DIRECTORY="SmokeTests/spm"
+	@$(MAKE) smoke-test-ios TEST_DIRECTORY="SmokeTests/carthage"
+	@$(MAKE) smoke-test-ios TEST_DIRECTORY="SmokeTests/cocoapods"
+	@$(MAKE) smoke-test-ios TEST_DIRECTORY="SmokeTests/xcframeworks"
+
+# Run smoke tests for specified TEST_DIRECTORY using tvOS Simulator
+smoke-test-tvos:
+	@$(call require_param,TEST_DIRECTORY)
+	@:$(eval OS ?= $(DEFAULT_TVOS_OS))
+	@:$(eval PLATFORM ?= $(DEFAULT_TVOS_PLATFORM))
+	@:$(eval DEVICE ?= $(DEFAULT_TVOS_DEVICE))
+	@$(MAKE) smoke-test TEST_DIRECTORY="$(TEST_DIRECTORY)" OS="$(OS)" PLATFORM="$(PLATFORM)" DEVICE="$(DEVICE)"
+
+# Run all smoke tests using tvOS Simulator
+smoke-test-tvos-all:
+	@$(MAKE) smoke-test-tvos TEST_DIRECTORY="SmokeTests/spm"
+	@$(MAKE) smoke-test-tvos TEST_DIRECTORY="SmokeTests/carthage"
+	@$(MAKE) smoke-test-tvos TEST_DIRECTORY="SmokeTests/cocoapods"
+	@$(MAKE) smoke-test-tvos TEST_DIRECTORY="SmokeTests/xcframeworks"
+
+# Builds SPM package SCHEME for specified DESTINATION
+spm-build:
+	@$(call require_param,SCHEME)
+	@$(call require_param,DESTINATION)
+	@$(ECHO_TITLE) "make spm-build SCHEME='$(SCHEME)' DESTINATION='$(DESTINATION)'"
+	./tools/spm-build.sh --scheme "$(SCHEME)" --destination "$(DESTINATION)"
+
+# Builds SPM package for iOS
+spm-build-ios:
+	@$(MAKE) spm-build SCHEME="Datadog-Package" DESTINATION="generic/platform=ios"
+
+# Builds SPM package for tvOS
+spm-build-tvos:
+	@$(MAKE) spm-build SCHEME="Datadog-Package" DESTINATION="generic/platform=tvOS"
+
+# Builds SPM package for visionOS
+spm-build-visionos:
+	@$(MAKE) spm-build SCHEME="Datadog-Package" DESTINATION="generic/platform=visionOS"
+
+# Builds SPM package for macOS (and Mac Catalyst)
+spm-build-macos:
+	# Whole package for Mac Catalyst:
+	@$(MAKE) spm-build SCHEME="Datadog-Package" DESTINATION="platform=macOS,variant=Mac Catalyst"
+	# Only compatible schemes for macOS:
+	@$(MAKE) spm-build DESTINATION="platform=macOS" SCHEME="DatadogCore"
+	@$(MAKE) spm-build DESTINATION="platform=macOS" SCHEME="DatadogLogs"
+	@$(MAKE) spm-build DESTINATION="platform=macOS" SCHEME="DatadogTrace"
+	@$(MAKE) spm-build DESTINATION="platform=macOS" SCHEME="DatadogCrashReporting"
+
 xcodeproj-session-replay:
 		@echo "⚙️  Generating 'DatadogSessionReplay.xcodeproj'..."
 		@cd DatadogSessionReplay/ && swift package generate-xcodeproj
@@ -190,22 +255,6 @@ templates:
 	@$(ECHO_TITLE) "make templates"
 	./tools/xcode-templates/install-xcode-templates.sh
 
-# Tests if current branch ships a valid SPM package.
-test-spm:
-		@cd dependency-manager-tests/spm && $(MAKE)
-
-# Tests if current branch ships a valid Carthage project.
-test-carthage:
-		@cd dependency-manager-tests/carthage && $(MAKE)
-
-# Tests if current branch ships a valid Cocoapods project.
-test-cocoapods:
-		@cd dependency-manager-tests/cocoapods && $(MAKE)
-
-# Tests if current branch ships valid a XCFrameworks project.
-test-xcframeworks:
-		@cd dependency-manager-tests/xcframeworks && $(MAKE)
-
 # Generate data models from https://github.com/DataDog/rum-events-format
 models-generate:
 	@$(call require_param,PRODUCT) # 'rum' or 'sr'
@@ -296,17 +345,3 @@ bump:
 
 e2e-upload:
 		./tools/code-sign.sh -- $(MAKE) -C E2ETests
-
-# Helpers
-
-ECHO_TITLE=./tools/utils/echo_color.sh --title
-ECHO_ERROR=./tools/utils/echo_color.sh --err
-ECHO_WARNING=./tools/utils/echo_color.sh --warn
-ECHO_SUCCESS=./tools/utils/echo_color.sh --succ
-
-define require_param
-    if [ -z "$${$(1)}" ]; then \
-        $(ECHO_ERROR) "Error:" "$(1) parameter is required but not provided."; \
-        exit 1; \
-    fi
-endef
diff --git a/dependency-manager-tests/.gitignore → SmokeTests/.gitignore b/dependency-manager-tests/.gitignore → SmokeTests/.gitignore
diff --git a/...ager-tests/carthage/App/AppDelegate.swift → SmokeTests/carthage/App/AppDelegate.swift b/...ager-tests/carthage/App/AppDelegate.swift → SmokeTests/carthage/App/AppDelegate.swift
diff --git a/...ncy-manager-tests/carthage/App/Info.plist → SmokeTests/carthage/App/Info.plist b/...ncy-manager-tests/carthage/App/Info.plist → SmokeTests/carthage/App/Info.plist
diff --git a/...-tests/carthage/App/PrivacyInfo.xcprivacy → ...eTests/carthage/App/PrivacyInfo.xcprivacy b/...-tests/carthage/App/PrivacyInfo.xcprivacy → ...eTests/carthage/App/PrivacyInfo.xcprivacy
diff --git a/...er-tests/carthage/App/SceneDelegate.swift → SmokeTests/carthage/App/SceneDelegate.swift b/...er-tests/carthage/App/SceneDelegate.swift → SmokeTests/carthage/App/SceneDelegate.swift
diff --git a/...r-tests/carthage/App/ViewController.swift → SmokeTests/carthage/App/ViewController.swift b/...r-tests/carthage/App/ViewController.swift → SmokeTests/carthage/App/ViewController.swift
diff --git a/...er-tests/carthage/AppTests/AppTests.swift → SmokeTests/carthage/AppTests/AppTests.swift b/...er-tests/carthage/AppTests/AppTests.swift → SmokeTests/carthage/AppTests/AppTests.swift
diff --git a/...anager-tests/carthage/AppTests/Info.plist → SmokeTests/carthage/AppTests/Info.plist b/...anager-tests/carthage/AppTests/Info.plist → SmokeTests/carthage/AppTests/Info.plist
diff --git a/...ests/carthage/AppUITests/AppUITests.swift → ...ests/carthage/AppUITests/AppUITests.swift b/...ests/carthage/AppUITests/AppUITests.swift → ...ests/carthage/AppUITests/AppUITests.swift
diff --git a/...ager-tests/carthage/AppUITests/Info.plist → SmokeTests/carthage/AppUITests/Info.plist b/...ager-tests/carthage/AppUITests/Info.plist → SmokeTests/carthage/AppUITests/Info.plist
diff --git a/...thage/CTProject.xcodeproj/project.pbxproj → ...thage/CTProject.xcodeproj/project.pbxproj b/...thage/CTProject.xcodeproj/project.pbxproj → ...thage/CTProject.xcodeproj/project.pbxproj
diff --git a/...ject.xcworkspace/contents.xcworkspacedata → ...ject.xcworkspace/contents.xcworkspacedata b/...ject.xcworkspace/contents.xcworkspacedata → ...ject.xcworkspace/contents.xcworkspacedata
diff --git a/...ace/xcshareddata/IDEWorkspaceChecks.plist → ...ace/xcshareddata/IDEWorkspaceChecks.plist b/...ace/xcshareddata/IDEWorkspaceChecks.plist → ...ace/xcshareddata/IDEWorkspaceChecks.plist
diff --git a/...j/xcshareddata/xcschemes/App iOS.xcscheme → ...j/xcshareddata/xcschemes/App iOS.xcscheme b/...j/xcshareddata/xcschemes/App iOS.xcscheme → ...j/xcshareddata/xcschemes/App iOS.xcscheme
diff --git a/.../xcshareddata/xcschemes/App tvOS.xcscheme → .../xcshareddata/xcschemes/App tvOS.xcscheme b/.../xcshareddata/xcschemes/App tvOS.xcscheme → .../xcshareddata/xcschemes/App tvOS.xcscheme
diff --git a/SmokeTests/carthage/Cartfile.src b/SmokeTests/carthage/Cartfile.src
@@ -0,0 +1 @@
+github "DataDog/dd-sdk-ios" "GIT_REFERENCE"