From 5003898afff4d5df5a9f4746a2f9df19f783ca7b Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Wed, 7 Aug 2024 13:45:41 +0200 Subject: [PATCH 1/7] RUM-5555 Collect memory metric --- .../BenchmarkTests.xcodeproj/project.pbxproj | 13 +- .../xcshareddata/xcschemes/Runner.xcscheme | 12 ++ BenchmarkTests/Benchmarks/Package.swift | 6 +- .../Benchmarks/Sources/Benchmarks.swift | 125 +++++++++++++++ .../Benchmarks/Sources/DatadogExporter.swift | 46 ------ .../Benchmarks/Sources/Exporter.swift | 26 +++ .../Benchmarks/Sources/MetricExporter.swift | 151 ++++++++++++++++++ .../Benchmarks/Sources/Metrics.swift | 40 +++++ BenchmarkTests/Runner/AppConfiguration.swift | 10 +- BenchmarkTests/Runner/AppDelegate.swift | 62 ++++++- .../Runner/Scenarios/DefaultScenario.swift | 54 ------- .../Runner/Scenarios/Scenario.swift | 61 ++----- .../SessionReplay/SessionReplayScenario.swift | 10 +- .../Runner/Scenarios/SyntheticScenario.swift | 69 ++++++++ 14 files changed, 513 insertions(+), 172 deletions(-) create mode 100644 BenchmarkTests/Benchmarks/Sources/Benchmarks.swift delete mode 100644 BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift create mode 100644 BenchmarkTests/Benchmarks/Sources/Exporter.swift create mode 100644 BenchmarkTests/Benchmarks/Sources/MetricExporter.swift create mode 100644 BenchmarkTests/Benchmarks/Sources/Metrics.swift delete mode 100644 BenchmarkTests/Runner/Scenarios/DefaultScenario.swift create mode 100644 BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj index 4cb16fa4c4..4bedf276ce 100644 --- a/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj @@ -8,10 +8,10 @@ /* Begin PBXBuildFile section */ D23DD32D2C58D80C00B90C4C /* DatadogBenchmarks in Frameworks */ = {isa = PBXBuildFile; productRef = D23DD32C2C58D80C00B90C4C /* DatadogBenchmarks */; }; + D24BFD472C6B916B00AB9604 /* SyntheticScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */; }; D276069F2C514F37002D2A14 /* SessionReplay.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = D27606962C514F37002D2A14 /* SessionReplay.storyboard */; }; D27606A02C514F37002D2A14 /* SessionReplayController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606972C514F37002D2A14 /* SessionReplayController.swift */; }; D27606A12C514F37002D2A14 /* SessionReplayScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606982C514F37002D2A14 /* SessionReplayScenario.swift */; }; - D27606A22C514F37002D2A14 /* DefaultScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069A2C514F37002D2A14 /* DefaultScenario.swift */; }; D27606A32C514F37002D2A14 /* Scenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069B2C514F37002D2A14 /* Scenario.swift */; }; D27606A42C514F37002D2A14 /* AppConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069D2C514F37002D2A14 /* AppConfiguration.swift */; }; D27606A72C514F77002D2A14 /* DatadogCore in Frameworks */ = {isa = PBXBuildFile; productRef = D27606A62C514F77002D2A14 /* DatadogCore */; }; @@ -36,10 +36,10 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyntheticScenario.swift; sourceTree = ""; }; D27606962C514F37002D2A14 /* SessionReplay.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = SessionReplay.storyboard; sourceTree = ""; }; D27606972C514F37002D2A14 /* SessionReplayController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayController.swift; sourceTree = ""; }; D27606982C514F37002D2A14 /* SessionReplayScenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayScenario.swift; sourceTree = ""; }; - D276069A2C514F37002D2A14 /* DefaultScenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DefaultScenario.swift; sourceTree = ""; }; D276069B2C514F37002D2A14 /* Scenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Scenario.swift; sourceTree = ""; }; D276069D2C514F37002D2A14 /* AppConfiguration.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppConfiguration.swift; sourceTree = ""; }; D27606B22C526908002D2A14 /* Benchmarks.local.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Benchmarks.local.xcconfig; sourceTree = ""; }; @@ -82,9 +82,9 @@ D276069C2C514F37002D2A14 /* Scenarios */ = { isa = PBXGroup; children = ( - D27606992C514F37002D2A14 /* SessionReplay */, - D276069A2C514F37002D2A14 /* DefaultScenario.swift */, D276069B2C514F37002D2A14 /* Scenario.swift */, + D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */, + D27606992C514F37002D2A14 /* SessionReplay */, ); path = Scenarios; sourceTree = ""; @@ -219,8 +219,8 @@ files = ( D27606A42C514F37002D2A14 /* AppConfiguration.swift in Sources */, D29F75502C4AA07E00288638 /* AppDelegate.swift in Sources */, - D27606A22C514F37002D2A14 /* DefaultScenario.swift in Sources */, D27606A12C514F37002D2A14 /* SessionReplayScenario.swift in Sources */, + D24BFD472C6B916B00AB9604 /* SyntheticScenario.swift in Sources */, D27606A32C514F37002D2A14 /* Scenario.swift in Sources */, D27606A02C514F37002D2A14 /* SessionReplayController.swift in Sources */, ); @@ -295,6 +295,7 @@ CURRENT_PROJECT_VERSION = f34790fea; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = Runner/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "Datadog Benchmark Runner"; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; @@ -442,6 +443,7 @@ CURRENT_PROJECT_VERSION = f34790fea; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = Runner/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "Datadog Benchmark Runner"; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; @@ -470,6 +472,7 @@ CURRENT_PROJECT_VERSION = f34790fea; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = Runner/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "Datadog Benchmark Runner"; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 47396a3dc5..7210b46fda 100644 --- a/BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -50,6 +50,18 @@ ReferencedContainer = "container:BenchmarkTests.xcodeproj"> + + + + + + SpanExporterResultCode { - return .success - } - - public func export(metrics: [Metric], shouldCancel: (() -> Bool)?) -> MetricExporterResultCode { - return .success - } - - public func flush() -> SpanExporterResultCode { - return .success - } - - public func shutdown() { - - } -} - -#endif diff --git a/BenchmarkTests/Benchmarks/Sources/Exporter.swift b/BenchmarkTests/Benchmarks/Sources/Exporter.swift new file mode 100644 index 0000000000..d7329ac2e1 --- /dev/null +++ b/BenchmarkTests/Benchmarks/Sources/Exporter.swift @@ -0,0 +1,26 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import OpenTelemetrySdk + +internal class Exporter { + class SessionDelegate: NSObject {} + + let session: URLSession + + init() { + let configuration: URLSessionConfiguration = .ephemeral + configuration.urlCache = nil + self.session = URLSession(configuration: configuration, delegate: SessionDelegate(), delegateQueue: nil) + } +} + +extension Exporter.SessionDelegate: URLSessionTaskDelegate { + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: (any Error)?) { + + } +} diff --git a/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift b/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift new file mode 100644 index 0000000000..13f1ec016f --- /dev/null +++ b/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift @@ -0,0 +1,151 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import OpenTelemetrySdk + +enum MetricExporterError: Error { + case unsupportedMetric(aggregation: AggregationType, dataType: Any.Type) +} + +/// Replacement of otel `DatadogExporter` for metrics. +/// +/// This version doesn not store data to disk, it uploads to the intake directly. +/// Additionally, it does not crash. +final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { + struct Configuration { + let apiKey: String + let version: String + } + + /// The type of metric. The available types are 0 (unspecified), 1 (count), 2 (rate), and 3 (gauge). Allowed enum values: 0,1,2,3 + enum MetricType: Int, Codable { + case unspecified = 0 + case count = 1 + case rate = 2 + case gauge = 3 + } + + internal struct Serie: Codable { + struct Point: Codable { + let timestamp: Int64 + let value: Double + } + + struct Resource: Codable { + let name: String + let type: String + } + + let type: MetricType + let interval: Int64? + let metric: String + let unit: String? + let points: [Point] + let resources: [Resource] + let tags: [String] + } + + let encoder = JSONEncoder() + let configuration: Configuration + + // swiftlint:disable force_unwrapping + let intake = URL(string: "https://api.datadoghq.com/api/v2/series")! + let prefix = "{ \"series\": [".data(using: .utf8)! + let separator = ",".data(using: .utf8)! + let suffix = "]}".data(using: .utf8)! + // swiftlint:enable force_unwrapping + + required init(configuration: Configuration) { + self.configuration = configuration + super.init() + } + + func export(metrics: [Metric], shouldCancel: (() -> Bool)?) -> MetricExporterResultCode { + do { + let series = try metrics.map { try export(metric: $0) } + try submit(series: series) + return.success + } catch { + return .failureNotRetryable + } + } + + func export(metric: Metric) throws -> Serie { + var tags: Set = [] + + let points: [Serie.Point] = try metric.data.map { data in + let timestamp = Int64(data.timestamp.timeIntervalSince1970) + + data.labels.forEach { tags.insert("\($0):\($1)") } + + switch data { + case let data as SumData: + return Serie.Point(timestamp: timestamp, value: data.sum) + case let data as SumData: + return Serie.Point(timestamp: timestamp, value: Double(data.sum)) + case let data as SummaryData: + return Serie.Point(timestamp: timestamp, value: data.sum) + case let data as SummaryData: + return Serie.Point(timestamp: timestamp, value: Double(data.sum)) +// case let data as HistogramData: +// return Serie.Point(timestamp: timestamp, value: Double(data.sum)) +// case let data as HistogramData: +// return Serie.Point(timestamp: timestamp, value: data.sum) + default: + throw MetricExporterError.unsupportedMetric( + aggregation: metric.aggregationType, + dataType: type(of: data) + ) + } + } + + return Serie( + type: MetricType(metric.aggregationType), + interval: nil, + metric: metric.name, + unit: nil, + points: points, + resources: [], + tags: Array(tags) + ) + } + + func submit(series: [Serie]) throws { + var data = try series.reduce(Data()) { data, serie in + try data + encoder.encode(serie) + separator + } + + // remove last separator + data.removeLast(separator.count) + + var request = URLRequest(url: intake) + request.httpMethod = "POST" + request.allHTTPHeaderFields = [ + "Content-Type": "application/json", + "DD-API-KEY": configuration.apiKey, + "DD-EVP-ORIGIN": "ios", + "DD-EVP-ORIGIN-VERSION": configuration.version, + "DD-REQUEST-ID": UUID().uuidString, + ] + + request.httpBody = prefix + data + suffix + session.dataTask(with: request).resume() + } +} + +private extension MetricExporter.MetricType { + init(_ type: OpenTelemetrySdk.AggregationType) { + switch type { + case .doubleSum, .intSum: + self = .count + case .intGauge, .doubleGauge: + self = .gauge + case .doubleSummary, .intSummary, .doubleHistogram, .intHistogram: + self = .unspecified + } + } +} diff --git a/BenchmarkTests/Benchmarks/Sources/Metrics.swift b/BenchmarkTests/Benchmarks/Sources/Metrics.swift new file mode 100644 index 0000000000..f8ca08d6f3 --- /dev/null +++ b/BenchmarkTests/Benchmarks/Sources/Metrics.swift @@ -0,0 +1,40 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation + +// The `TASK_VM_INFO_COUNT` and `TASK_VM_INFO_REV1_COUNT` macros are too +// complex for the Swift C importer, so we have to define them ourselves. +let TASK_VM_INFO_COUNT = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) +let TASK_VM_INFO_REV1_COUNT = mach_msg_type_number_t(MemoryLayout.offset(of: \task_vm_info_data_t.min_address)! / MemoryLayout.size) + +enum MachError: Error { + case kernelError(kern_return_t) +} + +public enum Memory { + /// Collects single sample of current memory footprint. + /// + /// The computation is based on https://developer.apple.com/forums/thread/105088 + /// It leverages recommended `phys_footprint` value, which returns values that are close to Xcode's _Memory Use_ + /// gauge and _Allocations Instrument_. + /// + /// - Returns: Current memory footprint in bytes, `throws` if failed to read. + static func footprint() throws -> Double { + var info = task_vm_info_data_t() + var count = TASK_VM_INFO_COUNT + let kr = withUnsafeMutablePointer(to: &info) { infoPtr in + infoPtr.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { intPtr in + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), intPtr, &count) + } + } + guard kr == KERN_SUCCESS, count >= TASK_VM_INFO_REV1_COUNT else { + throw MachError.kernelError(kr) + } + + return Double(info.phys_footprint) + } +} diff --git a/BenchmarkTests/Runner/AppConfiguration.swift b/BenchmarkTests/Runner/AppConfiguration.swift index d6e13ccdec..be251360c6 100644 --- a/BenchmarkTests/Runner/AppConfiguration.swift +++ b/BenchmarkTests/Runner/AppConfiguration.swift @@ -8,7 +8,7 @@ import Foundation import DatadogInternal import DatadogCore -/// Test info reads configuration from `Info.plist`. +/// Application info reads configuration from `Info.plist`. /// /// The expected format is as follow: /// @@ -27,7 +27,7 @@ import DatadogCore /// $(DD_SITE) /// /// -struct TestInfo: Decodable { +struct AppInfo: Decodable { let clientToken: String let applicationID: String let apiKey: String @@ -43,7 +43,7 @@ struct TestInfo: Decodable { } } -extension TestInfo { +extension AppInfo { init(bundle: Bundle = .main) throws { let decoder = AnyDecoder() let obj = bundle.object(forInfoDictionaryKey: "DatadogConfiguration") @@ -51,7 +51,7 @@ extension TestInfo { } } -extension TestInfo { +extension AppInfo { static var empty: Self { .init( clientToken: "", @@ -66,7 +66,7 @@ extension TestInfo { extension DatadogSite: Decodable {} extension Datadog.Configuration { - static func benchmark(info: TestInfo) -> Self { + static func benchmark(info: AppInfo) -> Self { .init( clientToken: info.clientToken, env: info.env, diff --git a/BenchmarkTests/Runner/AppDelegate.swift b/BenchmarkTests/Runner/AppDelegate.swift index 212ec75bad..12f6582b37 100644 --- a/BenchmarkTests/Runner/AppDelegate.swift +++ b/BenchmarkTests/Runner/AppDelegate.swift @@ -5,20 +5,76 @@ */ import UIKit +import DatadogInternal +import DatadogBenchmarks @main class AppDelegate: UIResponder, UIApplicationDelegate { var window: UIWindow? func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { - let info = try! TestInfo() // crash if test info are missing or malformed + guard + let scenario = SyntheticScenario(), + let run = SyntheticRun() + else { + return false + } - let scenario: Scenario = SyntheticScenario() ?? DefaultScenario() + let applicationInfo = try! AppInfo() // crash if info are missing or malformed + + switch run { + case .baseline, .metrics: + // measure metrics during baseline and metrics runs + Benchmarks.metrics( + with: Benchmarks.Configuration( + info: applicationInfo, + scenario: scenario, + run: run + ) + ) + case .profiling: + // collect profiles + break + } + + if run != .baseline { + // instrument the application with Datadog SDK + // when not in baseline run + scenario.instrument(with: applicationInfo) + } window = UIWindow(frame: UIScreen.main.bounds) - window?.rootViewController = scenario.start(info: info) + window?.rootViewController = scenario.initialViewController window?.makeKeyAndVisible() return true } } + +extension Benchmarks.Configuration { + init( + info: AppInfo, + scenario: SyntheticScenario, + run: SyntheticRun, + bundle: Bundle = .main, + sysctl: SysctlProviding = Sysctl(), + device: UIDevice = .current + ) { + self.init( + clientToken: info.clientToken, + apiKey: info.apiKey, + context: Benchmarks.Configuration.Context( + applicationIdentifier: bundle.bundleIdentifier!, + applicationName: bundle.object(forInfoDictionaryKey: "CFBundleExecutable") as! String, + applicationVersion: bundle.object(forInfoDictionaryKey: "CFBundleShortVersionString") as! String, + sdkVersion: "", + deviceModel: try! sysctl.model(), + osName: device.systemName, + osVersion: device.systemVersion, + run: run.rawValue, + scenario: scenario.name, + branch: "" + ) + ) + } +} diff --git a/BenchmarkTests/Runner/Scenarios/DefaultScenario.swift b/BenchmarkTests/Runner/Scenarios/DefaultScenario.swift deleted file mode 100644 index d6760fed82..0000000000 --- a/BenchmarkTests/Runner/Scenarios/DefaultScenario.swift +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. - * This product includes software developed at Datadog (https://www.datadoghq.com/). - * Copyright 2019-Present Datadog, Inc. - */ - -import Foundation -import UIKit -import SwiftUI - -/// The default scenario will present the list of Synthetic scenarios to run in development mode. -/// To skip this screen, you can set the `E2E_SCENARIO` environment variable with the name -/// the desired scenario. -struct DefaultScenario: Scenario { - func start(info: TestInfo) -> UIViewController { - UIHostingController(rootView: ContentView(info: info)) - } - - struct ContentView: View { - let info: TestInfo - - var body: some View { - NavigationView { - List(SyntheticScenario.allCases, id: \.rawValue) { scenario in - NavigationLink { - ScenarioView(info: info, scenario: scenario) - } label: { - Text(scenario.rawValue) - } - } - .navigationBarTitle("Scenarios") - } - } - } - - struct ScenarioView: UIViewControllerRepresentable { - let info: TestInfo - let scenario: Scenario - - func makeUIViewController(context: Context) -> UIViewController { - scenario.start(info: info) - } - - func updateUIViewController(_ uiViewController: UIViewController, context: Context) { } - } -} - -#if DEBUG -struct DefaultScenario_Previews: PreviewProvider { - static var previews: some View { - DefaultScenario.ContentView(info: .empty) - } -} -#endif diff --git a/BenchmarkTests/Runner/Scenarios/Scenario.swift b/BenchmarkTests/Runner/Scenarios/Scenario.swift index 7a5abd9f85..b844e8ce07 100644 --- a/BenchmarkTests/Runner/Scenarios/Scenario.swift +++ b/BenchmarkTests/Runner/Scenarios/Scenario.swift @@ -7,61 +7,18 @@ import Foundation import UIKit -/// A `Scenario` is the entry-point of the E2E runner application. +/// A `Scenario` is the entry-point of the Benchmark Runner Application. /// /// The compliant objects are responsible for initializing the SDK, enabling -/// Features, and create the root view-controller. +/// Features, and create the initial view-controller. protocol Scenario { - /// Starts the scenario. - /// - /// Starting the scenario should intialize the SDK and enable Features based on - /// the provided ``TestInfo`` and scenario's needs. - /// - /// The returned view-controller will be used as the root view controller of the - /// application window. - /// - /// - Parameter info: The test info for configuring the SDK. - /// - Returns: The root view-controller. - func start(info: TestInfo) -> UIViewController -} - -/// A Synthetic scenario can be initialized by defining a Synthetic Test Process Argument -/// named `BENCHMARK_SCENARIO`. -/// -/// Note: The raw value of enum case must match the test name defined in Synthetics. -enum SyntheticScenario: String, CaseIterable { - case sessionReplay - - /// Creates the scenario defined by the`BENCHMARK_SCENARIO` environment variable. - /// - /// - Parameter processInfo: The process info holding the environment variables. - init?(processInfo: ProcessInfo = .processInfo) { - guard - processInfo.environment["XCODE_RUNNING_FOR_PREVIEWS"] == nil, // skip SwiftUI preview - let rawValue = processInfo.environment["BENCHMARK_SCENARIO"], - let scenario = Self(rawValue: rawValue) - else { - return nil - } - - self = scenario - } - - /// Returns the scenario defined by the environment variable. - var scenario: Scenario { - switch self { - case .sessionReplay: - return SessionReplayScenario() - } - } -} + /// The initial view-controller of the scenario + var initialViewController: UIViewController { get } -extension SyntheticScenario: Scenario { - /// Starts the underlying scenario. + /// Start instrumenting the application by enabling the Datadog SDK and + /// its Features. /// - /// - Parameter info: The test info for configuring the SDK. - /// - Returns: The root view-controller. - func start(info: TestInfo) -> UIViewController { - scenario.start(info: info) - } + /// - Parameter info: The application information to use during SDK + /// initialisation. + func instrument(with info: AppInfo) } diff --git a/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift index 6b3b33ff96..09b5319e7a 100644 --- a/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift +++ b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift @@ -12,7 +12,12 @@ import DatadogRUM import DatadogSessionReplay struct SessionReplayScenario: Scenario { - func start(info: TestInfo) -> UIViewController { + var initialViewController: UIViewController { + let storyboard = UIStoryboard(name: "SessionReplay", bundle: nil) + return storyboard.instantiateInitialViewController()! + } + + func instrument(with info: AppInfo) { Datadog.initialize( with: .benchmark(info: info), trackingConsent: .granted @@ -34,8 +39,5 @@ struct SessionReplayScenario: Scenario { ) RUMMonitor.shared().addAttribute(forKey: "scenario", value: "SessionReplay") - - let storyboard = UIStoryboard(name: "SessionReplay", bundle: nil) - return storyboard.instantiateInitialViewController()! } } diff --git a/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift b/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift new file mode 100644 index 0000000000..b74fb093e4 --- /dev/null +++ b/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift @@ -0,0 +1,69 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import UIKit + +/// The Synthetics Scenario reads the `BENCHMARK_SCENARIO` environment +/// variable to instantiate a `Scenario` compliant object. +internal struct SyntheticScenario: Scenario { + /// The scenario's name. + let name: String + + /// The underlying scenario. + private let _scenario: Scenario + + /// Creates the scenario by reading the `BENCHMARK_SCENARIO` value from the + /// environment vairables. + /// + /// - Parameter processInfo: The `ProcessInfo` with environment vairables + /// configured + init?(processInfo: ProcessInfo = .processInfo) { + guard let name = processInfo.environment["BENCHMARK_SCENARIO"] else { + return nil + } + + switch name { + case "sessionReplay": + _scenario = SessionReplayScenario() + default: + return nil + } + + self.name = name + } + + var initialViewController: UIViewController { + _scenario.initialViewController + } + + func instrument(with info: AppInfo) { + _scenario.instrument(with: info) + } +} + +/// The Synthetics benchark run value. +internal enum SyntheticRun: String { + case baseline + case metrics + case profiling + + /// Creates the scenario by reading the `BENCHMARK_RUN` value from the + /// environment vairables. + /// + /// - Parameter processInfo: The `ProcessInfo` with environment vairables + /// configured + init?(processInfo: ProcessInfo = .processInfo) { + guard + let rawValue = processInfo.environment["BENCHMARK_RUN"], + let run = Self(rawValue: rawValue) + else { + return nil + } + + self = run + } +} From 5f2ced60018e5eebb48bafd3051faca616871f69 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Tue, 13 Aug 2024 16:46:21 +0200 Subject: [PATCH 2/7] Update README.md --- BenchmarkTests/README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/BenchmarkTests/README.md b/BenchmarkTests/README.md index 3d40e49f35..69edd00a87 100644 --- a/BenchmarkTests/README.md +++ b/BenchmarkTests/README.md @@ -52,7 +52,15 @@ import DatadogLogs struct LogsScenario: Scenario { - func start(info: TestInfo) -> UIViewController { + /// The initial view-controller of the scenario + let initialViewController: UIViewController = LoggerViewController() + + /// Start instrumenting the application by enabling the Datadog SDK and + /// its Features. + /// + /// - Parameter info: The application information to use during SDK + /// initialisation. + func instrument(with info: AppInfo) { Datadog.initialize( with: .benchmark(info: info), // SDK init with the benchmark configuration @@ -60,13 +68,11 @@ struct LogsScenario: Scenario { ) Logs.enable() - - return LoggerViewController() } } ``` -Add the test to the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enumeration so it can be selected, either manually or by setting the `BENCHMARK_SCENARIO` environment variable. +Add the test to the [`SyntheticScenario`](Runner/Scenarios/SyntheticScenario.swift#L12) object so it can be selected by setting the `BENCHMARK_SCENARIO` environment variable. ### Synthetics Configuration From 121b2fdfee4ec6b2d855956c286960113bbe1a50 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Tue, 13 Aug 2024 16:55:51 +0200 Subject: [PATCH 3/7] Add docs --- .../Benchmarks/Sources/Benchmarks.swift | 10 +++++++++- .../Benchmarks/Sources/MetricExporter.swift | 18 +++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift b/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift index c826400abf..b031a0dee0 100644 --- a/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift +++ b/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift @@ -15,8 +15,13 @@ import OpenTelemetrySdk let instrumentationName = "benchmarks" let instrumentationVersion = "1.0.0" +/// Benchmark entrypoint to configure opentelemetry with metrics meters +/// and tracer. public enum Benchmarks { + /// Configuration of the Benchmarks library. public struct Configuration { + /// Context of Benchmarks measures. + /// The context properties will be added metrics as tags. public struct Context { var applicationIdentifier: String var applicationName: String @@ -68,7 +73,10 @@ public enum Benchmarks { self.context = context } } - + + /// Configure OpenTelemetry metrics meter and start measuring Memory. + /// + /// - Parameter configuration: The Benchmark configuration. public static func metrics(with configuration: Configuration) { let loggerProvider = LoggerProviderBuilder() .build() diff --git a/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift b/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift index 13f1ec016f..9b51ff16d4 100644 --- a/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift +++ b/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift @@ -13,7 +13,7 @@ enum MetricExporterError: Error { /// Replacement of otel `DatadogExporter` for metrics. /// -/// This version doesn not store data to disk, it uploads to the intake directly. +/// This version does not store data to disk, it uploads to the intake directly. /// Additionally, it does not crash. final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { struct Configuration { @@ -29,6 +29,7 @@ final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { case gauge = 3 } + /// https://docs.datadoghq.com/api/latest/metrics/#submit-metrics internal struct Serie: Codable { struct Point: Codable { let timestamp: Int64 @@ -66,15 +67,19 @@ final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { func export(metrics: [Metric], shouldCancel: (() -> Bool)?) -> MetricExporterResultCode { do { - let series = try metrics.map { try export(metric: $0) } + let series = try metrics.map(transform) try submit(series: series) return.success } catch { return .failureNotRetryable } } - - func export(metric: Metric) throws -> Serie { + + /// Transforms otel `Metric` to Datadog `serie`. + /// + /// - Parameter metric: The otel metric + /// - Returns: The timeserie. + func transform(_ metric: Metric) throws -> Serie { var tags: Set = [] let points: [Serie.Point] = try metric.data.map { data in @@ -113,7 +118,10 @@ final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { tags: Array(tags) ) } - + + /// Submit timeseries to the Metrics intake. + /// + /// - Parameter series: The timeseries. func submit(series: [Serie]) throws { var data = try series.reduce(Data()) { data, serie in try data + encoder.encode(serie) + separator From 8cfc52100d591ba6daad10e75f9f0d91be59a135 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Wed, 14 Aug 2024 17:03:45 +0200 Subject: [PATCH 4/7] Update README.md --- BenchmarkTests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BenchmarkTests/README.md b/BenchmarkTests/README.md index 69edd00a87..38c1e6fcbc 100644 --- a/BenchmarkTests/README.md +++ b/BenchmarkTests/README.md @@ -5,7 +5,7 @@ ## CI -CI continuously builds, signs, and uploads a runner application to Synthetics which runs predefined tests. +CI continuously builds, signs, and uploads a runner application to Synthetics, which runs predefined tests. ### Build From 5a2ee6f27510e358f001b013a18136a78c08dec6 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Mon, 19 Aug 2024 13:30:53 +0200 Subject: [PATCH 5/7] Remove exporter base class --- .../Benchmarks/Sources/Exporter.swift | 26 ------------------- .../Benchmarks/Sources/MetricExporter.swift | 7 +++-- 2 files changed, 5 insertions(+), 28 deletions(-) delete mode 100644 BenchmarkTests/Benchmarks/Sources/Exporter.swift diff --git a/BenchmarkTests/Benchmarks/Sources/Exporter.swift b/BenchmarkTests/Benchmarks/Sources/Exporter.swift deleted file mode 100644 index d7329ac2e1..0000000000 --- a/BenchmarkTests/Benchmarks/Sources/Exporter.swift +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. - * This product includes software developed at Datadog (https://www.datadoghq.com/). - * Copyright 2019-Present Datadog, Inc. - */ - -import Foundation -import OpenTelemetrySdk - -internal class Exporter { - class SessionDelegate: NSObject {} - - let session: URLSession - - init() { - let configuration: URLSessionConfiguration = .ephemeral - configuration.urlCache = nil - self.session = URLSession(configuration: configuration, delegate: SessionDelegate(), delegateQueue: nil) - } -} - -extension Exporter.SessionDelegate: URLSessionTaskDelegate { - func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: (any Error)?) { - - } -} diff --git a/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift b/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift index 9b51ff16d4..598ce60239 100644 --- a/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift +++ b/BenchmarkTests/Benchmarks/Sources/MetricExporter.swift @@ -15,7 +15,7 @@ enum MetricExporterError: Error { /// /// This version does not store data to disk, it uploads to the intake directly. /// Additionally, it does not crash. -final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { +final class MetricExporter: OpenTelemetrySdk.MetricExporter { struct Configuration { let apiKey: String let version: String @@ -50,6 +50,7 @@ final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { let tags: [String] } + let session: URLSession let encoder = JSONEncoder() let configuration: Configuration @@ -61,8 +62,10 @@ final class MetricExporter: Exporter, OpenTelemetrySdk.MetricExporter { // swiftlint:enable force_unwrapping required init(configuration: Configuration) { + let sessionConfiguration: URLSessionConfiguration = .ephemeral + sessionConfiguration.urlCache = nil + self.session = URLSession(configuration: sessionConfiguration) self.configuration = configuration - super.init() } func export(metrics: [Metric], shouldCancel: (() -> Bool)?) -> MetricExporterResultCode { From 92a2b0acbb4bf79c0f215810f7d336827457d79d Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Mon, 19 Aug 2024 14:55:02 +0200 Subject: [PATCH 6/7] Use enum for scenarios --- BenchmarkTests/Runner/AppDelegate.swift | 2 +- .../Runner/Scenarios/SyntheticScenario.swift | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/BenchmarkTests/Runner/AppDelegate.swift b/BenchmarkTests/Runner/AppDelegate.swift index 12f6582b37..a476311e3f 100644 --- a/BenchmarkTests/Runner/AppDelegate.swift +++ b/BenchmarkTests/Runner/AppDelegate.swift @@ -72,7 +72,7 @@ extension Benchmarks.Configuration { osName: device.systemName, osVersion: device.systemVersion, run: run.rawValue, - scenario: scenario.name, + scenario: scenario.name.rawValue, branch: "" ) ) diff --git a/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift b/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift index b74fb093e4..02d678c98f 100644 --- a/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift +++ b/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift @@ -10,27 +10,32 @@ import UIKit /// The Synthetics Scenario reads the `BENCHMARK_SCENARIO` environment /// variable to instantiate a `Scenario` compliant object. internal struct SyntheticScenario: Scenario { + internal enum Name: String { + case sessionReplay + + } /// The scenario's name. - let name: String - + let name: Name + /// The underlying scenario. private let _scenario: Scenario /// Creates the scenario by reading the `BENCHMARK_SCENARIO` value from the - /// environment vairables. + /// environment variables. /// - /// - Parameter processInfo: The `ProcessInfo` with environment vairables + /// - Parameter processInfo: The `ProcessInfo` with environment variables /// configured init?(processInfo: ProcessInfo = .processInfo) { - guard let name = processInfo.environment["BENCHMARK_SCENARIO"] else { + guard + let rawValue = processInfo.environment["BENCHMARK_SCENARIO"], + let name = Name(rawValue: rawValue) + else { return nil } switch name { - case "sessionReplay": + case .sessionReplay: _scenario = SessionReplayScenario() - default: - return nil } self.name = name @@ -52,9 +57,9 @@ internal enum SyntheticRun: String { case profiling /// Creates the scenario by reading the `BENCHMARK_RUN` value from the - /// environment vairables. + /// environment variables. /// - /// - Parameter processInfo: The `ProcessInfo` with environment vairables + /// - Parameter processInfo: The `ProcessInfo` with environment variables /// configured init?(processInfo: ProcessInfo = .processInfo) { guard From 71769c441b76e1019d5f254109817734b4988f83 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Tue, 20 Aug 2024 12:06:04 +0200 Subject: [PATCH 7/7] Apply CR review suggestions --- BenchmarkTests/Benchmarks/Sources/Benchmarks.swift | 2 +- BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift b/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift index b031a0dee0..e1338cb737 100644 --- a/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift +++ b/BenchmarkTests/Benchmarks/Sources/Benchmarks.swift @@ -21,7 +21,7 @@ public enum Benchmarks { /// Configuration of the Benchmarks library. public struct Configuration { /// Context of Benchmarks measures. - /// The context properties will be added metrics as tags. + /// The context properties will be added to metrics as tags. public struct Context { var applicationIdentifier: String var applicationName: String diff --git a/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift b/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift index 02d678c98f..8a656f5d2b 100644 --- a/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift +++ b/BenchmarkTests/Runner/Scenarios/SyntheticScenario.swift @@ -10,9 +10,9 @@ import UIKit /// The Synthetics Scenario reads the `BENCHMARK_SCENARIO` environment /// variable to instantiate a `Scenario` compliant object. internal struct SyntheticScenario: Scenario { + /// The Synthetics benchmark scenario value. internal enum Name: String { case sessionReplay - } /// The scenario's name. let name: Name @@ -50,10 +50,10 @@ internal struct SyntheticScenario: Scenario { } } -/// The Synthetics benchark run value. +/// The Synthetics benchmark run value. internal enum SyntheticRun: String { case baseline - case metrics + case instrumented case profiling /// Creates the scenario by reading the `BENCHMARK_RUN` value from the