Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RUM-5555 Benchmarks: Collect Memory Metric #1993

Merged
merged 7 commits into from
Aug 20, 2024
+522 −177
Merged

RUM-5555 Benchmarks: Collect Memory Metric #1993

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5003898
RUM-5555 Collect memory metric
maxep Aug 7, 2024
5f2ced6
Update README.md
maxep Aug 13, 2024
121b2fd
Add docs
maxep Aug 13, 2024
8cfc521
Update README.md
maxep Aug 14, 2024
5a2ee6f
Remove exporter base class
maxep Aug 19, 2024
92a2b0a
Use enum for scenarios
maxep Aug 19, 2024
71769c4
Apply CR review suggestions
maxep Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@

/* Begin PBXBuildFile section */
D23DD32D2C58D80C00B90C4C /* DatadogBenchmarks in Frameworks */ = {isa = PBXBuildFile; productRef = D23DD32C2C58D80C00B90C4C /* DatadogBenchmarks */; };
D24BFD472C6B916B00AB9604 /* SyntheticScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */; };
D276069F2C514F37002D2A14 /* SessionReplay.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = D27606962C514F37002D2A14 /* SessionReplay.storyboard */; };
D27606A02C514F37002D2A14 /* SessionReplayController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606972C514F37002D2A14 /* SessionReplayController.swift */; };
D27606A12C514F37002D2A14 /* SessionReplayScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606982C514F37002D2A14 /* SessionReplayScenario.swift */; };
D27606A22C514F37002D2A14 /* DefaultScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069A2C514F37002D2A14 /* DefaultScenario.swift */; };
D27606A32C514F37002D2A14 /* Scenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069B2C514F37002D2A14 /* Scenario.swift */; };
D27606A42C514F37002D2A14 /* AppConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069D2C514F37002D2A14 /* AppConfiguration.swift */; };
D27606A72C514F77002D2A14 /* DatadogCore in Frameworks */ = {isa = PBXBuildFile; productRef = D27606A62C514F77002D2A14 /* DatadogCore */; };
Expand All @@ -36,10 +36,10 @@
/* End PBXCopyFilesBuildPhase section */

/* Begin PBXFileReference section */
D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyntheticScenario.swift; sourceTree = "<group>"; };
D27606962C514F37002D2A14 /* SessionReplay.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = SessionReplay.storyboard; sourceTree = "<group>"; };
D27606972C514F37002D2A14 /* SessionReplayController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayController.swift; sourceTree = "<group>"; };
D27606982C514F37002D2A14 /* SessionReplayScenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayScenario.swift; sourceTree = "<group>"; };
D276069A2C514F37002D2A14 /* DefaultScenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DefaultScenario.swift; sourceTree = "<group>"; };
D276069B2C514F37002D2A14 /* Scenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Scenario.swift; sourceTree = "<group>"; };
D276069D2C514F37002D2A14 /* AppConfiguration.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppConfiguration.swift; sourceTree = "<group>"; };
D27606B22C526908002D2A14 /* Benchmarks.local.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Benchmarks.local.xcconfig; sourceTree = "<group>"; };
Expand Down Expand Up @@ -82,9 +82,9 @@
D276069C2C514F37002D2A14 /* Scenarios */ = {
isa = PBXGroup;
children = (
D27606992C514F37002D2A14 /* SessionReplay */,
D276069A2C514F37002D2A14 /* DefaultScenario.swift */,
D276069B2C514F37002D2A14 /* Scenario.swift */,
D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */,
D27606992C514F37002D2A14 /* SessionReplay */,
);
path = Scenarios;
sourceTree = "<group>";
Expand Down Expand Up @@ -219,8 +219,8 @@
files = (
D27606A42C514F37002D2A14 /* AppConfiguration.swift in Sources */,
D29F75502C4AA07E00288638 /* AppDelegate.swift in Sources */,
D27606A22C514F37002D2A14 /* DefaultScenario.swift in Sources */,
D27606A12C514F37002D2A14 /* SessionReplayScenario.swift in Sources */,
D24BFD472C6B916B00AB9604 /* SyntheticScenario.swift in Sources */,
D27606A32C514F37002D2A14 /* Scenario.swift in Sources */,
D27606A02C514F37002D2A14 /* SessionReplayController.swift in Sources */,
);
Expand Down Expand Up @@ -295,6 +295,7 @@
CURRENT_PROJECT_VERSION = f34790fea;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Runner/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Datadog Benchmark Runner";
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
Expand Down Expand Up @@ -442,6 +443,7 @@
CURRENT_PROJECT_VERSION = f34790fea;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Runner/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Datadog Benchmark Runner";
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
Expand Down Expand Up @@ -470,6 +472,7 @@
CURRENT_PROJECT_VERSION = f34790fea;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Runner/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Datadog Benchmark Runner";
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
Expand Down
12 changes: 12 additions & 0 deletions BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,18 @@
ReferencedContainer = "container:BenchmarkTests.xcodeproj">
</BuildableReference>
</BuildableProductRunnable>
<EnvironmentVariables>
<EnvironmentVariable
key = "BENCHMARK_RUN"
value = "metrics"
isEnabled = "YES">
</EnvironmentVariable>
<EnvironmentVariable
key = "BENCHMARK_SCENARIO"
value = "sessionReplay"
isEnabled = "YES">
</EnvironmentVariable>
</EnvironmentVariables>
</LaunchAction>
<ProfileAction
buildConfiguration = "Release"
Expand Down
6 changes: 3 additions & 3 deletions BenchmarkTests/Benchmarks/Package.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ let package = Package(
products: [
.library(
name: "DatadogBenchmarks",
targets: ["Benchmarks"]
targets: ["DatadogBenchmarks"]
)
]
)
Expand All @@ -28,7 +28,7 @@ func addOpenTelemetryDependency(_ version: Version) {

package.targets = [
.target(
name: "Benchmarks",
name: "DatadogBenchmarks",
dependencies: [
.product(name: "OpenTelemetryApi", package: "opentelemetry-swift"),
.product(name: "OpenTelemetrySdk", package: "opentelemetry-swift")
Expand All @@ -44,7 +44,7 @@ func addOpenTelemetryDependency(_ version: Version) {

package.targets = [
.target(
name: "Benchmarks",
name: "DatadogBenchmarks",
dependencies: [
.product(name: "OpenTelemetryApi", package: "opentelemetry-swift-packages")
],
Expand Down
133 changes: 133 additions & 0 deletions BenchmarkTests/Benchmarks/Sources/Benchmarks.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
/*
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
* This product includes software developed at Datadog (https://www.datadoghq.com/).
* Copyright 2019-Present Datadog, Inc.
*/

#if OTEL_API
#error("Benchmarks depends on opentelemetry-swift. Please open the project with 'make benchmark-tests-open'.")
#endif

import Foundation
import OpenTelemetryApi
import OpenTelemetrySdk

let instrumentationName = "benchmarks"
let instrumentationVersion = "1.0.0"

/// Benchmark entrypoint to configure opentelemetry with metrics meters
/// and tracer.
public enum Benchmarks {
/// Configuration of the Benchmarks library.
public struct Configuration {
/// Context of Benchmarks measures.
/// The context properties will be added metrics as tags.
public struct Context {
var applicationIdentifier: String
var applicationName: String
var applicationVersion: String
var sdkVersion: String
var deviceModel: String
var osName: String
var osVersion: String
var run: String
var scenario: String
var branch: String

public init(
applicationIdentifier: String,
applicationName: String,
applicationVersion: String,
sdkVersion: String,
deviceModel: String,
osName: String,
osVersion: String,
run: String,
scenario: String,
branch: String
) {
self.applicationIdentifier = applicationIdentifier
self.applicationName = applicationName
self.applicationVersion = applicationVersion
self.sdkVersion = sdkVersion
self.deviceModel = deviceModel
self.osName = osName
self.osVersion = osVersion
self.run = run
self.scenario = scenario
self.branch = branch
}
}

var clientToken: String
var apiKey: String
var context: Context

public init(
clientToken: String,
apiKey: String,
context: Context
) {
self.clientToken = clientToken
self.apiKey = apiKey
self.context = context
}
}

/// Configure OpenTelemetry metrics meter and start measuring Memory.
///
/// - Parameter configuration: The Benchmark configuration.
public static func metrics(with configuration: Configuration) {
let loggerProvider = LoggerProviderBuilder()
.build()

let metricExporter = MetricExporter(
configuration: MetricExporter.Configuration(
apiKey: configuration.apiKey,
version: instrumentationVersion
)
)

let meterProvider = MeterProviderBuilder()
.with(pushInterval: 10)
.with(processor: MetricProcessorSdk())
.with(exporter: metricExporter)
.with(resource: Resource())
.build()

let logger = loggerProvider
.loggerBuilder(instrumentationScopeName: instrumentationName)
.build()

let meter = meterProvider.get(
instrumentationName: instrumentationName,
instrumentationVersion: instrumentationVersion
)

let labels = [
"device_model": configuration.context.deviceModel,
"os": configuration.context.osName,
"os_version": configuration.context.osVersion,
"run": configuration.context.run,
"scenario": configuration.context.scenario,
"application_id": configuration.context.applicationIdentifier,
"sdk_version": configuration.context.sdkVersion,
"branch": configuration.context.branch,
]

_ = meter.createDoubleObservableGauge(name: "ios.benchmark.memory") { metric in
do {
let mem = try Memory.footprint()
metric.observe(value: mem, labels: labels)
Comment on lines +118 to +121
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question/ How does it work? We create Gauge metric ✅ but when / how frequently will this closure be called?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is based on Asynchronous Gauge from otel specs where Callback functions will be called only when the Meter is being observed.
The meter is observed when there is a push, in our case it will be every 10s

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, thanks!

} catch {
logger.logRecordBuilder()
.setSeverity(.error)
.setAttributes(labels.mapValues { .string($0) })
.setBody("Failed to read Memory Metric: \(error)")
.emit()
}
}

OpenTelemetry.registerMeterProvider(meterProvider: meterProvider)
}
}
46 changes: 0 additions & 46 deletions BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift
View file
Open in desktop

This file was deleted.

Loading