From 278b7b1ffbbecdb3ebb4ed1df653c94d7cf02cd3 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Mon, 28 Aug 2023 11:52:38 +0100 Subject: [PATCH 01/15] REPLAY-1963 Add background task support --- CHANGELOG.md | 2 + Datadog/Datadog.xcodeproj/project.pbxproj | 12 ++ .../Upload/BackgroundTaskCoordinator.swift | 63 +++++++ .../Sources/Core/Upload/DataUploadDelay.swift | 21 ++- .../Core/Upload/DataUploadWorker.swift | 29 +++- .../Sources/Core/Upload/FeatureUpload.swift | 9 +- .../Core/Upload/DataUploadDelayTests.swift | 4 +- .../Core/Upload/DataUploadWorkerTests.swift | 162 ++++++++++-------- .../UIKitBackgroundTaskCoordinatorTests.swift | 83 +++++++++ 9 files changed, 299 insertions(+), 86 deletions(-) create mode 100644 DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift create mode 100644 DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index ee28adaa64..6a7171e905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - [IMPROVEMENT] Upgrade to PLCrashReporter 1.11.1. - [FEATURE] Report session sample rate to the backend with RUM events. See [#1410][] - [IMPROVEMENT] Expose Session Replay to Objective-C. see [#1419][] +- [IMPROVEMENT] Add UIBackgroundTask for uploading jobs. See [#1412][] # 2.0.0 / 31-07-2023 @@ -513,6 +514,7 @@ Release `2.0` introduces breaking changes. Follow the [Migration Guide](MIGRATIO [#1419]: https://github.com/DataDog/dd-sdk-ios/pull/1419 [#1428]: https://github.com/DataDog/dd-sdk-ios/pull/1428 [#1444]: https://github.com/DataDog/dd-sdk-ios/pull/1444 +[#1412]: https://github.com/DataDog/dd-sdk-ios/pull/1412 [@00fa9a]: https://github.com/00FA9A [@britton-earnin]: https://github.com/Britton-Earnin [@hengyu]: https://github.com/Hengyu diff --git a/Datadog/Datadog.xcodeproj/project.pbxproj b/Datadog/Datadog.xcodeproj/project.pbxproj index dcb943da57..ddddd1cdb1 100644 --- a/Datadog/Datadog.xcodeproj/project.pbxproj +++ b/Datadog/Datadog.xcodeproj/project.pbxproj @@ -450,6 +450,8 @@ 9E68FB55244707FD0013A8AA /* ObjcExceptionHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = 9E68FB53244707FD0013A8AA /* ObjcExceptionHandler.m */; }; 9E68FB56244707FD0013A8AA /* ObjcExceptionHandler.h in Headers */ = {isa = PBXBuildFile; fileRef = 9E68FB54244707FD0013A8AA /* ObjcExceptionHandler.h */; settings = {ATTRIBUTES = (Public, ); }; }; 9EE5AD8226205B82001E699E /* DDNSURLSessionDelegateTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9EE5AD8126205B82001E699E /* DDNSURLSessionDelegateTests.swift */; }; + A70A82652A935F210072F5DC /* BackgroundTaskCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A70A82642A935F210072F5DC /* BackgroundTaskCoordinator.swift */; }; + A70A82662A935F210072F5DC /* BackgroundTaskCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A70A82642A935F210072F5DC /* BackgroundTaskCoordinator.swift */; }; A728ADAB2934EA2100397996 /* W3CHTTPHeadersWriter+objc.swift in Sources */ = {isa = PBXBuildFile; fileRef = A728ADAA2934EA2100397996 /* W3CHTTPHeadersWriter+objc.swift */; }; A728ADAC2934EA2100397996 /* W3CHTTPHeadersWriter+objc.swift in Sources */ = {isa = PBXBuildFile; fileRef = A728ADAA2934EA2100397996 /* W3CHTTPHeadersWriter+objc.swift */; }; A728ADB02934EB0900397996 /* DDW3CHTTPHeadersWriter+apiTests.m in Sources */ = {isa = PBXBuildFile; fileRef = A728ADAD2934EB0300397996 /* DDW3CHTTPHeadersWriter+apiTests.m */; }; @@ -458,6 +460,8 @@ A79B0F65292BD074008742B3 /* DDB3HTTPHeadersWriter+apiTests.m in Sources */ = {isa = PBXBuildFile; fileRef = A79B0F63292BD074008742B3 /* DDB3HTTPHeadersWriter+apiTests.m */; }; A79B0F66292BD7CA008742B3 /* B3HTTPHeadersWriter+objc.swift in Sources */ = {isa = PBXBuildFile; fileRef = A79B0F5E292BA435008742B3 /* B3HTTPHeadersWriter+objc.swift */; }; A79B0F67292BD7CC008742B3 /* B3HTTPHeadersWriter+objc.swift in Sources */ = {isa = PBXBuildFile; fileRef = A79B0F5E292BA435008742B3 /* B3HTTPHeadersWriter+objc.swift */; }; + A7C816AB2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7C816AA2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift */; }; + A7C816AC2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7C816AA2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift */; }; D20605A3287464F40047275C /* ContextValuePublisher.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20605A2287464F40047275C /* ContextValuePublisher.swift */; }; D20605A4287464F40047275C /* ContextValuePublisher.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20605A2287464F40047275C /* ContextValuePublisher.swift */; }; D20605A6287476230047275C /* ServerOffsetPublisher.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20605A5287476230047275C /* ServerOffsetPublisher.swift */; }; @@ -2321,6 +2325,7 @@ 9EC8B5D92668197B000F7529 /* VitalCPUReader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VitalCPUReader.swift; sourceTree = ""; }; 9EC8B5ED2668E4DB000F7529 /* VitalCPUReaderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VitalCPUReaderTests.swift; sourceTree = ""; }; 9EE5AD8126205B82001E699E /* DDNSURLSessionDelegateTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDNSURLSessionDelegateTests.swift; sourceTree = ""; }; + A70A82642A935F210072F5DC /* BackgroundTaskCoordinator.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BackgroundTaskCoordinator.swift; sourceTree = ""; }; A728AD9C2934CE4400397996 /* W3CHTTPHeaders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = W3CHTTPHeaders.swift; sourceTree = ""; }; A728AD9E2934CE5000397996 /* W3CHTTPHeadersWriter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = W3CHTTPHeadersWriter.swift; sourceTree = ""; }; A728ADA02934CE5D00397996 /* W3CHTTPHeadersReader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = W3CHTTPHeadersReader.swift; sourceTree = ""; }; @@ -2332,6 +2337,7 @@ A79B0F5E292BA435008742B3 /* B3HTTPHeadersWriter+objc.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "B3HTTPHeadersWriter+objc.swift"; sourceTree = ""; }; A79B0F60292BB071008742B3 /* B3HTTPHeadersReaderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = B3HTTPHeadersReaderTests.swift; sourceTree = ""; }; A79B0F63292BD074008742B3 /* DDB3HTTPHeadersWriter+apiTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = "DDB3HTTPHeadersWriter+apiTests.m"; sourceTree = ""; }; + A7C816AA2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIKitBackgroundTaskCoordinatorTests.swift; sourceTree = ""; }; A7F773D32924EA2D00AC1A62 /* B3HTTPHeaders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = B3HTTPHeaders.swift; sourceTree = ""; }; A7F773DB29253F8B00AC1A62 /* B3HTTPHeadersWriter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = B3HTTPHeadersWriter.swift; sourceTree = ""; }; A7F773DC29253F8B00AC1A62 /* B3HTTPHeadersReader.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = B3HTTPHeadersReader.swift; sourceTree = ""; }; @@ -3875,6 +3881,7 @@ 61133C322423990D00786299 /* DataUploaderTests.swift */, 61133C342423990D00786299 /* URLSessionClientTests.swift */, 61133C332423990D00786299 /* RequestBuilderTests.swift */, + A7C816AA2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift */, ); path = Upload; sourceTree = ""; @@ -5379,6 +5386,7 @@ D26C49B428893E5300802B2D /* Upload */ = { isa = PBXGroup; children = ( + A70A82642A935F210072F5DC /* BackgroundTaskCoordinator.swift */, D26C49BE288982DA00802B2D /* FeatureUpload.swift */, 61133BB32423979B00786299 /* DataUploadDelay.swift */, 61ED39D326C2A36B002C0F26 /* DataUploadStatus.swift */, @@ -7219,6 +7227,7 @@ 61DA8CA928609C5B0074A606 /* Directories.swift in Sources */, D2EFA868286DA85700F1FAA6 /* DatadogContextProvider.swift in Sources */, D26C49BF288982DA00802B2D /* FeatureUpload.swift in Sources */, + A70A82652A935F210072F5DC /* BackgroundTaskCoordinator.swift in Sources */, 61D3E0D2277B23F1008BE766 /* KronosInternetAddress.swift in Sources */, D2553826288F0B1A00727FAD /* BatteryStatusPublisher.swift in Sources */, 61D3E0D5277B23F1008BE766 /* KronosNTPPacket.swift in Sources */, @@ -7286,6 +7295,7 @@ 61B8BA91281812F60068AFF4 /* KronosInternetAddressTests.swift in Sources */, 614798962A459AA80095CB02 /* DDTraceTests.swift in Sources */, D25085102976E30000E931C3 /* DatadogRemoteFeatureMock.swift in Sources */, + A7C816AB2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift in Sources */, 61A1A44929643254007909E7 /* DatadogCoreProxy.swift in Sources */, D2A1EE3B287EECC000D28DFB /* CarrierInfoPublisherTests.swift in Sources */, D22743D829DEB8B4001A7EF9 /* VitalInfoTests.swift in Sources */, @@ -8294,6 +8304,7 @@ D20605AA2874C1CD0047275C /* NetworkConnectionInfoPublisher.swift in Sources */, D29CDD3328211A2200F7DAA5 /* DataBlock.swift in Sources */, D2612F48290197C700509B7D /* LaunchTimePublisher.swift in Sources */, + A70A82662A935F210072F5DC /* BackgroundTaskCoordinator.swift in Sources */, D20605CA2875A83D0047275C /* ContextValueReader.swift in Sources */, D2A1EE24287740B500D28DFB /* ApplicationStatePublisher.swift in Sources */, D2CB6E2927C50EAE00A62B57 /* KronosInternetAddress.swift in Sources */, @@ -8388,6 +8399,7 @@ D24C9C7229A7D57A002057CF /* DirectoriesMock.swift in Sources */, 61DA8CB3286215DE0074A606 /* CryptographyTests.swift in Sources */, D2CB6F0427C520D400A62B57 /* DDTracerTests.swift in Sources */, + A7C816AC2A98CEBA00BF097B /* UIKitBackgroundTaskCoordinatorTests.swift in Sources */, D24C9C6129A7CB0C002057CF /* DatadogLogsFeatureTests.swift in Sources */, D29A9FCF29DDC4BC005C54A4 /* RUMFeatureMocks.swift in Sources */, D22743DE29DEB8B5001A7EF9 /* VitalInfoSamplerTests.swift in Sources */, diff --git a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift new file mode 100644 index 0000000000..23a10d2701 --- /dev/null +++ b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift @@ -0,0 +1,63 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation + +/// The `BackgroundTaskCoordinator` protocol provides an abstraction for managing background tasks and includes methods for registering and ending background tasks. +/// It serves as a useful abstraction for testing purposes as well as allows decoupling from UIKit in order to maintain Catalyst compliation. To abstract from UIKit, it leverages +/// the fact that UIBackgroundTaskIdentifier raw value is based on Int. +internal protocol BackgroundTaskCoordinator { + func beginBackgroundTask(expirationHandler handler: @escaping (() -> Void)) -> Int + func endBackgroundTaskIfActive(_ backgroundTaskIdentifier: Int) +} + +#if canImport(UIKit) +import UIKit + +internal protocol UIKitAppBackgroundTaskCoordinator { + func beginBackgroundTask(expirationHandler handler: (() -> Void)?) -> UIBackgroundTaskIdentifier + func endBackgroundTask(_ identifier: UIBackgroundTaskIdentifier) +} + +extension UIApplication: UIKitAppBackgroundTaskCoordinator {} + +/// Manages background tasks using UIKit. +/// This coordinator conforms to the `BackgroundTaskCoordinator` protocol and provides an implementation of managing background tasks using the UIKit framework. +/// It allows for registering and ending background tasks. +internal class UIKitBackgroundTaskCoordinator: BackgroundTaskCoordinator { + private let queue: DispatchQueue + private let app: UIKitAppBackgroundTaskCoordinator? + + internal init( + queue: DispatchQueue, + app: UIKitAppBackgroundTaskCoordinator? = UIApplication.dd.managedShared + ) { + self.queue = queue + self.app = app + } + + internal func beginBackgroundTask(expirationHandler handler: @escaping (() -> Void)) -> Int { + guard let app = app else { + return UIBackgroundTaskIdentifier.invalid.rawValue + } + return app.beginBackgroundTask(expirationHandler: { [weak self] in + self?.queue.async { + handler() + } + }).rawValue + } + + func endBackgroundTaskIfActive(_ backgroundTaskIdentifier: Int) { + let task = UIBackgroundTaskIdentifier(rawValue: backgroundTaskIdentifier) + guard task != .invalid else { + return + } + DispatchQueue.main.async { [app] in + app?.endBackgroundTask(task) + } + } +} +#endif diff --git a/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift b/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift index fbbcea7fd5..0a27965cb6 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift @@ -9,31 +9,30 @@ import DatadogInternal internal protocol Delay { var current: TimeInterval { get } - mutating func decrease() - mutating func increase() + func decrease() + func increase() } /// Mutable interval used for periodic data uploads. -internal struct DataUploadDelay: Delay { +internal class DataUploadDelay: Delay { private let minDelay: TimeInterval private let maxDelay: TimeInterval private let changeRate: Double - private var delay: TimeInterval + + var current: TimeInterval init(performance: UploadPerformancePreset) { self.minDelay = performance.minUploadDelay self.maxDelay = performance.maxUploadDelay self.changeRate = performance.uploadDelayChangeRate - self.delay = performance.initialUploadDelay + self.current = performance.initialUploadDelay } - var current: TimeInterval { delay } - - mutating func decrease() { - delay = max(minDelay, delay * (1.0 - changeRate)) + func decrease() { + current = max(minDelay, current * (1.0 - changeRate)) } - mutating func increase() { - delay = min(delay * (1.0 + changeRate), maxDelay) + func increase() { + current = min(current * (1.0 + changeRate), maxDelay) } } diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift index 8b51dbcc44..a7f3557dc3 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift @@ -27,12 +27,16 @@ internal class DataUploadWorker: DataUploadWorkerType { /// The core context provider private let contextProvider: DatadogContextProvider /// Delay used to schedule consecutive uploads. - private var delay: Delay + private let delay: Delay + /// Upload work scheduled by this worker. private var uploadWork: DispatchWorkItem? /// Telemetry interface. private let telemetry: Telemetry + private var backgroundTaskCoordinator: BackgroundTaskCoordinator? + private var taskID: Int? + init( queue: DispatchQueue, fileReader: Reader, @@ -41,13 +45,15 @@ internal class DataUploadWorker: DataUploadWorkerType { uploadConditions: DataUploadConditions, delay: Delay, featureName: String, - telemetry: Telemetry + telemetry: Telemetry, + backgroundTaskCoordinator: BackgroundTaskCoordinator? = nil ) { self.queue = queue self.fileReader = fileReader self.uploadConditions = uploadConditions self.dataUploader = dataUploader self.contextProvider = contextProvider + self.backgroundTaskCoordinator = backgroundTaskCoordinator self.delay = delay self.featureName = featureName self.telemetry = telemetry @@ -56,12 +62,22 @@ internal class DataUploadWorker: DataUploadWorkerType { guard let self = self else { return } - let context = contextProvider.read() let blockersForUpload = self.uploadConditions.blockersForUpload(with: context) let isSystemReady = blockersForUpload.isEmpty - let nextBatch = isSystemReady ? self.fileReader.readNextBatch() : nil + let batch = self.fileReader.readNextBatch() + let nextBatch = isSystemReady ? batch : nil if let batch = nextBatch { + if let taskID = taskID { + self.backgroundTaskCoordinator?.endBackgroundTaskIfActive(taskID) + self.taskID = nil + } + self.taskID = self.backgroundTaskCoordinator?.beginBackgroundTask { [backgroundTaskCoordinator, taskID] in + guard let taskID = taskID else { + return + } + backgroundTaskCoordinator?.endBackgroundTaskIfActive(taskID) + } DD.logger.debug("⏳ (\(self.featureName)) Uploading batch...") do { @@ -102,8 +118,11 @@ internal class DataUploadWorker: DataUploadWorkerType { DD.logger.debug("💡 (\(self.featureName)) No upload. Batch to upload: \(batchLabel), System conditions: \(blockersForUpload.description)") self.delay.increase() + if let taskID = taskID { + self.backgroundTaskCoordinator?.endBackgroundTaskIfActive(taskID) + self.taskID = nil + } } - self.scheduleNextUpload(after: self.delay.current) } diff --git a/DatadogCore/Sources/Core/Upload/FeatureUpload.swift b/DatadogCore/Sources/Core/Upload/FeatureUpload.swift index deac2225f2..3b0147e16a 100644 --- a/DatadogCore/Sources/Core/Upload/FeatureUpload.swift +++ b/DatadogCore/Sources/Core/Upload/FeatureUpload.swift @@ -31,6 +31,12 @@ internal struct FeatureUpload { requestBuilder: requestBuilder ) + #if canImport(UIKit) + let backgroundTaskCoordinator = UIKitBackgroundTaskCoordinator(queue: uploadQueue) + #else + let backgroundTaskCoordinator = nil + #endif + self.init( uploader: DataUploadWorker( queue: uploadQueue, @@ -40,7 +46,8 @@ internal struct FeatureUpload { uploadConditions: DataUploadConditions(), delay: DataUploadDelay(performance: performance), featureName: featureName, - telemetry: telemetry + telemetry: telemetry, + backgroundTaskCoordinator: backgroundTaskCoordinator ) ) } diff --git a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadDelayTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadDelayTests.swift index 97de679496..0f8bd29089 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadDelayTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadDelayTests.swift @@ -22,7 +22,7 @@ class DataUploadDelayTests: XCTestCase { } func testWhenDecreasing_itGoesDownToMinimumDelay() { - var delay = DataUploadDelay(performance: mockPerformance) + let delay = DataUploadDelay(performance: mockPerformance) var previousValue: TimeInterval = delay.current while previousValue > mockPerformance.minUploadDelay { @@ -41,7 +41,7 @@ class DataUploadDelayTests: XCTestCase { } func testWhenIncreasing_itClampsToMaximumDelay() { - var delay = DataUploadDelay(performance: mockPerformance) + let delay = DataUploadDelay(performance: mockPerformance) var previousValue: TimeInterval = delay.current while previousValue < mockPerformance.maxUploadDelay { diff --git a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift index 90fd4e726f..84175fc7c6 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift @@ -173,28 +173,23 @@ class DataUploadWorkerTests: XCTestCase { func testWhenThereIsNoBatch_thenIntervalIncreases() { let delayChangeExpectation = expectation(description: "Upload delay is increased") - let mockDelay = MockDelay { command in - if case .increase = command { - delayChangeExpectation.fulfill() - } else { - XCTFail("Wrong command is sent!") - } - } + let initialUploadDelay = 0.01 + let delay = DataUploadDelay( + performance: UploadPerformanceMock( + initialUploadDelay: initialUploadDelay, + minUploadDelay: 0, + maxUploadDelay: 1, + uploadDelayChangeRate: 0.01 + ) + ) // When XCTAssertEqual(try orchestrator.directory.files().count, 0) - let server = ServerMock(delivery: .success(response: .mockResponseWith(statusCode: 200))) - let httpClient = URLSessionClient(session: server.getInterceptedURLSession()) - - let dataUploader = DataUploader( - httpClient: httpClient, - requestBuilder: FeatureRequestBuilderMock() - ) let worker = DataUploadWorker( queue: uploaderQueue, fileReader: reader, - dataUploader: dataUploader, + dataUploader: DataUploaderMock(uploadStatus: .mockWith()), contextProvider: .mockAny(), uploadConditions: DataUploadConditions.neverUpload(), delay: mockDelay, @@ -203,35 +198,34 @@ class DataUploadWorkerTests: XCTestCase { ) // Then - server.waitFor(requestsCompletion: 0) - waitForExpectations(timeout: 1, handler: nil) + wait(until: { [uploaderQueue] in + uploaderQueue.sync { + delay.current > initialUploadDelay + } + }, andThenFulfill: delayChangeExpectation) + wait(for: [delayChangeExpectation]) worker.cancelSynchronously() } func testWhenBatchFails_thenIntervalIncreases() { let delayChangeExpectation = expectation(description: "Upload delay is increased") - let mockDelay = MockDelay { command in - if case .increase = command { - delayChangeExpectation.fulfill() - } else { - XCTFail("Wrong command is sent!") - } - } + let initialUploadDelay = 0.01 + let delay = DataUploadDelay( + performance: UploadPerformanceMock( + initialUploadDelay: initialUploadDelay, + minUploadDelay: 0, + maxUploadDelay: 1, + uploadDelayChangeRate: 0.01 + ) + ) // When writer.write(value: ["k1": "v1"]) - let server = ServerMock(delivery: .success(response: .mockResponseWith(statusCode: 500))) - let httpClient = URLSessionClient(session: server.getInterceptedURLSession()) - - let dataUploader = DataUploader( - httpClient: httpClient, - requestBuilder: FeatureRequestBuilderMock() - ) let worker = DataUploadWorker( queue: uploaderQueue, fileReader: reader, - dataUploader: dataUploader, + dataUploader: DataUploaderMock(uploadStatus: .mockWith(needsRetry: true)), contextProvider: .mockAny(), uploadConditions: DataUploadConditions.alwaysUpload(), delay: mockDelay, @@ -240,35 +234,33 @@ class DataUploadWorkerTests: XCTestCase { ) // Then - server.waitFor(requestsCompletion: 1) - waitForExpectations(timeout: 1, handler: nil) + wait(until: { [uploaderQueue] in + uploaderQueue.sync { + delay.current > initialUploadDelay + } + }, andThenFulfill: delayChangeExpectation) + wait(for: [delayChangeExpectation]) worker.cancelSynchronously() } func testWhenBatchSucceeds_thenIntervalDecreases() { let delayChangeExpectation = expectation(description: "Upload delay is decreased") - let mockDelay = MockDelay { command in - if case .decrease = command { - delayChangeExpectation.fulfill() - } else { - XCTFail("Wrong command is sent!") - } - } - + let initialUploadDelay = 0.05 + let delay = DataUploadDelay( + performance: UploadPerformanceMock( + initialUploadDelay: initialUploadDelay, + minUploadDelay: 0, + maxUploadDelay: 1, + uploadDelayChangeRate: 0.01 + ) + ) // When writer.write(value: ["k1": "v1"]) - let server = ServerMock(delivery: .success(response: .mockResponseWith(statusCode: 200))) - let httpClient = URLSessionClient(session: server.getInterceptedURLSession()) - - let dataUploader = DataUploader( - httpClient: httpClient, - requestBuilder: FeatureRequestBuilderMock() - ) let worker = DataUploadWorker( queue: uploaderQueue, fileReader: reader, - dataUploader: dataUploader, + dataUploader: DataUploaderMock(uploadStatus: .mockWith(needsRetry: false)), contextProvider: .mockAny(), uploadConditions: DataUploadConditions.alwaysUpload(), delay: mockDelay, @@ -277,8 +269,12 @@ class DataUploadWorkerTests: XCTestCase { ) // Then - server.waitFor(requestsCompletion: 1) - waitForExpectations(timeout: 2, handler: nil) + wait(until: { [uploaderQueue] in + uploaderQueue.sync { + delay.current < initialUploadDelay + } + }, andThenFulfill: delayChangeExpectation) + wait(for: [delayChangeExpectation]) worker.cancelSynchronously() } @@ -535,23 +531,33 @@ class DataUploadWorkerTests: XCTestCase { worker.cancelSynchronously() } -} - -struct MockDelay: Delay { - enum Command { - case increase, decrease - } - var callback: ((Command) -> Void)? - let current: TimeInterval = 0.1 + func testItTriggersBackgroundTaskRegistration() { + let expectTaskRegistered = expectation(description: "task should be registered") + let expectTaskEnded = expectation(description: "task should be ended") + let backgroundTaskCoordinator = SpyBackgroundTaskCoordinator( + beginBackgroundTaskCalled: { + expectTaskRegistered.fulfill() + }, endBackgroundTaskCalled: { + expectTaskEnded.fulfill() + } + ) + let worker = DataUploadWorker( + queue: uploaderQueue, + fileReader: reader, + dataUploader: DataUploaderMock(uploadStatus: .mockWith()), + contextProvider: .mockAny(), + uploadConditions: .alwaysUpload(), + delay: DataUploadDelay(performance: UploadPerformanceMock.veryQuick), + featureName: .mockAny(), + backgroundTaskCoordinator: backgroundTaskCoordinator + ) + writer.write(value: ["k1": "v1"]) - mutating func decrease() { - callback?(.decrease) - callback = nil - } - mutating func increase() { - callback?(.increase) - callback = nil + // Then + withExtendedLifetime(worker) { + wait(for: [expectTaskRegistered, expectTaskEnded]) + } } } @@ -564,3 +570,25 @@ private extension DataUploadConditions { return DataUploadConditions(minBatteryLevel: 1) } } + +private class SpyBackgroundTaskCoordinator: BackgroundTaskCoordinator { + private let beginBackgroundTaskCalled: () -> Void + private let endBackgroundTaskCalled: () -> Void + + init( + beginBackgroundTaskCalled: @escaping () -> Void, + endBackgroundTaskCalled: @escaping () -> Void + ) { + self.beginBackgroundTaskCalled = beginBackgroundTaskCalled + self.endBackgroundTaskCalled = endBackgroundTaskCalled + } + + func beginBackgroundTask(expirationHandler handler: @escaping (() -> Void)) -> Int { + beginBackgroundTaskCalled() + return Int.mockRandom() + } + + func endBackgroundTaskIfActive(_ backgroundTaskIdentifier: Int) { + endBackgroundTaskCalled() + } +} diff --git a/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift new file mode 100644 index 0000000000..7a01cfe281 --- /dev/null +++ b/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift @@ -0,0 +1,83 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import XCTest +import DatadogInternal +@testable import DatadogCore + +class UIKitBackgroundTaskCoordinatorTests: XCTestCase { + var appSpy: AppSpy? + var coordinator: UIKitBackgroundTaskCoordinator? + + override func setUp() { + super.setUp() + appSpy = AppSpy() + coordinator = UIKitBackgroundTaskCoordinator( + queue: DispatchQueue.main, + app: appSpy + ) + } + + func testBeginBackgroundTask() { + let backgroundTaskIdentifier = coordinator?.beginBackgroundTask { } + + XCTAssertEqual(backgroundTaskIdentifier, 1) + XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, true) + XCTAssertEqual(appSpy?.endBackgroundTaskCalled, false) + } + + func testEndBackgroundTask() throws { + let backgroundTaskIdentifier = try XCTUnwrap(coordinator?.beginBackgroundTask(expirationHandler: { })) + coordinator?.endBackgroundTaskIfActive(backgroundTaskIdentifier) + + XCTAssertEqual(backgroundTaskIdentifier, 1) + XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, true) + XCTAssertEqual(appSpy?.endBackgroundTaskCalled, true) + } + + func testHanderFromTheSameQueue() { + let expectHandlerCalled = expectation(description: "handler called") + _ = coordinator?.beginBackgroundTask { + XCTAssertEqual(Thread.current, Thread.main) + expectHandlerCalled.fulfill() + } + appSpy?.fireHandler(from: .main) + wait(for: [expectHandlerCalled]) + } + + func testHandlerFromDifferentQueue() { + let expectHandlerCalled = expectation(description: "handler called") + _ = coordinator?.beginBackgroundTask { + XCTAssertEqual(Thread.current, Thread.main) + expectHandlerCalled.fulfill() + } + appSpy?.fireHandler(from: .global(qos: .background)) + wait(for: [expectHandlerCalled]) + } +} + +class AppSpy: UIKitAppBackgroundTaskCoordinator { + var beginBackgroundTaskCalled = false + var endBackgroundTaskCalled = false + + private var handler: (() -> Void)? = nil + + func beginBackgroundTask(expirationHandler handler: (() -> Void)?) -> UIBackgroundTaskIdentifier { + self.handler = handler + beginBackgroundTaskCalled = true + return UIBackgroundTaskIdentifier(rawValue: 1) + } + + func endBackgroundTask(_ identifier: UIBackgroundTaskIdentifier) { + endBackgroundTaskCalled = true + } + + func fireHandler(from: DispatchQueue) { + from.async { [handler] in + handler?() + } + } +} From 7f3af6e01a9b2cf1ad9e8a52d3c8a39030d99a3e Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Mon, 28 Aug 2023 12:08:03 +0100 Subject: [PATCH 02/15] REPLAY-1963 Remove queue --- .../Sources/Core/Upload/BackgroundTaskCoordinator.swift | 9 +-------- DatadogCore/Sources/Core/Upload/FeatureUpload.swift | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift index 23a10d2701..90516414ba 100644 --- a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift +++ b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift @@ -28,14 +28,11 @@ extension UIApplication: UIKitAppBackgroundTaskCoordinator {} /// This coordinator conforms to the `BackgroundTaskCoordinator` protocol and provides an implementation of managing background tasks using the UIKit framework. /// It allows for registering and ending background tasks. internal class UIKitBackgroundTaskCoordinator: BackgroundTaskCoordinator { - private let queue: DispatchQueue private let app: UIKitAppBackgroundTaskCoordinator? internal init( - queue: DispatchQueue, app: UIKitAppBackgroundTaskCoordinator? = UIApplication.dd.managedShared ) { - self.queue = queue self.app = app } @@ -43,11 +40,7 @@ internal class UIKitBackgroundTaskCoordinator: BackgroundTaskCoordinator { guard let app = app else { return UIBackgroundTaskIdentifier.invalid.rawValue } - return app.beginBackgroundTask(expirationHandler: { [weak self] in - self?.queue.async { - handler() - } - }).rawValue + return app.beginBackgroundTask(expirationHandler: handler).rawValue } func endBackgroundTaskIfActive(_ backgroundTaskIdentifier: Int) { diff --git a/DatadogCore/Sources/Core/Upload/FeatureUpload.swift b/DatadogCore/Sources/Core/Upload/FeatureUpload.swift index 3b0147e16a..bec9ad15aa 100644 --- a/DatadogCore/Sources/Core/Upload/FeatureUpload.swift +++ b/DatadogCore/Sources/Core/Upload/FeatureUpload.swift @@ -32,7 +32,7 @@ internal struct FeatureUpload { ) #if canImport(UIKit) - let backgroundTaskCoordinator = UIKitBackgroundTaskCoordinator(queue: uploadQueue) + let backgroundTaskCoordinator = UIKitBackgroundTaskCoordinator() #else let backgroundTaskCoordinator = nil #endif From 6e824aa17ecae6a5e3a0acc8c2618aa1ce6db800 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Wed, 30 Aug 2023 13:22:16 +0100 Subject: [PATCH 03/15] REPLAY-1963 Add opt in configuration for background tasks --- DatadogCore/Sources/Core/DatadogCore.swift | 8 +++++++- .../Sources/Core/Upload/BackgroundTaskCoordinator.swift | 4 +--- DatadogCore/Sources/Core/Upload/FeatureUpload.swift | 5 ++++- DatadogCore/Sources/Datadog.swift | 9 +++++++-- DatadogInternal/Sources/DatadogFeature.swift | 2 ++ 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/DatadogCore/Sources/Core/DatadogCore.swift b/DatadogCore/Sources/Core/DatadogCore.swift index f0e0b8e9f2..9bb1adf6e5 100644 --- a/DatadogCore/Sources/Core/DatadogCore.swift +++ b/DatadogCore/Sources/Core/DatadogCore.swift @@ -65,6 +65,8 @@ internal final class DatadogCore { /// The core context provider. internal let contextProvider: DatadogContextProvider + internal let backgroundTasksEnabled: Bool + /// Creates a core instance. /// /// - Parameters: @@ -84,7 +86,8 @@ internal final class DatadogCore { httpClient: HTTPClient, encryption: DataEncryption?, contextProvider: DatadogContextProvider, - applicationVersion: String + applicationVersion: String, + backgroundTasksEnabled: Bool ) { self.directory = directory self.dateProvider = dateProvider @@ -92,6 +95,8 @@ internal final class DatadogCore { self.httpClient = httpClient self.encryption = encryption self.contextProvider = contextProvider + self.backgroundTasksEnabled = backgroundTasksEnabled + self.applicationVersionPublisher = ApplicationVersionPublisher(version: applicationVersion) self.consentPublisher = TrackingConsentPublisher(consent: initialConsent) @@ -241,6 +246,7 @@ extension DatadogCore: DatadogCoreProtocol { requestBuilder: feature.requestBuilder, httpClient: httpClient, performance: performancePreset, + backgroundTasksEnabled: backgroundTasksEnabled, telemetry: telemetry ) diff --git a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift index 90516414ba..a5c3c2ed23 100644 --- a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift +++ b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift @@ -48,9 +48,7 @@ internal class UIKitBackgroundTaskCoordinator: BackgroundTaskCoordinator { guard task != .invalid else { return } - DispatchQueue.main.async { [app] in - app?.endBackgroundTask(task) - } + app?.endBackgroundTask(task) } } #endif diff --git a/DatadogCore/Sources/Core/Upload/FeatureUpload.swift b/DatadogCore/Sources/Core/Upload/FeatureUpload.swift index bec9ad15aa..ef6b527ccb 100644 --- a/DatadogCore/Sources/Core/Upload/FeatureUpload.swift +++ b/DatadogCore/Sources/Core/Upload/FeatureUpload.swift @@ -18,6 +18,7 @@ internal struct FeatureUpload { requestBuilder: FeatureRequestBuilder, httpClient: HTTPClient, performance: PerformancePreset, + backgroundTasksEnabled: Bool, telemetry: Telemetry ) { let uploadQueue = DispatchQueue( @@ -32,7 +33,9 @@ internal struct FeatureUpload { ) #if canImport(UIKit) - let backgroundTaskCoordinator = UIKitBackgroundTaskCoordinator() + let backgroundTaskCoordinator = backgroundTasksEnabled + ? UIKitBackgroundTaskCoordinator() + : nil #else let backgroundTaskCoordinator = nil #endif diff --git a/DatadogCore/Sources/Datadog.swift b/DatadogCore/Sources/Datadog.swift index ebc1258173..0a69418caf 100644 --- a/DatadogCore/Sources/Datadog.swift +++ b/DatadogCore/Sources/Datadog.swift @@ -105,6 +105,8 @@ public struct Datadog { /// The bundle object that contains the current executable. public var bundle: Bundle + public var backgroundTasksEnabled: Bool + /// Creates a Datadog SDK Configuration object. /// /// - Parameters: @@ -150,7 +152,8 @@ public struct Datadog { uploadFrequency: UploadFrequency = .average, proxyConfiguration: [AnyHashable: Any]? = nil, encryption: DataEncryption? = nil, - serverDateProvider: ServerDateProvider? = nil + serverDateProvider: ServerDateProvider? = nil, + backgroundTasksEnabled: Bool = false ) { self.clientToken = clientToken self.env = env @@ -162,6 +165,7 @@ public struct Datadog { self.proxyConfiguration = proxyConfiguration self.encryption = encryption self.serverDateProvider = serverDateProvider ?? DatadogNTPDateProvider() + self.backgroundTasksEnabled = backgroundTasksEnabled } // MARK: - Internal @@ -389,7 +393,8 @@ public struct Datadog { dateProvider: configuration.dateProvider, serverDateProvider: configuration.serverDateProvider ), - applicationVersion: applicationVersion + applicationVersion: applicationVersion, + backgroundTasksEnabled: configuration.backgroundTasksEnabled ) core.telemetry.configuration( diff --git a/DatadogInternal/Sources/DatadogFeature.swift b/DatadogInternal/Sources/DatadogFeature.swift index fb6b262ecc..1fbd31029a 100644 --- a/DatadogInternal/Sources/DatadogFeature.swift +++ b/DatadogInternal/Sources/DatadogFeature.swift @@ -53,6 +53,8 @@ public protocol DatadogRemoteFeature: DatadogFeature { /// A Feature should use this interface for creating requests that needs be sent to its Datadog Intake. /// The request will be transported by `DatadogCore`. var requestBuilder: FeatureRequestBuilder { get } + + var backgroundTasksEnabled: Bool { get } } extension DatadogFeature { From 820992e00ed0f42607b4fbec94310556130fcc59 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Tue, 5 Sep 2023 12:05:46 +0100 Subject: [PATCH 04/15] REPLAY-1963 Simplify background task solution --- DatadogCore/Sources/Core/DatadogCore.swift | 1 - .../Upload/BackgroundTaskCoordinator.swift | 32 ++++++++------ .../Core/Upload/DataUploadWorker.swift | 18 ++------ .../Core/Upload/DataUploadWorkerTests.swift | 14 +++--- .../UIKitBackgroundTaskCoordinatorTests.swift | 44 +++++++------------ .../Context/FeatureContextTests.swift | 3 +- .../DatadogCore/DatadogCoreTests.swift | 18 +++++--- .../Logs/DatadogLogsFeatureTests.swift | 7 ++- .../DatadogInternal/DatadogCoreProxy.swift | 3 +- .../Tests/Datadog/RUM/RUMFeatureTests.swift | 7 ++- .../Tracing/DatadogTraceFeatureTests.swift | 7 ++- DatadogInternal/Sources/DatadogFeature.swift | 2 - 12 files changed, 75 insertions(+), 81 deletions(-) diff --git a/DatadogCore/Sources/Core/DatadogCore.swift b/DatadogCore/Sources/Core/DatadogCore.swift index 9bb1adf6e5..fccc4062a6 100644 --- a/DatadogCore/Sources/Core/DatadogCore.swift +++ b/DatadogCore/Sources/Core/DatadogCore.swift @@ -96,7 +96,6 @@ internal final class DatadogCore { self.encryption = encryption self.contextProvider = contextProvider self.backgroundTasksEnabled = backgroundTasksEnabled - self.applicationVersionPublisher = ApplicationVersionPublisher(version: applicationVersion) self.consentPublisher = TrackingConsentPublisher(consent: initialConsent) diff --git a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift index a5c3c2ed23..f997ed074f 100644 --- a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift +++ b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift @@ -10,12 +10,13 @@ import Foundation /// It serves as a useful abstraction for testing purposes as well as allows decoupling from UIKit in order to maintain Catalyst compliation. To abstract from UIKit, it leverages /// the fact that UIBackgroundTaskIdentifier raw value is based on Int. internal protocol BackgroundTaskCoordinator { - func beginBackgroundTask(expirationHandler handler: @escaping (() -> Void)) -> Int - func endBackgroundTaskIfActive(_ backgroundTaskIdentifier: Int) + func beginBackgroundTask() + func endCurrentBackgroundTaskIfActive() } #if canImport(UIKit) import UIKit +import DatadogInternal internal protocol UIKitAppBackgroundTaskCoordinator { func beginBackgroundTask(expirationHandler handler: (() -> Void)?) -> UIBackgroundTaskIdentifier @@ -24,31 +25,36 @@ internal protocol UIKitAppBackgroundTaskCoordinator { extension UIApplication: UIKitAppBackgroundTaskCoordinator {} -/// Manages background tasks using UIKit. -/// This coordinator conforms to the `BackgroundTaskCoordinator` protocol and provides an implementation of managing background tasks using the UIKit framework. -/// It allows for registering and ending background tasks. internal class UIKitBackgroundTaskCoordinator: BackgroundTaskCoordinator { private let app: UIKitAppBackgroundTaskCoordinator? + @ReadWriteLock + private var currentTaskId: UIBackgroundTaskIdentifier? + internal init( app: UIKitAppBackgroundTaskCoordinator? = UIApplication.dd.managedShared ) { self.app = app } - internal func beginBackgroundTask(expirationHandler handler: @escaping (() -> Void)) -> Int { - guard let app = app else { - return UIBackgroundTaskIdentifier.invalid.rawValue + internal func beginBackgroundTask() { + endCurrentBackgroundTaskIfActive() + currentTaskId = app?.beginBackgroundTask { [weak self] in + guard let self = self else { + return + } + self.endCurrentBackgroundTaskIfActive() } - return app.beginBackgroundTask(expirationHandler: handler).rawValue } - func endBackgroundTaskIfActive(_ backgroundTaskIdentifier: Int) { - let task = UIBackgroundTaskIdentifier(rawValue: backgroundTaskIdentifier) - guard task != .invalid else { + func endCurrentBackgroundTaskIfActive() { + guard let currentTaskId = currentTaskId else { return } - app?.endBackgroundTask(task) + if currentTaskId != .invalid { + app?.endBackgroundTask(currentTaskId) + } + self.currentTaskId = nil } } #endif diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift index a7f3557dc3..81cfe9c07d 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift @@ -34,8 +34,8 @@ internal class DataUploadWorker: DataUploadWorkerType { /// Telemetry interface. private let telemetry: Telemetry + /// Background task coordinator responsible for registering and ending background tasks for UIKit targets. private var backgroundTaskCoordinator: BackgroundTaskCoordinator? - private var taskID: Int? init( queue: DispatchQueue, @@ -68,16 +68,7 @@ internal class DataUploadWorker: DataUploadWorkerType { let batch = self.fileReader.readNextBatch() let nextBatch = isSystemReady ? batch : nil if let batch = nextBatch { - if let taskID = taskID { - self.backgroundTaskCoordinator?.endBackgroundTaskIfActive(taskID) - self.taskID = nil - } - self.taskID = self.backgroundTaskCoordinator?.beginBackgroundTask { [backgroundTaskCoordinator, taskID] in - guard let taskID = taskID else { - return - } - backgroundTaskCoordinator?.endBackgroundTaskIfActive(taskID) - } + self.backgroundTaskCoordinator?.beginBackgroundTask() DD.logger.debug("⏳ (\(self.featureName)) Uploading batch...") do { @@ -118,10 +109,7 @@ internal class DataUploadWorker: DataUploadWorkerType { DD.logger.debug("💡 (\(self.featureName)) No upload. Batch to upload: \(batchLabel), System conditions: \(blockersForUpload.description)") self.delay.increase() - if let taskID = taskID { - self.backgroundTaskCoordinator?.endBackgroundTaskIfActive(taskID) - self.taskID = nil - } + self.backgroundTaskCoordinator?.endCurrentBackgroundTaskIfActive() } self.scheduleNextUpload(after: self.delay.current) } diff --git a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift index 84175fc7c6..83a13564b8 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift @@ -192,7 +192,7 @@ class DataUploadWorkerTests: XCTestCase { dataUploader: DataUploaderMock(uploadStatus: .mockWith()), contextProvider: .mockAny(), uploadConditions: DataUploadConditions.neverUpload(), - delay: mockDelay, + delay: delay, featureName: .mockAny(), telemetry: NOPTelemetry() ) @@ -228,7 +228,7 @@ class DataUploadWorkerTests: XCTestCase { dataUploader: DataUploaderMock(uploadStatus: .mockWith(needsRetry: true)), contextProvider: .mockAny(), uploadConditions: DataUploadConditions.alwaysUpload(), - delay: mockDelay, + delay: delay, featureName: .mockAny(), telemetry: NOPTelemetry() ) @@ -263,7 +263,7 @@ class DataUploadWorkerTests: XCTestCase { dataUploader: DataUploaderMock(uploadStatus: .mockWith(needsRetry: false)), contextProvider: .mockAny(), uploadConditions: DataUploadConditions.alwaysUpload(), - delay: mockDelay, + delay: delay, featureName: .mockAny(), telemetry: NOPTelemetry() ) @@ -480,7 +480,7 @@ class DataUploadWorkerTests: XCTestCase { dataUploader: dataUploader, contextProvider: .mockAny(), uploadConditions: DataUploadConditions.neverUpload(), - delay: MockDelay(), + delay: DataUploadDelay(performance: UploadPerformanceMock.veryQuick), featureName: .mockAny(), telemetry: NOPTelemetry() ) @@ -550,6 +550,7 @@ class DataUploadWorkerTests: XCTestCase { uploadConditions: .alwaysUpload(), delay: DataUploadDelay(performance: UploadPerformanceMock.veryQuick), featureName: .mockAny(), + telemetry: NOPTelemetry(), backgroundTaskCoordinator: backgroundTaskCoordinator ) writer.write(value: ["k1": "v1"]) @@ -583,12 +584,11 @@ private class SpyBackgroundTaskCoordinator: BackgroundTaskCoordinator { self.endBackgroundTaskCalled = endBackgroundTaskCalled } - func beginBackgroundTask(expirationHandler handler: @escaping (() -> Void)) -> Int { + func beginBackgroundTask() { beginBackgroundTaskCalled() - return Int.mockRandom() } - func endBackgroundTaskIfActive(_ backgroundTaskIdentifier: Int) { + func endCurrentBackgroundTaskIfActive() { endBackgroundTaskCalled() } } diff --git a/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift index 7a01cfe281..b638579339 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift @@ -16,46 +16,38 @@ class UIKitBackgroundTaskCoordinatorTests: XCTestCase { super.setUp() appSpy = AppSpy() coordinator = UIKitBackgroundTaskCoordinator( - queue: DispatchQueue.main, app: appSpy ) } func testBeginBackgroundTask() { - let backgroundTaskIdentifier = coordinator?.beginBackgroundTask { } + coordinator?.beginBackgroundTask() - XCTAssertEqual(backgroundTaskIdentifier, 1) XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, true) XCTAssertEqual(appSpy?.endBackgroundTaskCalled, false) } func testEndBackgroundTask() throws { - let backgroundTaskIdentifier = try XCTUnwrap(coordinator?.beginBackgroundTask(expirationHandler: { })) - coordinator?.endBackgroundTaskIfActive(backgroundTaskIdentifier) + coordinator?.beginBackgroundTask() + coordinator?.endCurrentBackgroundTaskIfActive() - XCTAssertEqual(backgroundTaskIdentifier, 1) XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, true) XCTAssertEqual(appSpy?.endBackgroundTaskCalled, true) } - func testHanderFromTheSameQueue() { - let expectHandlerCalled = expectation(description: "handler called") - _ = coordinator?.beginBackgroundTask { - XCTAssertEqual(Thread.current, Thread.main) - expectHandlerCalled.fulfill() - } - appSpy?.fireHandler(from: .main) - wait(for: [expectHandlerCalled]) + func testEndBackgroundTaskNotCalledWhenNotBegan() throws { + coordinator?.endCurrentBackgroundTaskIfActive() + + XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, false) + XCTAssertEqual(appSpy?.endBackgroundTaskCalled, false) } - func testHandlerFromDifferentQueue() { - let expectHandlerCalled = expectation(description: "handler called") - _ = coordinator?.beginBackgroundTask { - XCTAssertEqual(Thread.current, Thread.main) - expectHandlerCalled.fulfill() - } - appSpy?.fireHandler(from: .global(qos: .background)) - wait(for: [expectHandlerCalled]) + func testBeginEndsPreviousTask() throws { + coordinator?.beginBackgroundTask() + coordinator?.beginBackgroundTask() + + XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, true) + XCTAssertEqual(appSpy?.endBackgroundTaskCalled, true) } } @@ -63,7 +55,7 @@ class AppSpy: UIKitAppBackgroundTaskCoordinator { var beginBackgroundTaskCalled = false var endBackgroundTaskCalled = false - private var handler: (() -> Void)? = nil + var handler: (() -> Void)? = nil func beginBackgroundTask(expirationHandler handler: (() -> Void)?) -> UIBackgroundTaskIdentifier { self.handler = handler @@ -74,10 +66,4 @@ class AppSpy: UIKitAppBackgroundTaskCoordinator { func endBackgroundTask(_ identifier: UIBackgroundTaskIdentifier) { endBackgroundTaskCalled = true } - - func fireHandler(from: DispatchQueue) { - from.async { [handler] in - handler?() - } - } } diff --git a/DatadogCore/Tests/Datadog/DatadogCore/Context/FeatureContextTests.swift b/DatadogCore/Tests/Datadog/DatadogCore/Context/FeatureContextTests.swift index 8d4ff45a63..5ac784a41a 100644 --- a/DatadogCore/Tests/Datadog/DatadogCore/Context/FeatureContextTests.swift +++ b/DatadogCore/Tests/Datadog/DatadogCore/Context/FeatureContextTests.swift @@ -20,7 +20,8 @@ class FeatureContextTests: XCTestCase { httpClient: HTTPClientMock(), encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { temporaryCoreDirectory.delete() } diff --git a/DatadogCore/Tests/Datadog/DatadogCore/DatadogCoreTests.swift b/DatadogCore/Tests/Datadog/DatadogCore/DatadogCoreTests.swift index e8e30038bb..cadc43a892 100644 --- a/DatadogCore/Tests/Datadog/DatadogCore/DatadogCoreTests.swift +++ b/DatadogCore/Tests/Datadog/DatadogCore/DatadogCoreTests.swift @@ -42,7 +42,8 @@ class DatadogCoreTests: XCTestCase { httpClient: HTTPClientMock(), encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core.flushAndTearDown() } @@ -87,7 +88,8 @@ class DatadogCoreTests: XCTestCase { httpClient: HTTPClientMock(), encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core.flushAndTearDown() } @@ -140,7 +142,8 @@ class DatadogCoreTests: XCTestCase { httpClient: HTTPClientMock(), encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core.flushAndTearDown() } @@ -190,7 +193,8 @@ class DatadogCoreTests: XCTestCase { httpClient: HTTPClientMock(), encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) let core2 = DatadogCore( directory: temporaryCoreDirectory, @@ -200,7 +204,8 @@ class DatadogCoreTests: XCTestCase { httpClient: HTTPClientMock(), encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core1.flushAndTearDown() @@ -245,7 +250,8 @@ class DatadogCoreTests: XCTestCase { httpClient: HTTPClientMock(), encryption: nil, contextProvider: contextProvider, - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core.flushAndTearDown() } try core.register(feature: FeatureMock()) diff --git a/DatadogCore/Tests/Datadog/Logs/DatadogLogsFeatureTests.swift b/DatadogCore/Tests/Datadog/Logs/DatadogLogsFeatureTests.swift index 1d43c9f3a6..81584ddd3d 100644 --- a/DatadogCore/Tests/Datadog/Logs/DatadogLogsFeatureTests.swift +++ b/DatadogCore/Tests/Datadog/Logs/DatadogLogsFeatureTests.swift @@ -36,6 +36,7 @@ class DatadogLogsFeatureTests: XCTestCase { let randomDeviceOSName: String = .mockRandom() let randomDeviceOSVersion: String = .mockRandom() let randomEncryption: DataEncryption? = Bool.random() ? DataEncryptionMock() : nil + let randomBackgroundTasksEnabled: Bool = .mockRandom() let server = ServerMock(delivery: .success(response: .mockResponseWith(statusCode: 200))) let httpClient = URLSessionClient(session: server.getInterceptedURLSession()) @@ -65,7 +66,8 @@ class DatadogLogsFeatureTests: XCTestCase { ) ) ), - applicationVersion: randomApplicationVersion + applicationVersion: randomApplicationVersion, + backgroundTasksEnabled: randomBackgroundTasksEnabled ) defer { core.flushAndTearDown() } @@ -126,7 +128,8 @@ class DatadogLogsFeatureTests: XCTestCase { httpClient: httpClient, encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core.flushAndTearDown() } diff --git a/DatadogCore/Tests/Datadog/Mocks/DatadogInternal/DatadogCoreProxy.swift b/DatadogCore/Tests/Datadog/Mocks/DatadogInternal/DatadogCoreProxy.swift index 05904ce94c..561c046d1b 100644 --- a/DatadogCore/Tests/Datadog/Mocks/DatadogInternal/DatadogCoreProxy.swift +++ b/DatadogCore/Tests/Datadog/Mocks/DatadogInternal/DatadogCoreProxy.swift @@ -45,7 +45,8 @@ internal class DatadogCoreProxy: DatadogCoreProtocol { httpClient: HTTPClientMock(), encryption: nil, contextProvider: DatadogContextProvider(context: context), - applicationVersion: context.version + applicationVersion: context.version, + backgroundTasksEnabled: .mockAny() ) // override the message-bus's core instance diff --git a/DatadogCore/Tests/Datadog/RUM/RUMFeatureTests.swift b/DatadogCore/Tests/Datadog/RUM/RUMFeatureTests.swift index cd3ab7791a..80d44407f1 100644 --- a/DatadogCore/Tests/Datadog/RUM/RUMFeatureTests.swift +++ b/DatadogCore/Tests/Datadog/RUM/RUMFeatureTests.swift @@ -38,6 +38,7 @@ class RUMFeatureTests: XCTestCase { let randomDeviceOSName: String = .mockRandom() let randomDeviceOSVersion: String = .mockRandom() let randomEncryption: DataEncryption? = Bool.random() ? DataEncryptionMock() : nil + let randomBackgroundTasksEnabled: Bool = .mockRandom() let server = ServerMock(delivery: .success(response: .mockResponseWith(statusCode: 200))) let httpClient = URLSessionClient(session: server.getInterceptedURLSession()) @@ -69,7 +70,8 @@ class RUMFeatureTests: XCTestCase { ) ) ), - applicationVersion: randomApplicationVersion + applicationVersion: randomApplicationVersion, + backgroundTasksEnabled: randomBackgroundTasksEnabled ) defer { core.flushAndTearDown() } @@ -135,7 +137,8 @@ class RUMFeatureTests: XCTestCase { httpClient: httpClient, encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core.flushAndTearDown() } diff --git a/DatadogCore/Tests/Datadog/Tracing/DatadogTraceFeatureTests.swift b/DatadogCore/Tests/Datadog/Tracing/DatadogTraceFeatureTests.swift index 0e0f5f65ea..b27e5745e2 100644 --- a/DatadogCore/Tests/Datadog/Tracing/DatadogTraceFeatureTests.swift +++ b/DatadogCore/Tests/Datadog/Tracing/DatadogTraceFeatureTests.swift @@ -36,6 +36,7 @@ class DatadogTraceFeatureTests: XCTestCase { let randomDeviceOSName: String = .mockRandom() let randomDeviceOSVersion: String = .mockRandom() let randomEncryption: DataEncryption? = Bool.random() ? DataEncryptionMock() : nil + let randomBackgroundTasksEnabled: Bool = .mockRandom() let server = ServerMock(delivery: .success(response: .mockResponseWith(statusCode: 200))) let httpClient = URLSessionClient(session: server.getInterceptedURLSession()) @@ -65,7 +66,8 @@ class DatadogTraceFeatureTests: XCTestCase { ) ) ), - applicationVersion: randomApplicationVersion + applicationVersion: randomApplicationVersion, + backgroundTasksEnabled: randomBackgroundTasksEnabled ) defer { core.flushAndTearDown() } @@ -127,7 +129,8 @@ class DatadogTraceFeatureTests: XCTestCase { httpClient: httpClient, encryption: nil, contextProvider: .mockAny(), - applicationVersion: .mockAny() + applicationVersion: .mockAny(), + backgroundTasksEnabled: .mockAny() ) defer { core.flushAndTearDown() } diff --git a/DatadogInternal/Sources/DatadogFeature.swift b/DatadogInternal/Sources/DatadogFeature.swift index 1fbd31029a..fb6b262ecc 100644 --- a/DatadogInternal/Sources/DatadogFeature.swift +++ b/DatadogInternal/Sources/DatadogFeature.swift @@ -53,8 +53,6 @@ public protocol DatadogRemoteFeature: DatadogFeature { /// A Feature should use this interface for creating requests that needs be sent to its Datadog Intake. /// The request will be transported by `DatadogCore`. var requestBuilder: FeatureRequestBuilder { get } - - var backgroundTasksEnabled: Bool { get } } extension DatadogFeature { From de22b443efc28446e25d308200a83d5c53814525 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Tue, 5 Sep 2023 12:20:58 +0100 Subject: [PATCH 05/15] REPLAY-1963 PR fixes --- DatadogCore/Sources/Core/Upload/DataUploadWorker.swift | 5 +++-- DatadogCore/Sources/Datadog.swift | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift index 81cfe9c07d..0f611c162d 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift @@ -62,11 +62,11 @@ internal class DataUploadWorker: DataUploadWorkerType { guard let self = self else { return } + let context = contextProvider.read() let blockersForUpload = self.uploadConditions.blockersForUpload(with: context) let isSystemReady = blockersForUpload.isEmpty - let batch = self.fileReader.readNextBatch() - let nextBatch = isSystemReady ? batch : nil + let nextBatch = isSystemReady ? self.fileReader.readNextBatch() : nil if let batch = nextBatch { self.backgroundTaskCoordinator?.beginBackgroundTask() DD.logger.debug("⏳ (\(self.featureName)) Uploading batch...") @@ -111,6 +111,7 @@ internal class DataUploadWorker: DataUploadWorkerType { self.delay.increase() self.backgroundTaskCoordinator?.endCurrentBackgroundTaskIfActive() } + self.scheduleNextUpload(after: self.delay.current) } diff --git a/DatadogCore/Sources/Datadog.swift b/DatadogCore/Sources/Datadog.swift index 0a69418caf..fd00f12633 100644 --- a/DatadogCore/Sources/Datadog.swift +++ b/DatadogCore/Sources/Datadog.swift @@ -105,6 +105,7 @@ public struct Datadog { /// The bundle object that contains the current executable. public var bundle: Bundle + /// Flag that determines if UIKit's background tasks are utilized to perform uploads in background. public var backgroundTasksEnabled: Bool /// Creates a Datadog SDK Configuration object. From 046714810e7c28ed3dbd0ed9873aa22222d54fd1 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Tue, 5 Sep 2023 13:14:35 +0100 Subject: [PATCH 06/15] REPLAY-1963 Fix linting issues --- DatadogCore/Sources/Core/Upload/DataUploadWorker.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift index 0f611c162d..a93aa498a7 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift @@ -111,7 +111,7 @@ internal class DataUploadWorker: DataUploadWorkerType { self.delay.increase() self.backgroundTaskCoordinator?.endCurrentBackgroundTaskIfActive() } - + self.scheduleNextUpload(after: self.delay.current) } From c0201eb31c3ac3b646492db68e35d3ebb5994223 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Wed, 6 Sep 2023 10:20:00 +0100 Subject: [PATCH 07/15] REPLAY-1963 Add in background telemetry --- .../Sources/Core/Storage/FilesOrchestrator.swift | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift index 26f3597da0..5fa100c7a7 100644 --- a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift +++ b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift @@ -7,6 +7,10 @@ import Foundation import DatadogInternal +#if canImport(UIKit) +import UIKit +#endif + internal protocol FilesOrchestratorType: AnyObject { var performance: StoragePerformancePreset { get } @@ -244,6 +248,12 @@ internal class FilesOrchestrator: FilesOrchestratorType { let batchAge = dateProvider.now.timeIntervalSince(fileCreationDateFrom(fileName: batchFile.name)) + #if canImport(UIKit) + let inBackground = UIApplication.dd.managedShared?.applicationState == .background + #else + let inBackground = false + #endif + telemetry.metric( name: BatchDeletedMetric.name, attributes: [ @@ -256,7 +266,7 @@ internal class FilesOrchestrator: FilesOrchestratorType { BatchDeletedMetric.uploaderWindowKey: performance.uploaderWindow.toMilliseconds, BatchDeletedMetric.batchAgeKey: batchAge.toMilliseconds, BatchDeletedMetric.batchRemovalReasonKey: deletionReason.toString(), - BatchDeletedMetric.inBackgroundKey: false, + BatchDeletedMetric.inBackgroundKey: inBackground, ] ) } From eb72b75676cc87afb109b70834bfaf50f6c17495 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Wed, 6 Sep 2023 10:28:00 +0100 Subject: [PATCH 08/15] REPLAY-1963 Add documentation --- .../Sources/Core/Upload/BackgroundTaskCoordinator.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift index f997ed074f..28728d87fc 100644 --- a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift +++ b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift @@ -10,7 +10,9 @@ import Foundation /// It serves as a useful abstraction for testing purposes as well as allows decoupling from UIKit in order to maintain Catalyst compliation. To abstract from UIKit, it leverages /// the fact that UIBackgroundTaskIdentifier raw value is based on Int. internal protocol BackgroundTaskCoordinator { + /// Requests additional background execution time for the app. func beginBackgroundTask() + /// Marks the end of a specific long-running background task. func endCurrentBackgroundTaskIfActive() } @@ -18,6 +20,7 @@ internal protocol BackgroundTaskCoordinator { import UIKit import DatadogInternal +/// Bridge protocol that matches UIApplication's interface for background tasks. Allows easier testablity. internal protocol UIKitAppBackgroundTaskCoordinator { func beginBackgroundTask(expirationHandler handler: (() -> Void)?) -> UIBackgroundTaskIdentifier func endBackgroundTask(_ identifier: UIBackgroundTaskIdentifier) @@ -47,7 +50,7 @@ internal class UIKitBackgroundTaskCoordinator: BackgroundTaskCoordinator { } } - func endCurrentBackgroundTaskIfActive() { + internal func endCurrentBackgroundTaskIfActive() { guard let currentTaskId = currentTaskId else { return } From 78bd067bcf1b881e371b30bd32e5053f31d671f1 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Wed, 6 Sep 2023 12:25:57 +0100 Subject: [PATCH 09/15] REPLAY-1963 Proper solution for inBackground telemetry --- .../Core/Storage/FilesOrchestrator.swift | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift index 5fa100c7a7..bc3197e402 100644 --- a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift +++ b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift @@ -30,7 +30,6 @@ internal class FilesOrchestrator: FilesOrchestratorType { let dateProvider: DateProvider /// Performance rules for writing and reading files. let performance: StoragePerformancePreset - /// Name of the last file returned by `getWritableFile()`. private var lastWritableFileName: String? = nil /// Tracks number of times the last file was returned from `getWritableFile(writeSize:)`. @@ -42,6 +41,15 @@ internal class FilesOrchestrator: FilesOrchestratorType { /// Telemetry interface. let telemetry: Telemetry + #if canImport(UIKit) + /// Application state publisher for enriching telemetry. + let applicationStatePublisher: ApplicationStatePublisher = .init() + var subscription: ContextValueSubscription? + #endif + /// Flag indicating if the application is in background state. + @ReadWriteLock + var inBackground = false + /// Extra information for metrics set from this orchestrator. struct MetricsData { /// The name of the track reported for this orchestrator. @@ -65,6 +73,16 @@ internal class FilesOrchestrator: FilesOrchestratorType { self.dateProvider = dateProvider self.telemetry = telemetry self.metricsData = metricsData + + #if canImport(UIKit) + self.subscription = applicationStatePublisher.subscribe { [weak self] appStateHistory in + self?.inBackground = appStateHistory.currentSnapshot.state == .background + } + #endif + } + + deinit { + self.subscription?.cancel() } // MARK: - `WritableFile` orchestration @@ -248,12 +266,6 @@ internal class FilesOrchestrator: FilesOrchestratorType { let batchAge = dateProvider.now.timeIntervalSince(fileCreationDateFrom(fileName: batchFile.name)) - #if canImport(UIKit) - let inBackground = UIApplication.dd.managedShared?.applicationState == .background - #else - let inBackground = false - #endif - telemetry.metric( name: BatchDeletedMetric.name, attributes: [ From a693cb759149db3f2ccec055bdc4088c1045fcb4 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Thu, 7 Sep 2023 13:48:12 +0100 Subject: [PATCH 10/15] REPLAY-1963 PR fixes --- DatadogCore/Sources/Core/DatadogCore.swift | 1 + .../Sources/Core/Storage/FeatureStorage.swift | 3 +++ .../Core/Storage/FilesOrchestrator.swift | 27 +++++-------------- .../Sources/Core/Upload/DataUploadDelay.swift | 10 ++----- .../Core/Upload/DataUploadWorker.swift | 4 +-- DatadogCore/Sources/Datadog.swift | 7 ++++- .../Tests/Datadog/Core/FeatureTests.swift | 1 + .../FilesOrchestrator+MetricsTests.swift | 1 + .../Persistence/FilesOrchestratorTests.swift | 2 ++ .../Persistence/Reading/FileReaderTests.swift | 3 +++ .../Persistence/Writing/FileWriterTests.swift | 9 +++++++ .../Core/Upload/DataUploadWorkerTests.swift | 1 + .../RUMEventFileOutputTests.swift | 1 + 13 files changed, 38 insertions(+), 32 deletions(-) diff --git a/DatadogCore/Sources/Core/DatadogCore.swift b/DatadogCore/Sources/Core/DatadogCore.swift index fccc4062a6..db6da12b25 100644 --- a/DatadogCore/Sources/Core/DatadogCore.swift +++ b/DatadogCore/Sources/Core/DatadogCore.swift @@ -230,6 +230,7 @@ extension DatadogCore: DatadogCoreProtocol { if let feature = feature as? DatadogRemoteFeature { let storage = FeatureStorage( featureName: T.name, + contextProvider: contextProvider, queue: readWriteQueue, directories: featureDirectories, dateProvider: dateProvider, diff --git a/DatadogCore/Sources/Core/Storage/FeatureStorage.swift b/DatadogCore/Sources/Core/Storage/FeatureStorage.swift index d1ff663106..75e61efcab 100644 --- a/DatadogCore/Sources/Core/Storage/FeatureStorage.swift +++ b/DatadogCore/Sources/Core/Storage/FeatureStorage.swift @@ -113,6 +113,7 @@ internal struct FeatureStorage { extension FeatureStorage { init( featureName: String, + contextProvider: DatadogContextProvider, queue: DispatchQueue, directories: FeatureDirectories, dateProvider: DateProvider, @@ -124,6 +125,7 @@ extension FeatureStorage { directory: directories.authorized, performance: performance, dateProvider: dateProvider, + contextProvider: contextProvider, telemetry: telemetry, metricsData: { guard let trackName = BatchMetric.trackValue(for: featureName) else { @@ -137,6 +139,7 @@ extension FeatureStorage { directory: directories.unauthorized, performance: performance, dateProvider: dateProvider, + contextProvider: contextProvider, telemetry: telemetry, metricsData: nil // do not send metrics for unauthorized orchestrator ) diff --git a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift index bc3197e402..866fda32cf 100644 --- a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift +++ b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift @@ -7,10 +7,6 @@ import Foundation import DatadogInternal -#if canImport(UIKit) -import UIKit -#endif - internal protocol FilesOrchestratorType: AnyObject { var performance: StoragePerformancePreset { get } @@ -41,14 +37,11 @@ internal class FilesOrchestrator: FilesOrchestratorType { /// Telemetry interface. let telemetry: Telemetry - #if canImport(UIKit) - /// Application state publisher for enriching telemetry. - let applicationStatePublisher: ApplicationStatePublisher = .init() - var subscription: ContextValueSubscription? - #endif /// Flag indicating if the application is in background state. - @ReadWriteLock - var inBackground = false + var inBackground: Bool { + return contextProvider.read().applicationStateHistory.currentSnapshot.state == .background + } + let contextProvider: DatadogContextProvider /// Extra information for metrics set from this orchestrator. struct MetricsData { @@ -65,24 +58,16 @@ internal class FilesOrchestrator: FilesOrchestratorType { directory: Directory, performance: StoragePerformancePreset, dateProvider: DateProvider, + contextProvider: DatadogContextProvider, telemetry: Telemetry, metricsData: MetricsData? = nil ) { self.directory = directory self.performance = performance self.dateProvider = dateProvider + self.contextProvider = contextProvider self.telemetry = telemetry self.metricsData = metricsData - - #if canImport(UIKit) - self.subscription = applicationStatePublisher.subscribe { [weak self] appStateHistory in - self?.inBackground = appStateHistory.currentSnapshot.state == .background - } - #endif - } - - deinit { - self.subscription?.cancel() } // MARK: - `WritableFile` orchestration diff --git a/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift b/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift index 0a27965cb6..27b5f13111 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadDelay.swift @@ -7,19 +7,13 @@ import Foundation import DatadogInternal -internal protocol Delay { - var current: TimeInterval { get } - func decrease() - func increase() -} - /// Mutable interval used for periodic data uploads. -internal class DataUploadDelay: Delay { +internal class DataUploadDelay { private let minDelay: TimeInterval private let maxDelay: TimeInterval private let changeRate: Double - var current: TimeInterval + private(set) var current: TimeInterval init(performance: UploadPerformancePreset) { self.minDelay = performance.minUploadDelay diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift index a93aa498a7..f774c0922d 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift @@ -27,7 +27,7 @@ internal class DataUploadWorker: DataUploadWorkerType { /// The core context provider private let contextProvider: DatadogContextProvider /// Delay used to schedule consecutive uploads. - private let delay: Delay + private let delay: DataUploadDelay /// Upload work scheduled by this worker. private var uploadWork: DispatchWorkItem? @@ -43,7 +43,7 @@ internal class DataUploadWorker: DataUploadWorkerType { dataUploader: DataUploaderType, contextProvider: DatadogContextProvider, uploadConditions: DataUploadConditions, - delay: Delay, + delay: DataUploadDelay, featureName: String, telemetry: Telemetry, backgroundTaskCoordinator: BackgroundTaskCoordinator? = nil diff --git a/DatadogCore/Sources/Datadog.swift b/DatadogCore/Sources/Datadog.swift index fd00f12633..5ff46a341f 100644 --- a/DatadogCore/Sources/Datadog.swift +++ b/DatadogCore/Sources/Datadog.swift @@ -105,7 +105,12 @@ public struct Datadog { /// The bundle object that contains the current executable. public var bundle: Bundle - /// Flag that determines if UIKit's background tasks are utilized to perform uploads in background. + /// Flag that determines if UIKit's [`beginBackgroundTask(expirationHandler:)`](https://developer.apple.com/documentation/uikit/uiapplication/1623031-beginbackgroundtaskwithexpiratio) and [`endBackgroundTask:`](https://developer.apple.com/documentation/uikit/uiapplication/1622970-endbackgroundtask) + /// are utilized to perform background uploads. It may extend the amount of time the app is operating in background by 30 seconds. + /// + /// Tasks are normally stopped when there's nothing to upload or when encountering any upload blocker such us no internet connection or low battery. + /// + /// By default it's set to `false`. public var backgroundTasksEnabled: Bool /// Creates a Datadog SDK Configuration object. diff --git a/DatadogCore/Tests/Datadog/Core/FeatureTests.swift b/DatadogCore/Tests/Datadog/Core/FeatureTests.swift index 2e1d288dae..3c6e3b00a2 100644 --- a/DatadogCore/Tests/Datadog/Core/FeatureTests.swift +++ b/DatadogCore/Tests/Datadog/Core/FeatureTests.swift @@ -18,6 +18,7 @@ class FeatureStorageTests: XCTestCase { super.setUp() storage = FeatureStorage( featureName: .mockAny(), + contextProvider: .mockAny(), queue: queue, directories: temporaryFeatureDirectories, dateProvider: RelativeDateProvider(advancingBySeconds: 0.01), diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift index 0f1fc51661..2fec19d4d7 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift @@ -34,6 +34,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase { directory: Directory(url: temporaryDirectory), performance: PerformancePreset.combining(storagePerformance: storage, uploadPerformance: upload), dateProvider: dateProvider, + contextProvider: .mockAny(), telemetry: telemetry, metricsData: .init(trackName: "track name", uploaderPerformance: upload) ) diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift index 0a1e24621b..0a1b4cf2fa 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift @@ -28,6 +28,7 @@ class FilesOrchestratorTests: XCTestCase { directory: .init(url: temporaryDirectory), performance: performance, dateProvider: dateProvider, + contextProvider: .mockAny(), telemetry: NOPTelemetry() ) } @@ -137,6 +138,7 @@ class FilesOrchestratorTests: XCTestCase { maxObjectSize: .max ), dateProvider: RelativeDateProvider(advancingBySeconds: 1), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ) diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift index 6480fe8e87..c5c6f36a83 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift @@ -28,6 +28,7 @@ class FileReaderTests: XCTestCase { directory: directory, performance: StoragePerformanceMock.readAllFiles, dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), encryption: nil, @@ -75,6 +76,7 @@ class FileReaderTests: XCTestCase { directory: directory, performance: StoragePerformanceMock.readAllFiles, dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), encryption: DataEncryptionMock( @@ -102,6 +104,7 @@ class FileReaderTests: XCTestCase { directory: directory, performance: StoragePerformanceMock.readAllFiles, dateProvider: dateProvider, + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), encryption: nil, diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift index f83a330d72..cde4d55a1c 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift @@ -29,6 +29,7 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -67,6 +68,7 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -109,6 +111,7 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -141,6 +144,7 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: RelativeDateProvider(advancingBySeconds: 1), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: true, @@ -188,6 +192,7 @@ class FileWriterTests: XCTestCase { maxObjectSize: 23 // 23 bytes is enough for TLV with {"key1":"value1"} JSON ), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -221,6 +226,7 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -243,6 +249,7 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -274,6 +281,7 @@ class FileWriterTests: XCTestCase { maxObjectSize: .max ), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -337,6 +345,7 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, diff --git a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift index 83a13564b8..82876f48f9 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift @@ -17,6 +17,7 @@ class DataUploadWorkerTests: XCTestCase { directory: .init(url: temporaryDirectory), performance: StoragePerformanceMock.writeEachObjectToNewFileAndReadAllFiles, dateProvider: dateProvider, + contextProvider: .mockAny(), telemetry: NOPTelemetry() ) lazy var writer = FileWriter( diff --git a/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift b/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift index ee5f966cbc..f12cdfd520 100644 --- a/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift +++ b/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift @@ -35,6 +35,7 @@ class RUMEventFileOutputTests: XCTestCase { uploadPerformance: .noOp ), dateProvider: fileCreationDateProvider, + contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, From 79785d73198b65b20378061e18b77e46f780641e Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Mon, 11 Sep 2023 12:51:19 +0100 Subject: [PATCH 11/15] REPLAY-1963 Refactor context acquisition --- DatadogCore/Sources/Core/DatadogCore.swift | 5 +- .../Sources/Core/Storage/FeatureStorage.swift | 16 ++-- .../Core/Storage/FilesOrchestrator.swift | 74 ++++++++++++------- .../Core/Storage/Reading/DataReader.swift | 9 ++- .../Core/Storage/Reading/FileReader.swift | 8 +- .../Sources/Core/Storage/Reading/Reader.swift | 4 +- .../Core/Storage/Writing/FileWriter.swift | 10 ++- .../Core/Upload/DataUploadWorker.swift | 15 ++-- .../Tests/Datadog/Core/FeatureTests.swift | 58 +++++++-------- .../FilesOrchestrator+MetricsTests.swift | 21 +++--- .../Persistence/FilesOrchestratorTests.swift | 68 +++++++++-------- .../Persistence/Reading/FileReaderTests.swift | 15 ++-- .../Persistence/Writing/FileWriterTests.swift | 36 ++++----- .../Core/Upload/DataUploadWorkerTests.swift | 4 +- .../Tests/Datadog/Mocks/CoreMocks.swift | 16 ++-- .../RUMEventFileOutputTests.swift | 4 +- 16 files changed, 194 insertions(+), 169 deletions(-) diff --git a/DatadogCore/Sources/Core/DatadogCore.swift b/DatadogCore/Sources/Core/DatadogCore.swift index db6da12b25..28adb149c6 100644 --- a/DatadogCore/Sources/Core/DatadogCore.swift +++ b/DatadogCore/Sources/Core/DatadogCore.swift @@ -319,10 +319,7 @@ internal struct DatadogCoreFeatureScope: FeatureScope { // On user thread: request SDK context. contextProvider.read { context in // On context thread: request writer for current tracking consent. - let writer = storage.writer( - for: bypassConsent ? .granted : context.trackingConsent, - forceNewBatch: forceNewBatch - ) + let writer = storage.writer(for: context, bypassConsent: bypassConsent, forceNewBatch: forceNewBatch) // Still on context thread: send `Writer` to EWC caller. The writer implements `AsyncWriter`, so // the implementation of `writer.write(value:)` will run asynchronously without blocking the context thread. diff --git a/DatadogCore/Sources/Core/Storage/FeatureStorage.swift b/DatadogCore/Sources/Core/Storage/FeatureStorage.swift index 75e61efcab..0ecf761fbc 100644 --- a/DatadogCore/Sources/Core/Storage/FeatureStorage.swift +++ b/DatadogCore/Sources/Core/Storage/FeatureStorage.swift @@ -23,15 +23,20 @@ internal struct FeatureStorage { /// Telemetry interface. let telemetry: Telemetry - func writer(for trackingConsent: TrackingConsent, forceNewBatch: Bool) -> Writer { - switch trackingConsent { + func writer( + for context: DatadogContext, + bypassConsent: Bool = false, + forceNewBatch: Bool = false + ) -> Writer { + switch bypassConsent ? .granted : context.trackingConsent { case .granted: return AsyncWriter( execute: FileWriter( orchestrator: authorizedFilesOrchestrator, forceNewFile: forceNewBatch, encryption: encryption, - telemetry: telemetry + telemetry: telemetry, + context: context ), on: queue ) @@ -43,7 +48,8 @@ internal struct FeatureStorage { orchestrator: unauthorizedFilesOrchestrator, forceNewFile: forceNewBatch, encryption: encryption, - telemetry: telemetry + telemetry: telemetry, + context: context ), on: queue ) @@ -125,7 +131,6 @@ extension FeatureStorage { directory: directories.authorized, performance: performance, dateProvider: dateProvider, - contextProvider: contextProvider, telemetry: telemetry, metricsData: { guard let trackName = BatchMetric.trackValue(for: featureName) else { @@ -139,7 +144,6 @@ extension FeatureStorage { directory: directories.unauthorized, performance: performance, dateProvider: dateProvider, - contextProvider: contextProvider, telemetry: telemetry, metricsData: nil // do not send metrics for unauthorized orchestrator ) diff --git a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift index 866fda32cf..a5755b7c0d 100644 --- a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift +++ b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift @@ -10,10 +10,14 @@ import DatadogInternal internal protocol FilesOrchestratorType: AnyObject { var performance: StoragePerformancePreset { get } - func getNewWritableFile(writeSize: UInt64) throws -> WritableFile - func getWritableFile(writeSize: UInt64) throws -> WritableFile - func getReadableFile(excludingFilesNamed excludedFileNames: Set) -> ReadableFile? - func delete(readableFile: ReadableFile, deletionReason: BatchDeletedMetric.RemovalReason) + func getNewWritableFile(writeSize: UInt64, context: DatadogContext) throws -> WritableFile + func getWritableFile(writeSize: UInt64, context: DatadogContext) throws -> WritableFile + func getReadableFile(excludingFilesNamed excludedFileNames: Set, context: DatadogContext) -> ReadableFile? + func delete( + readableFile: ReadableFile, + deletionReason: BatchDeletedMetric.RemovalReason, + context: DatadogContext + ) var ignoreFilesAgeWhenReading: Bool { get set } } @@ -37,12 +41,6 @@ internal class FilesOrchestrator: FilesOrchestratorType { /// Telemetry interface. let telemetry: Telemetry - /// Flag indicating if the application is in background state. - var inBackground: Bool { - return contextProvider.read().applicationStateHistory.currentSnapshot.state == .background - } - let contextProvider: DatadogContextProvider - /// Extra information for metrics set from this orchestrator. struct MetricsData { /// The name of the track reported for this orchestrator. @@ -58,14 +56,12 @@ internal class FilesOrchestrator: FilesOrchestratorType { directory: Directory, performance: StoragePerformancePreset, dateProvider: DateProvider, - contextProvider: DatadogContextProvider, telemetry: Telemetry, metricsData: MetricsData? = nil ) { self.directory = directory self.performance = performance self.dateProvider = dateProvider - self.contextProvider = contextProvider self.telemetry = telemetry self.metricsData = metricsData } @@ -79,19 +75,19 @@ internal class FilesOrchestrator: FilesOrchestratorType { /// /// - Parameter writeSize: the size of data to be written /// - Returns: `WritableFile` capable of writing data of given size - func getNewWritableFile(writeSize: UInt64) throws -> WritableFile { + func getNewWritableFile(writeSize: UInt64, context: DatadogContext) throws -> WritableFile { try validate(writeSize: writeSize) if let closedBatchName = lastWritableFileName { sendBatchClosedMetric(fileName: closedBatchName, forcedNew: true) } - return try createNewWritableFile(writeSize: writeSize) + return try createNewWritableFile(writeSize: writeSize, context: context) } /// Returns writable file accordingly to default heuristic of creating and reusing files. /// /// - Parameter writeSize: the size of data to be written /// - Returns: `WritableFile` capable of writing data of given size - func getWritableFile(writeSize: UInt64) throws -> WritableFile { + func getWritableFile(writeSize: UInt64, context: DatadogContext) throws -> WritableFile { try validate(writeSize: writeSize) if let lastWritableFile = reuseLastWritableFileIfPossible(writeSize: writeSize) { // if last writable file can be reused @@ -102,7 +98,7 @@ internal class FilesOrchestrator: FilesOrchestratorType { if let closedBatchName = lastWritableFileName { sendBatchClosedMetric(fileName: closedBatchName, forcedNew: false) } - return try createNewWritableFile(writeSize: writeSize) + return try createNewWritableFile(writeSize: writeSize, context: context) } } @@ -112,13 +108,13 @@ internal class FilesOrchestrator: FilesOrchestratorType { } } - private func createNewWritableFile(writeSize: UInt64) throws -> WritableFile { + private func createNewWritableFile(writeSize: UInt64, context: DatadogContext) throws -> WritableFile { // NOTE: RUMM-610 Because purging files directory is a memory-expensive operation, do it only when a new file // is created (we assume here that this won't happen too often). In details, this is to avoid over-allocating // internal `_FileCache` and `_NSFastEnumerationEnumerator` objects in downstream `FileManager` routines. // This optimisation results with flat allocation graph in a long term (vs endlessly growing if purging // happens too often). - try purgeFilesDirectoryIfNeeded() + try purgeFilesDirectoryIfNeeded(context: context) let newFileName = fileNameFrom(fileCreationDate: dateProvider.now) let newFile = try directory.createFile(named: newFileName) @@ -156,11 +152,16 @@ internal class FilesOrchestrator: FilesOrchestratorType { // MARK: - `ReadableFile` orchestration - func getReadableFile(excludingFilesNamed excludedFileNames: Set = []) -> ReadableFile? { + func getReadableFile( + excludingFilesNamed excludedFileNames: Set = [], + context: DatadogContext + ) -> ReadableFile? { do { let filesWithCreationDate = try directory.files() .map { (file: $0, creationDate: fileCreationDateFrom(fileName: $0.name)) } - .compactMap { try deleteFileIfItsObsolete(file: $0.file, fileCreationDate: $0.creationDate) } + .compactMap { + try deleteFileIfItsObsolete(file: $0.file, fileCreationDate: $0.creationDate, context: context) + } guard let (oldestFile, creationDate) = filesWithCreationDate .filter({ excludedFileNames.contains($0.file.name) == false }) @@ -186,10 +187,18 @@ internal class FilesOrchestrator: FilesOrchestratorType { } } - func delete(readableFile: ReadableFile, deletionReason: BatchDeletedMetric.RemovalReason) { + func delete( + readableFile: ReadableFile, + deletionReason: BatchDeletedMetric.RemovalReason, + context: DatadogContext + ) { do { try readableFile.delete() - sendBatchDeletedMetric(batchFile: readableFile, deletionReason: deletionReason) + sendBatchDeletedMetric( + batchFile: readableFile, + deletionReason: deletionReason, + context: context + ) } catch { telemetry.error("Failed to delete file", error: error) } @@ -201,7 +210,7 @@ internal class FilesOrchestrator: FilesOrchestratorType { // MARK: - Directory size management /// Removes oldest files from the directory if it becomes too big. - private func purgeFilesDirectoryIfNeeded() throws { + private func purgeFilesDirectoryIfNeeded(context: DatadogContext) throws { let filesSortedByCreationDate = try directory.files() .map { (file: $0, creationDate: fileCreationDateFrom(fileName: $0.name)) } .sorted { $0.creationDate < $1.creationDate } @@ -218,18 +227,22 @@ internal class FilesOrchestrator: FilesOrchestratorType { while sizeFreed < sizeToFree && !filesWithSizeSortedByCreationDate.isEmpty { let fileWithSize = filesWithSizeSortedByCreationDate.removeFirst() try fileWithSize.file.delete() - sendBatchDeletedMetric(batchFile: fileWithSize.file, deletionReason: .purged) + sendBatchDeletedMetric( + batchFile: fileWithSize.file, + deletionReason: .purged, + context: context + ) sizeFreed += fileWithSize.size } } } - private func deleteFileIfItsObsolete(file: File, fileCreationDate: Date) throws -> (file: File, creationDate: Date)? { + private func deleteFileIfItsObsolete(file: File, fileCreationDate: Date, context: DatadogContext) throws -> (file: File, creationDate: Date)? { let fileAge = dateProvider.now.timeIntervalSince(fileCreationDate) if fileAge > performance.maxFileAgeForRead { try file.delete() - sendBatchDeletedMetric(batchFile: file, deletionReason: .obsolete) + sendBatchDeletedMetric(batchFile: file, deletionReason: .obsolete, context: context) return nil } else { return (file: file, creationDate: fileCreationDate) @@ -244,13 +257,18 @@ internal class FilesOrchestrator: FilesOrchestratorType { /// - deletionReason: The reason of deleting this file. /// /// Note: The `batchFile` doesn't exist at this point. - private func sendBatchDeletedMetric(batchFile: ReadableFile, deletionReason: BatchDeletedMetric.RemovalReason) { + private func sendBatchDeletedMetric( + batchFile: ReadableFile, + deletionReason: BatchDeletedMetric.RemovalReason, + context: DatadogContext + ) { guard let metricsData = metricsData, deletionReason.includeInMetric else { return // do not track metrics for this orchestrator or deletion reason } let batchAge = dateProvider.now.timeIntervalSince(fileCreationDateFrom(fileName: batchFile.name)) - + let inBackground = context.applicationStateHistory.currentSnapshot.state == .background + print("👾 In background: \(inBackground)") telemetry.metric( name: BatchDeletedMetric.name, attributes: [ diff --git a/DatadogCore/Sources/Core/Storage/Reading/DataReader.swift b/DatadogCore/Sources/Core/Storage/Reading/DataReader.swift index e007ee5726..97527f27b3 100644 --- a/DatadogCore/Sources/Core/Storage/Reading/DataReader.swift +++ b/DatadogCore/Sources/Core/Storage/Reading/DataReader.swift @@ -5,6 +5,7 @@ */ import Foundation +import DatadogInternal /// Synchronizes the work of `FileReader` on given read/write queue. internal final class DataReader: Reader { @@ -17,15 +18,15 @@ internal final class DataReader: Reader { self.fileReader = fileReader } - func readNextBatch() -> Batch? { + func readNextBatch(context: DatadogContext) -> Batch? { queue.sync { - self.fileReader.readNextBatch() + self.fileReader.readNextBatch(context: context) } } - func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason) { + func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason, context: DatadogContext) { queue.sync { - self.fileReader.markBatchAsRead(batch, reason: reason) + self.fileReader.markBatchAsRead(batch, reason: reason, context: context) } } } diff --git a/DatadogCore/Sources/Core/Storage/Reading/FileReader.swift b/DatadogCore/Sources/Core/Storage/Reading/FileReader.swift index 1a9ac0dff4..ff2fc2414e 100644 --- a/DatadogCore/Sources/Core/Storage/Reading/FileReader.swift +++ b/DatadogCore/Sources/Core/Storage/Reading/FileReader.swift @@ -30,8 +30,8 @@ internal final class FileReader: Reader { // MARK: - Reading batches - func readNextBatch() -> Batch? { - guard let file = orchestrator.getReadableFile(excludingFilesNamed: filesRead) else { + func readNextBatch(context: DatadogContext) -> Batch? { + guard let file = orchestrator.getReadableFile(excludingFilesNamed: filesRead, context: context) else { return nil } @@ -95,8 +95,8 @@ internal final class FileReader: Reader { // MARK: - Accepting batches - func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason) { - orchestrator.delete(readableFile: batch.file, deletionReason: reason) + func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason, context: DatadogContext) { + orchestrator.delete(readableFile: batch.file, deletionReason: reason, context: context) filesRead.insert(batch.file.name) } } diff --git a/DatadogCore/Sources/Core/Storage/Reading/Reader.swift b/DatadogCore/Sources/Core/Storage/Reading/Reader.swift index 1c465440e3..1a58f6ba07 100644 --- a/DatadogCore/Sources/Core/Storage/Reading/Reader.swift +++ b/DatadogCore/Sources/Core/Storage/Reading/Reader.swift @@ -24,6 +24,6 @@ extension Batch { /// A type, reading batched data. internal protocol Reader { - func readNextBatch() -> Batch? - func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason) + func readNextBatch(context: DatadogContext) -> Batch? + func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason, context: DatadogContext) } diff --git a/DatadogCore/Sources/Core/Storage/Writing/FileWriter.swift b/DatadogCore/Sources/Core/Storage/Writing/FileWriter.swift index 11e230a60a..cab41b7371 100644 --- a/DatadogCore/Sources/Core/Storage/Writing/FileWriter.swift +++ b/DatadogCore/Sources/Core/Storage/Writing/FileWriter.swift @@ -20,17 +20,21 @@ internal struct FileWriter: Writer { let forceNewFile: Bool /// Telemetry interface. let telemetry: Telemetry + /// Current context of the SDK. + let context: DatadogContext init( orchestrator: FilesOrchestratorType, forceNewFile: Bool, encryption: DataEncryption?, - telemetry: Telemetry + telemetry: Telemetry, + context: DatadogContext ) { self.orchestrator = orchestrator self.encryption = encryption self.forceNewFile = forceNewFile self.telemetry = telemetry + self.context = context } // MARK: - Writing data @@ -55,7 +59,9 @@ internal struct FileWriter: Writer { // This is to avoid a situation where event is written to one file and event metadata to another. // If this happens, the reader will not be able to match event with its metadata. let writeSize = UInt64(encoded.count) - let file = try forceNewFile ? orchestrator.getNewWritableFile(writeSize: writeSize) : orchestrator.getWritableFile(writeSize: writeSize) + let file = try forceNewFile + ? orchestrator.getNewWritableFile(writeSize: writeSize, context: context) + : orchestrator.getWritableFile(writeSize: writeSize, context: context) try file.append(data: encoded) } catch { DD.logger.error("Failed to write data", error: error) diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift index f774c0922d..c175da93a2 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift @@ -66,7 +66,7 @@ internal class DataUploadWorker: DataUploadWorkerType { let context = contextProvider.read() let blockersForUpload = self.uploadConditions.blockersForUpload(with: context) let isSystemReady = blockersForUpload.isEmpty - let nextBatch = isSystemReady ? self.fileReader.readNextBatch() : nil + let nextBatch = isSystemReady ? self.fileReader.readNextBatch(context: context) : nil if let batch = nextBatch { self.backgroundTaskCoordinator?.beginBackgroundTask() DD.logger.debug("⏳ (\(self.featureName)) Uploading batch...") @@ -84,7 +84,11 @@ internal class DataUploadWorker: DataUploadWorkerType { DD.logger.debug(" → (\(self.featureName)) not delivered, will be retransmitted: \(uploadStatus.userDebugDescription)") } else { - self.fileReader.markBatchAsRead(batch, reason: .intakeCode(responseCode: uploadStatus.responseCode ?? -1)) // -1 is unexpected here + self.fileReader.markBatchAsRead( + batch, + reason: .intakeCode(responseCode: uploadStatus.responseCode ?? -1), + context: context + ) // -1 is unexpected here self.delay.decrease() DD.logger.debug(" → (\(self.featureName)) accepted, won't be retransmitted: \(uploadStatus.userDebugDescription)") @@ -101,7 +105,7 @@ internal class DataUploadWorker: DataUploadWorkerType { } } catch let error { // If upload can't be initiated do not retry, so drop the batch: - self.fileReader.markBatchAsRead(batch, reason: .invalid) + self.fileReader.markBatchAsRead(batch, reason: .invalid, context: context) telemetry.error("Failed to initiate '\(self.featureName)' data upload", error: error) } } else { @@ -132,12 +136,13 @@ internal class DataUploadWorker: DataUploadWorkerType { /// - It performs arbitrary upload (without checking upload condition and without re-transmitting failed uploads). internal func flushSynchronously() { queue.sync { - while let nextBatch = self.fileReader.readNextBatch() { + let context = contextProvider.read() + while let nextBatch = self.fileReader.readNextBatch(context: context) { defer { // RUMM-3459 Delete the underlying batch with `.flushed` reason that will be ignored in reported // metrics or telemetry. This is legitimate as long as `flush()` routine is only available for testing // purposes and never run in production apps. - self.fileReader.markBatchAsRead(nextBatch, reason: .flushed) + self.fileReader.markBatchAsRead(nextBatch, reason: .flushed, context: context) } do { // Try uploading the batch and do one more retry on failure. diff --git a/DatadogCore/Tests/Datadog/Core/FeatureTests.swift b/DatadogCore/Tests/Datadog/Core/FeatureTests.swift index 3c6e3b00a2..8bb676aa68 100644 --- a/DatadogCore/Tests/Datadog/Core/FeatureTests.swift +++ b/DatadogCore/Tests/Datadog/Core/FeatureTests.swift @@ -39,67 +39,67 @@ class FeatureStorageTests: XCTestCase { func testWhenWritingEventsWithoutForcingNewBatch_itShouldWriteAllEventsToTheSameBatch() throws { // When - storage.writer(for: .granted, forceNewBatch: false).write(value: ["event1": "1"]) - storage.writer(for: .granted, forceNewBatch: false).write(value: ["event2": "2"]) - storage.writer(for: .granted, forceNewBatch: false).write(value: ["event3": "3"]) + storage.writer(for: .mockWith(trackingConsent: .granted)).write(value: ["event1": "1"]) + storage.writer(for: .mockWith(trackingConsent: .granted)).write(value: ["event2": "2"]) + storage.writer(for: .mockWith(trackingConsent: .granted)).write(value: ["event3": "3"]) // Then storage.setIgnoreFilesAgeWhenReading(to: true) - let batch = try XCTUnwrap(storage.reader.readNextBatch()) + let batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.count, 3, "All 3 events should be written to the same batch") storage.reader.markBatchAsRead(batch) - XCTAssertNil(storage.reader.readNextBatch(), "There must be no other batche") + XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batche") } func testWhenWritingEventsWithForcingNewBatch_itShouldWriteEachEventToSeparateBatch() throws { // When - storage.writer(for: .granted, forceNewBatch: true).write(value: ["event1": "1"]) - storage.writer(for: .granted, forceNewBatch: true).write(value: ["event2": "2"]) - storage.writer(for: .granted, forceNewBatch: true).write(value: ["event3": "3"]) + storage.writer(for: .mockWith(trackingConsent: .granted), forceNewBatch: true).write(value: ["event1": "1"]) + storage.writer(for: .mockWith(trackingConsent: .granted), forceNewBatch: true).write(value: ["event2": "2"]) + storage.writer(for: .mockWith(trackingConsent: .granted), forceNewBatch: true).write(value: ["event3": "3"]) // Then storage.setIgnoreFilesAgeWhenReading(to: true) - var batch = try XCTUnwrap(storage.reader.readNextBatch()) + var batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.count, 1) storage.reader.markBatchAsRead(batch) - batch = try XCTUnwrap(storage.reader.readNextBatch()) + batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.count, 1) storage.reader.markBatchAsRead(batch) - batch = try XCTUnwrap(storage.reader.readNextBatch()) + batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.count, 1) storage.reader.markBatchAsRead(batch) - XCTAssertNil(storage.reader.readNextBatch(), "There must be no other batche") + XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batche") } // MARK: - Behaviours on tracking consent func testWhenWritingEventsInDifferentConsents_itOnlyReadsGrantedEvents() throws { // When - storage.writer(for: .granted, forceNewBatch: false).write(value: ["event.consent": "granted"]) - storage.writer(for: .pending, forceNewBatch: false).write(value: ["event.consent": "pending"]) - storage.writer(for: .notGranted, forceNewBatch: false).write(value: ["event.consent": "notGranted"]) + storage.writer(for: .mockWith(trackingConsent: .granted)).write(value: ["event.consent": "granted"]) + storage.writer(for: .mockWith(trackingConsent: .pending)).write(value: ["event.consent": "pending"]) + storage.writer(for: .mockWith(trackingConsent: .notGranted)).write(value: ["event.consent": "notGranted"]) // Then storage.setIgnoreFilesAgeWhenReading(to: true) - let batch = try XCTUnwrap(storage.reader.readNextBatch()) + let batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.map { $0.data.utf8String }, [#"{"event.consent":"granted"}"#]) storage.reader.markBatchAsRead(batch) - XCTAssertNil(storage.reader.readNextBatch(), "There must be no other batches") + XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batches") } func testGivenEventsWrittenInDifferentConsents_whenChangingConsentToGranted_itMakesPendingEventsReadable() throws { // Given - storage.writer(for: .granted, forceNewBatch: false).write(value: ["event.consent": "granted"]) - storage.writer(for: .pending, forceNewBatch: false).write(value: ["event.consent": "pending"]) - storage.writer(for: .notGranted, forceNewBatch: false).write(value: ["event.consent": "notGranted"]) + storage.writer(for: .mockWith(trackingConsent: .granted), forceNewBatch: false).write(value: ["event.consent": "granted"]) + storage.writer(for: .mockWith(trackingConsent: .pending), forceNewBatch: false).write(value: ["event.consent": "pending"]) + storage.writer(for: .mockWith(trackingConsent: .notGranted), forceNewBatch: false).write(value: ["event.consent": "notGranted"]) // When storage.migrateUnauthorizedData(toConsent: .granted) @@ -107,22 +107,22 @@ class FeatureStorageTests: XCTestCase { // Then storage.setIgnoreFilesAgeWhenReading(to: true) - var batch = try XCTUnwrap(storage.reader.readNextBatch()) + var batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.map { $0.data.utf8String }, [#"{"event.consent":"granted"}"#]) storage.reader.markBatchAsRead(batch) - batch = try XCTUnwrap(storage.reader.readNextBatch()) + batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.map { $0.data.utf8String }, [#"{"event.consent":"pending"}"#]) storage.reader.markBatchAsRead(batch) - XCTAssertNil(storage.reader.readNextBatch(), "There must be no other batches") + XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batches") } func testGivenEventsWrittenInDifferentConsents_whenChangingConsentToNotGranted_itDeletesPendingEvents() throws { // Given - storage.writer(for: .granted, forceNewBatch: false).write(value: ["event.consent": "granted"]) - storage.writer(for: .pending, forceNewBatch: false).write(value: ["event.consent": "pending"]) - storage.writer(for: .notGranted, forceNewBatch: false).write(value: ["event.consent": "notGranted"]) + storage.writer(for: .mockWith(trackingConsent: .granted), forceNewBatch: false).write(value: ["event.consent": "granted"]) + storage.writer(for: .mockWith(trackingConsent: .pending), forceNewBatch: false).write(value: ["event.consent": "pending"]) + storage.writer(for: .mockWith(trackingConsent: .notGranted), forceNewBatch: false).write(value: ["event.consent": "notGranted"]) // When storage.migrateUnauthorizedData(toConsent: .notGranted) @@ -130,14 +130,14 @@ class FeatureStorageTests: XCTestCase { // Then storage.setIgnoreFilesAgeWhenReading(to: true) - let batch = try XCTUnwrap(storage.reader.readNextBatch()) + let batch = try XCTUnwrap(storage.reader.readNextBatch(context: .mockAny())) XCTAssertEqual(batch.events.map { $0.data.utf8String }, [#"{"event.consent":"granted"}"#]) storage.reader.markBatchAsRead(batch) - XCTAssertNil(storage.reader.readNextBatch(), "There must be no other batches") + XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batches") storage.migrateUnauthorizedData(toConsent: .granted) - XCTAssertNil(storage.reader.readNextBatch(), "There must be no other batches, because pending events were deleted") + XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batches, because pending events were deleted") } // MARK: - Data migration diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift index 2fec19d4d7..a5b318f148 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift @@ -34,7 +34,6 @@ class FilesOrchestrator_MetricsTests: XCTestCase { directory: Directory(url: temporaryDirectory), performance: PerformancePreset.combining(storagePerformance: storage, uploadPerformance: upload), dateProvider: dateProvider, - contextProvider: .mockAny(), telemetry: telemetry, metricsData: .init(trackName: "track name", uploaderPerformance: upload) ) @@ -45,13 +44,13 @@ class FilesOrchestrator_MetricsTests: XCTestCase { func testWhenReadableFileIsDeleted_itSendsBatchDeletedMetric() throws { // Given let orchestrator = createOrchestrator() - let file = try XCTUnwrap(orchestrator.getWritableFile(writeSize: 1) as? ReadableFile) + let file = try XCTUnwrap(orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) as? ReadableFile) let expectedBatchAge = storage.minFileAgeForRead + 1 // When: // - wait and delete the file dateProvider.advance(bySeconds: expectedBatchAge) - orchestrator.delete(readableFile: file, deletionReason: .intakeCode(responseCode: 202)) + orchestrator.delete(readableFile: file, deletionReason: .intakeCode(responseCode: 202), context: .mockAny()) // Then let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Deleted")) @@ -73,13 +72,13 @@ class FilesOrchestrator_MetricsTests: XCTestCase { // Given: // - request some batch to be created let orchestrator = createOrchestrator() - _ = try orchestrator.getWritableFile(writeSize: 1) + _ = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) // When: // - wait more than batch obsolescence limit // - then request readable file, which should trigger obsolete files deletion dateProvider.advance(bySeconds: storage.maxFileAgeForRead + 1) - _ = orchestrator.getReadableFile() + _ = orchestrator.getReadableFile(context: .mockAny()) // Then let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Deleted")) @@ -103,14 +102,14 @@ class FilesOrchestrator_MetricsTests: XCTestCase { // - write more data than allowed directory size limit storage.maxDirectorySize = 10 // 10 bytes let orchestrator = createOrchestrator() - let file = try orchestrator.getWritableFile(writeSize: storage.maxDirectorySize + 1) + let file = try orchestrator.getWritableFile(writeSize: storage.maxDirectorySize + 1, context: .mockAny()) try file.append(data: .mockRandom(ofSize: storage.maxDirectorySize + 1)) let expectedBatchAge = storage.minFileAgeForRead + 1 // When: // - then request new batch, which triggers directory purging dateProvider.advance(bySeconds: expectedBatchAge) - _ = try orchestrator.getNewWritableFile(writeSize: 1) + _ = try orchestrator.getNewWritableFile(writeSize: 1, context: .mockAny()) // Then let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Deleted")) @@ -137,14 +136,14 @@ class FilesOrchestrator_MetricsTests: XCTestCase { let orchestrator = createOrchestrator() let expectedWrites: [UInt64] = [10, 5, 2] try expectedWrites.forEach { writeSize in - _ = try orchestrator.getWritableFile(writeSize: writeSize) + _ = try orchestrator.getWritableFile(writeSize: writeSize, context: .mockAny()) } // When // - wait more than allowed batch age for writes, so next batch request will create another batch // - then request another batch, which will close the previous one dateProvider.advance(bySeconds: (storage.maxFileAgeForWrite + 1)) - _ = try orchestrator.getWritableFile(writeSize: 1) + _ = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) // Then let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Closed")) @@ -166,7 +165,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase { let orchestrator = createOrchestrator() let expectedWrites: [UInt64] = [10, 5, 2] try expectedWrites.forEach { writeSize in - _ = try orchestrator.getWritableFile(writeSize: writeSize) + _ = try orchestrator.getWritableFile(writeSize: writeSize, context: .mockAny()) } let expectedBatchDuration = storage.maxFileAgeForWrite - 1 @@ -174,7 +173,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase { // - wait less than allowed batch age for writes // - then request new batch, which closes the previous one dateProvider.advance(bySeconds: expectedBatchDuration) - _ = try orchestrator.getNewWritableFile(writeSize: 1) + _ = try orchestrator.getNewWritableFile(writeSize: 1, context: .mockAny()) // Then let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Closed")) diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift index 0a1b4cf2fa..e77db72185 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift @@ -28,7 +28,6 @@ class FilesOrchestratorTests: XCTestCase { directory: .init(url: temporaryDirectory), performance: performance, dateProvider: dateProvider, - contextProvider: .mockAny(), telemetry: NOPTelemetry() ) } @@ -39,7 +38,7 @@ class FilesOrchestratorTests: XCTestCase { let dateProvider = RelativeDateProvider() let orchestrator = configureOrchestrator(using: dateProvider) - _ = try orchestrator.getWritableFile(writeSize: 1) + _ = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertEqual(try orchestrator.directory.files().count, 1) XCTAssertNotNil(try orchestrator.directory.file(named: dateProvider.now.toFileName)) @@ -47,9 +46,9 @@ class FilesOrchestratorTests: XCTestCase { func testWhenWritableFileIsObtainedAnotherTime_itReusesSameFile() throws { let orchestrator = configureOrchestrator(using: RelativeDateProvider(advancingBySeconds: 0.001)) - let file1 = try orchestrator.getWritableFile(writeSize: 1) + let file1 = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) - let file2 = try orchestrator.getWritableFile(writeSize: 1) + let file2 = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertEqual(try orchestrator.directory.files().count, 1) XCTAssertEqual(file1.name, file2.name) @@ -57,17 +56,17 @@ class FilesOrchestratorTests: XCTestCase { func testWhenSameWritableFileWasUsedMaxNumberOfTimes_itCreatesNewFile() throws { let orchestrator = configureOrchestrator(using: RelativeDateProvider(advancingBySeconds: 0.001)) - var previousFile: WritableFile = try orchestrator.getWritableFile(writeSize: 1) // first use of a new file + var previousFile: WritableFile = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) // first use of a new file var nextFile: WritableFile for _ in (0..<5) { for _ in (0 ..< performance.maxObjectsInFile).dropLast() { // skip first use - nextFile = try orchestrator.getWritableFile(writeSize: 1) + nextFile = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertEqual(nextFile.name, previousFile.name, "It should reuse the file \(performance.maxObjectsInFile) times") previousFile = nextFile } - nextFile = try orchestrator.getWritableFile(writeSize: 1) // first use of a new file + nextFile = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) // first use of a new file XCTAssertNotEqual(nextFile.name, previousFile.name, "It should create a new file when previous one is used \(performance.maxObjectsInFile) times") previousFile = nextFile } @@ -80,31 +79,31 @@ class FilesOrchestratorTests: XCTestCase { maxChunkSize: performance.maxObjectSize ) - let file1 = try orchestrator.getWritableFile(writeSize: performance.maxObjectSize) + let file1 = try orchestrator.getWritableFile(writeSize: performance.maxObjectSize, context: .mockAny()) try chunkedData.forEach { chunk in try file1.append(data: chunk) } - let file2 = try orchestrator.getWritableFile(writeSize: 1) + let file2 = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertNotEqual(file1.name, file2.name) } func testWhenWritableFileIsTooOld_itCreatesNewFile() throws { let dateProvider = RelativeDateProvider() let orchestrator = configureOrchestrator(using: dateProvider) - let file1 = try orchestrator.getWritableFile(writeSize: 1) + let file1 = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) dateProvider.advance(bySeconds: 1 + performance.maxFileAgeForWrite) - let file2 = try orchestrator.getWritableFile(writeSize: 1) + let file2 = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertNotEqual(file1.name, file2.name) } func testWhenWritableFileWasDeleted_itCreatesNewFile() throws { let orchestrator = configureOrchestrator(using: RelativeDateProvider(advancingBySeconds: 0.001)) - let file1 = try orchestrator.getWritableFile(writeSize: 1) + let file1 = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) try orchestrator.directory.files().forEach { try $0.delete() } - let file2 = try orchestrator.getWritableFile(writeSize: 1) + let file2 = try orchestrator.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertNotEqual(file1.name, file2.name) } @@ -116,10 +115,10 @@ class FilesOrchestratorTests: XCTestCase { using: RelativeDateProvider(startingFrom: Date().secondsAgo(0.01)) // simulate time difference ) - _ = try orchestrator1.getWritableFile(writeSize: 1) + _ = try orchestrator1.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertEqual(try orchestrator1.directory.files().count, 1) - _ = try orchestrator2.getWritableFile(writeSize: 1) + _ = try orchestrator2.getWritableFile(writeSize: 1, context: .mockAny()) XCTAssertEqual(try orchestrator2.directory.files().count, 2) } @@ -138,32 +137,31 @@ class FilesOrchestratorTests: XCTestCase { maxObjectSize: .max ), dateProvider: RelativeDateProvider(advancingBySeconds: 1), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ) // write 1MB to first file (1MB of directory size in total) - let file1 = try orchestrator.getWritableFile(writeSize: oneMB) + let file1 = try orchestrator.getWritableFile(writeSize: oneMB, context: .mockAny()) try file1.append(data: .mock(ofSize: oneMB)) // write 1MB to second file (2MB of directory size in total) - let file2 = try orchestrator.getWritableFile(writeSize: oneMB) + let file2 = try orchestrator.getWritableFile(writeSize: oneMB, context: .mockAny()) try file2.append(data: .mock(ofSize: oneMB)) // write 1MB to third file (3MB of directory size in total) - let file3 = try orchestrator.getWritableFile(writeSize: oneMB + 1) // +1 byte to exceed the limit + let file3 = try orchestrator.getWritableFile(writeSize: oneMB + 1, context: .mockAny()) // +1 byte to exceed the limit try file3.append(data: .mock(ofSize: oneMB + 1)) XCTAssertEqual(try orchestrator.directory.files().count, 3) // At this point, directory reached its maximum size. // Asking for the next file should purge the oldest one. - let file4 = try orchestrator.getWritableFile(writeSize: oneMB) + let file4 = try orchestrator.getWritableFile(writeSize: oneMB, context: .mockAny()) XCTAssertEqual(try orchestrator.directory.files().count, 3) XCTAssertNil(try? orchestrator.directory.file(named: file1.name)) try file4.append(data: .mock(ofSize: oneMB + 1)) - _ = try orchestrator.getWritableFile(writeSize: oneMB) + _ = try orchestrator.getWritableFile(writeSize: oneMB, context: .mockAny()) XCTAssertEqual(try orchestrator.directory.files().count, 3) XCTAssertNil(try? orchestrator.directory.file(named: file2.name)) } @@ -171,9 +169,9 @@ class FilesOrchestratorTests: XCTestCase { func testWhenNewWritableFileIsObtained_itAlwaysCreatesNewFile() throws { let orchestrator = configureOrchestrator(using: RelativeDateProvider(advancingBySeconds: 0.001)) - let file1 = try orchestrator.getNewWritableFile(writeSize: 1) - let file2 = try orchestrator.getNewWritableFile(writeSize: 1) - let file3 = try orchestrator.getNewWritableFile(writeSize: 1) + let file1 = try orchestrator.getNewWritableFile(writeSize: 1, context: .mockAny()) + let file2 = try orchestrator.getNewWritableFile(writeSize: 1, context: .mockAny()) + let file3 = try orchestrator.getNewWritableFile(writeSize: 1, context: .mockAny()) XCTAssertEqual(try orchestrator.directory.files().count, 3) XCTAssertNotEqual(file1.name, file2.name) @@ -189,7 +187,7 @@ class FilesOrchestratorTests: XCTestCase { let orchestrator = configureOrchestrator(using: dateProvider) dateProvider.advance(bySeconds: 1 + performance.minFileAgeForRead) - XCTAssertNil(orchestrator.getReadableFile()) + XCTAssertNil(orchestrator.getReadableFile(context: .mockAny())) } func testWhenReadableFileIsOldEnough_itReturnsFile() throws { @@ -199,7 +197,7 @@ class FilesOrchestratorTests: XCTestCase { dateProvider.advance(bySeconds: 1 + performance.minFileAgeForRead) - XCTAssertEqual(orchestrator.getReadableFile()?.name, file.name) + XCTAssertEqual(orchestrator.getReadableFile(context: .mockAny())?.name, file.name) } func testWhenReadableFileIsNotOldEnough_itReturnsNil() throws { @@ -209,7 +207,7 @@ class FilesOrchestratorTests: XCTestCase { dateProvider.advance(bySeconds: 0.5 * performance.minFileAgeForRead) - XCTAssertNil(orchestrator.getReadableFile()) + XCTAssertNil(orchestrator.getReadableFile(context: .mockAny())) } func testWhenThereAreMultipleReadableFiles_itReturnsOldestFile() throws { @@ -220,15 +218,15 @@ class FilesOrchestratorTests: XCTestCase { try fileNames.forEach { fileName in _ = try orchestrator.directory.createFile(named: fileName) } dateProvider.advance(bySeconds: 1 + performance.minFileAgeForRead) - XCTAssertEqual(orchestrator.getReadableFile()?.name, fileNames[0]) + XCTAssertEqual(orchestrator.getReadableFile(context: .mockAny())?.name, fileNames[0]) try orchestrator.directory.file(named: fileNames[0]).delete() - XCTAssertEqual(orchestrator.getReadableFile()?.name, fileNames[1]) + XCTAssertEqual(orchestrator.getReadableFile(context: .mockAny())?.name, fileNames[1]) try orchestrator.directory.file(named: fileNames[1]).delete() - XCTAssertEqual(orchestrator.getReadableFile()?.name, fileNames[2]) + XCTAssertEqual(orchestrator.getReadableFile(context: .mockAny())?.name, fileNames[2]) try orchestrator.directory.file(named: fileNames[2]).delete() - XCTAssertEqual(orchestrator.getReadableFile()?.name, fileNames[3]) + XCTAssertEqual(orchestrator.getReadableFile(context: .mockAny())?.name, fileNames[3]) try orchestrator.directory.file(named: fileNames[3]).delete() - XCTAssertNil(orchestrator.getReadableFile()) + XCTAssertNil(orchestrator.getReadableFile(context: .mockAny())) } func testsWhenThereAreMultipleReadableFiles_itReturnsFileByExcludingCertainNames() throws { @@ -240,7 +238,7 @@ class FilesOrchestratorTests: XCTestCase { dateProvider.advance(bySeconds: 1 + performance.minFileAgeForRead) XCTAssertEqual( - orchestrator.getReadableFile(excludingFilesNamed: Set(fileNames[0...2]))?.name, + orchestrator.getReadableFile(excludingFilesNamed: Set(fileNames[0...2]), context: .mockAny())?.name, fileNames[3] ) } @@ -252,7 +250,7 @@ class FilesOrchestratorTests: XCTestCase { dateProvider.advance(bySeconds: 2 * performance.maxFileAgeForRead) - XCTAssertNil(orchestrator.getReadableFile()) + XCTAssertNil(orchestrator.getReadableFile(context: .mockAny())) XCTAssertEqual(try orchestrator.directory.files().count, 0) } @@ -265,7 +263,7 @@ class FilesOrchestratorTests: XCTestCase { dateProvider.advance(bySeconds: 1 + performance.minFileAgeForRead) - let readableFile = try orchestrator.getReadableFile().unwrapOrThrow() + let readableFile = try orchestrator.getReadableFile(context: .mockAny()).unwrapOrThrow() XCTAssertEqual(try orchestrator.directory.files().count, 1) orchestrator.delete(readableFile: readableFile) XCTAssertEqual(try orchestrator.directory.files().count, 0) diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift index c5c6f36a83..83dfaf69a5 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift @@ -28,7 +28,6 @@ class FileReaderTests: XCTestCase { directory: directory, performance: StoragePerformanceMock.readAllFiles, dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), encryption: nil, @@ -46,7 +45,7 @@ class FileReaderTests: XCTestCase { .append(data: data) XCTAssertEqual(try directory.files().count, 1) - let batch = reader.readNextBatch() + let batch = reader.readNextBatch(context: .mockAny()) let expected = [ Event(data: "ABCD".utf8Data, metadata: "EFGH".utf8Data) @@ -76,7 +75,6 @@ class FileReaderTests: XCTestCase { directory: directory, performance: StoragePerformanceMock.readAllFiles, dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), encryption: DataEncryptionMock( @@ -86,7 +84,7 @@ class FileReaderTests: XCTestCase { ) // When - let batch = reader.readNextBatch() + let batch = reader.readNextBatch(context: .mockAny()) // Then let expected = [ @@ -104,7 +102,6 @@ class FileReaderTests: XCTestCase { directory: directory, performance: StoragePerformanceMock.readAllFiles, dateProvider: dateProvider, - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), encryption: nil, @@ -128,19 +125,19 @@ class FileReaderTests: XCTestCase { ] var batch: Batch - batch = try reader.readNextBatch().unwrapOrThrow() + batch = try reader.readNextBatch(context: .mockAny()).unwrapOrThrow() XCTAssertEqual(batch.events.first, expected[0]) reader.markBatchAsRead(batch) - batch = try reader.readNextBatch().unwrapOrThrow() + batch = try reader.readNextBatch(context: .mockAny()).unwrapOrThrow() XCTAssertEqual(batch.events.first, expected[1]) reader.markBatchAsRead(batch) - batch = try reader.readNextBatch().unwrapOrThrow() + batch = try reader.readNextBatch(context: .mockAny()).unwrapOrThrow() XCTAssertEqual(batch.events.first, expected[2]) reader.markBatchAsRead(batch) - XCTAssertNil(reader.readNextBatch()) + XCTAssertNil(reader.readNextBatch(context: .mockAny())) XCTAssertEqual(try directory.files().count, 0) } } diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift index cde4d55a1c..b819c82a12 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift @@ -29,12 +29,12 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) writer.write(value: ["key1": "value1"], metadata: ["meta1": "metaValue1"]) @@ -68,7 +68,6 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, @@ -77,7 +76,8 @@ class FileWriterTests: XCTestCase { "encrypted".utf8Data + data + "encrypted".utf8Data } ), - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) writer.write(value: ["key1": "value1"], metadata: ["meta1": "metaValue1"]) @@ -111,12 +111,12 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) writer.write(value: ["key1": "value1"]) @@ -144,12 +144,12 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: RelativeDateProvider(advancingBySeconds: 1), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: true, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) writer.write(value: ["key1": "value1"]) @@ -192,12 +192,12 @@ class FileWriterTests: XCTestCase { maxObjectSize: 23 // 23 bytes is enough for TLV with {"key1":"value1"} JSON ), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) writer.write(value: ["key1": "value1"]) // will be written @@ -226,12 +226,12 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) writer.write(value: FailingEncodableMock(errorMessage: "failed to encode `FailingEncodable`.")) @@ -249,12 +249,12 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) writer.write(value: ["ok"]) // will create the file @@ -281,12 +281,12 @@ class FileWriterTests: XCTestCase { maxObjectSize: .max ), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) let ioInterruptionQueue = DispatchQueue(label: "com.datadohq.file-writer-random-io") @@ -345,14 +345,14 @@ class FileWriterTests: XCTestCase { directory: directory, performance: PerformancePreset.mockAny(), dateProvider: SystemDateProvider(), - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: DataEncryptionMock( encrypt: { _ in "foo".utf8Data } ), - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) // When diff --git a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift index 82876f48f9..fe02188bdd 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift @@ -17,14 +17,14 @@ class DataUploadWorkerTests: XCTestCase { directory: .init(url: temporaryDirectory), performance: StoragePerformanceMock.writeEachObjectToNewFileAndReadAllFiles, dateProvider: dateProvider, - contextProvider: .mockAny(), telemetry: NOPTelemetry() ) lazy var writer = FileWriter( orchestrator: orchestrator, forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) lazy var reader = FileReader( orchestrator: orchestrator, diff --git a/DatadogCore/Tests/Datadog/Mocks/CoreMocks.swift b/DatadogCore/Tests/Datadog/Mocks/CoreMocks.swift index 8c29385796..07e4e22fb7 100644 --- a/DatadogCore/Tests/Datadog/Mocks/CoreMocks.swift +++ b/DatadogCore/Tests/Datadog/Mocks/CoreMocks.swift @@ -247,20 +247,20 @@ extension FeatureUpload { extension Reader { func markBatchAsRead(_ batch: Batch) { // We can ignore `reason` in most tests (used for sending metric), so we provide this convenience variant. - markBatchAsRead(batch, reason: .flushed) + markBatchAsRead(batch, reason: .flushed, context: .mockAny()) } } extension FilesOrchestratorType { func delete(readableFile: ReadableFile) { // We can ignore `deletionReason` in most tests (used for sending metric), so we provide this convenience variant. - delete(readableFile: readableFile, deletionReason: .flushed) + delete(readableFile: readableFile, deletionReason: .flushed, context: .mockAny()) } } class NOPReader: Reader { - func readNextBatch() -> Batch? { nil } - func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason) {} + func readNextBatch(context: DatadogContext) -> Batch? { nil } + func markBatchAsRead(_ batch: Batch, reason: BatchDeletedMetric.RemovalReason, context: DatadogContext) {} } internal class NOPFilesOrchestrator: FilesOrchestratorType { @@ -274,10 +274,10 @@ internal class NOPFilesOrchestrator: FilesOrchestratorType { var performance: StoragePerformancePreset { StoragePerformanceMock.noOp } - func getNewWritableFile(writeSize: UInt64) throws -> WritableFile { NOPFile() } - func getWritableFile(writeSize: UInt64) throws -> WritableFile { NOPFile() } - func getReadableFile(excludingFilesNamed excludedFileNames: Set) -> ReadableFile? { NOPFile() } - func delete(readableFile: ReadableFile, deletionReason: BatchDeletedMetric.RemovalReason) { } + func getNewWritableFile(writeSize: UInt64, context: DatadogContext) throws -> WritableFile { NOPFile() } + func getWritableFile(writeSize: UInt64, context: DatadogContext) throws -> WritableFile { NOPFile() } + func getReadableFile(excludingFilesNamed excludedFileNames: Set, context: DatadogContext) -> ReadableFile? { NOPFile() } + func delete(readableFile: ReadableFile, deletionReason: BatchDeletedMetric.RemovalReason, context: DatadogContext) { } var ignoreFilesAgeWhenReading = false } diff --git a/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift b/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift index f12cdfd520..1a8011d188 100644 --- a/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift +++ b/DatadogCore/Tests/Datadog/RUM/RUMEventOutputs/RUMEventFileOutputTests.swift @@ -35,12 +35,12 @@ class RUMEventFileOutputTests: XCTestCase { uploadPerformance: .noOp ), dateProvider: fileCreationDateProvider, - contextProvider: .mockAny(), telemetry: NOPTelemetry() ), forceNewFile: false, encryption: nil, - telemetry: NOPTelemetry() + telemetry: NOPTelemetry(), + context: .mockAny() ) let dataModel1 = RUMDataModelMock(attribute: "foo", context: RUMEventAttributes(contextInfo: ["custom.attribute": "value"])) From 8312bcb1d212e15f25bab84f834761f528a519e2 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Mon, 11 Sep 2023 12:56:47 +0100 Subject: [PATCH 12/15] REPLAY-1963 Fix linting issues --- DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift | 2 +- .../Datadog/Core/Persistence/FilesOrchestratorTests.swift | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift index a5755b7c0d..c2a5c384b7 100644 --- a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift +++ b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift @@ -153,7 +153,7 @@ internal class FilesOrchestrator: FilesOrchestratorType { // MARK: - `ReadableFile` orchestration func getReadableFile( - excludingFilesNamed excludedFileNames: Set = [], + excludingFilesNamed excludedFileNames: Set, context: DatadogContext ) -> ReadableFile? { do { diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift index e77db72185..23e8c2dcd6 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestratorTests.swift @@ -304,3 +304,11 @@ class FilesOrchestratorTests: XCTestCase { } // swiftlint:enable number_separator } + +extension FilesOrchestrator { + func getReadableFile( + context: DatadogContext + ) -> ReadableFile? { + getReadableFile(excludingFilesNamed: [], context: context) + } +} From a8daf2c1d0d7ecc09c2c41cc22e71f9750b14fdc Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Tue, 12 Sep 2023 09:41:50 +0100 Subject: [PATCH 13/15] REPLAY-1963 PR fixes --- CHANGELOG.md | 3 +- DatadogCore/Sources/Core/DatadogCore.swift | 1 - .../Sources/Core/Storage/FeatureStorage.swift | 1 - .../Core/Storage/FilesOrchestrator.swift | 1 - .../Upload/BackgroundTaskCoordinator.swift | 17 +++--- .../Core/Upload/DataUploadWorker.swift | 2 +- DatadogCore/Sources/Datadog.swift | 9 ++- .../Tests/Datadog/Core/FeatureTests.swift | 3 +- .../Core/Upload/DataUploadWorkerTests.swift | 59 ++++++++++++++++++- .../UIKitBackgroundTaskCoordinatorTests.swift | 4 +- 10 files changed, 80 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a7171e905..8395c1ff51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased +- [IMPROVEMENT] Add UIBackgroundTask for uploading jobs. See [#1412][] + # 2.1.2 / 29-08-2023 - [BUGFIX] Do not embed DatadogInternal while building Trace and RUM xcframeworks. See [#1444][]. @@ -17,7 +19,6 @@ - [IMPROVEMENT] Upgrade to PLCrashReporter 1.11.1. - [FEATURE] Report session sample rate to the backend with RUM events. See [#1410][] - [IMPROVEMENT] Expose Session Replay to Objective-C. see [#1419][] -- [IMPROVEMENT] Add UIBackgroundTask for uploading jobs. See [#1412][] # 2.0.0 / 31-07-2023 diff --git a/DatadogCore/Sources/Core/DatadogCore.swift b/DatadogCore/Sources/Core/DatadogCore.swift index 28adb149c6..1ee5124511 100644 --- a/DatadogCore/Sources/Core/DatadogCore.swift +++ b/DatadogCore/Sources/Core/DatadogCore.swift @@ -230,7 +230,6 @@ extension DatadogCore: DatadogCoreProtocol { if let feature = feature as? DatadogRemoteFeature { let storage = FeatureStorage( featureName: T.name, - contextProvider: contextProvider, queue: readWriteQueue, directories: featureDirectories, dateProvider: dateProvider, diff --git a/DatadogCore/Sources/Core/Storage/FeatureStorage.swift b/DatadogCore/Sources/Core/Storage/FeatureStorage.swift index 0ecf761fbc..e7d1b12f84 100644 --- a/DatadogCore/Sources/Core/Storage/FeatureStorage.swift +++ b/DatadogCore/Sources/Core/Storage/FeatureStorage.swift @@ -119,7 +119,6 @@ internal struct FeatureStorage { extension FeatureStorage { init( featureName: String, - contextProvider: DatadogContextProvider, queue: DispatchQueue, directories: FeatureDirectories, dateProvider: DateProvider, diff --git a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift index c2a5c384b7..d6eb0a2da4 100644 --- a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift +++ b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift @@ -268,7 +268,6 @@ internal class FilesOrchestrator: FilesOrchestratorType { let batchAge = dateProvider.now.timeIntervalSince(fileCreationDateFrom(fileName: batchFile.name)) let inBackground = context.applicationStateHistory.currentSnapshot.state == .background - print("👾 In background: \(inBackground)") telemetry.metric( name: BatchDeletedMetric.name, attributes: [ diff --git a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift index 28728d87fc..54a8f4e3ef 100644 --- a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift +++ b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift @@ -7,20 +7,21 @@ import Foundation /// The `BackgroundTaskCoordinator` protocol provides an abstraction for managing background tasks and includes methods for registering and ending background tasks. -/// It serves as a useful abstraction for testing purposes as well as allows decoupling from UIKit in order to maintain Catalyst compliation. To abstract from UIKit, it leverages -/// the fact that UIBackgroundTaskIdentifier raw value is based on Int. internal protocol BackgroundTaskCoordinator { /// Requests additional background execution time for the app. func beginBackgroundTask() - /// Marks the end of a specific long-running background task. - func endCurrentBackgroundTaskIfActive() + /// Marks the end of a background task. + /// + /// You must call this method to end a task that was started using the `beginBackgroundTask()` method. + /// If you do not, the system may terminate your app. + func endBackgroundTask() } #if canImport(UIKit) import UIKit import DatadogInternal -/// Bridge protocol that matches UIApplication's interface for background tasks. Allows easier testablity. +/// Bridge protocol that matches `UIApplication` interface for background tasks. Allows easier testablity. internal protocol UIKitAppBackgroundTaskCoordinator { func beginBackgroundTask(expirationHandler handler: (() -> Void)?) -> UIBackgroundTaskIdentifier func endBackgroundTask(_ identifier: UIBackgroundTaskIdentifier) @@ -41,16 +42,16 @@ internal class UIKitBackgroundTaskCoordinator: BackgroundTaskCoordinator { } internal func beginBackgroundTask() { - endCurrentBackgroundTaskIfActive() + endBackgroundTask() currentTaskId = app?.beginBackgroundTask { [weak self] in guard let self = self else { return } - self.endCurrentBackgroundTaskIfActive() + self.endBackgroundTask() } } - internal func endCurrentBackgroundTaskIfActive() { + internal func endBackgroundTask() { guard let currentTaskId = currentTaskId else { return } diff --git a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift index c175da93a2..355ea26070 100644 --- a/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift +++ b/DatadogCore/Sources/Core/Upload/DataUploadWorker.swift @@ -113,7 +113,7 @@ internal class DataUploadWorker: DataUploadWorkerType { DD.logger.debug("💡 (\(self.featureName)) No upload. Batch to upload: \(batchLabel), System conditions: \(blockersForUpload.description)") self.delay.increase() - self.backgroundTaskCoordinator?.endCurrentBackgroundTaskIfActive() + self.backgroundTaskCoordinator?.endBackgroundTask() } self.scheduleNextUpload(after: self.delay.current) diff --git a/DatadogCore/Sources/Datadog.swift b/DatadogCore/Sources/Datadog.swift index 5ff46a341f..e9cf752ae6 100644 --- a/DatadogCore/Sources/Datadog.swift +++ b/DatadogCore/Sources/Datadog.swift @@ -105,7 +105,7 @@ public struct Datadog { /// The bundle object that contains the current executable. public var bundle: Bundle - /// Flag that determines if UIKit's [`beginBackgroundTask(expirationHandler:)`](https://developer.apple.com/documentation/uikit/uiapplication/1623031-beginbackgroundtaskwithexpiratio) and [`endBackgroundTask:`](https://developer.apple.com/documentation/uikit/uiapplication/1622970-endbackgroundtask) + /// Flag that determines if UIApplication methods [`beginBackgroundTask(expirationHandler:)`](https://developer.apple.com/documentation/uikit/uiapplication/1623031-beginbackgroundtaskwithexpiratio) and [`endBackgroundTask:`](https://developer.apple.com/documentation/uikit/uiapplication/1622970-endbackgroundtask) /// are utilized to perform background uploads. It may extend the amount of time the app is operating in background by 30 seconds. /// /// Tasks are normally stopped when there's nothing to upload or when encountering any upload blocker such us no internet connection or low battery. @@ -148,6 +148,13 @@ public struct Datadog { /// https://www.ntppool.org/ . Using different pools or setting a no-op `ServerDateProvider` /// implementation will result in desynchronization of the SDK instance and the Datadog servers. /// This can lead to significant time shift in RUM sessions or distributed traces. + /// - backgroundTasksEnabled: A flag that determines if `UIApplication` methods + /// `beginBackgroundTask(expirationHandler:)` and `endBackgroundTask:` + /// are used to perform background uploads. + /// It may extend the amount of time the app is operating in background by 30 seconds. + /// Tasks are normally stopped when there's nothing to upload or when encountering + /// any upload blocker such us no internet connection or low battery. + /// By default it's set to `false`. public init( clientToken: String, env: String, diff --git a/DatadogCore/Tests/Datadog/Core/FeatureTests.swift b/DatadogCore/Tests/Datadog/Core/FeatureTests.swift index 8bb676aa68..72c12b94f5 100644 --- a/DatadogCore/Tests/Datadog/Core/FeatureTests.swift +++ b/DatadogCore/Tests/Datadog/Core/FeatureTests.swift @@ -18,7 +18,6 @@ class FeatureStorageTests: XCTestCase { super.setUp() storage = FeatureStorage( featureName: .mockAny(), - contextProvider: .mockAny(), queue: queue, directories: temporaryFeatureDirectories, dateProvider: RelativeDateProvider(advancingBySeconds: 0.01), @@ -74,7 +73,7 @@ class FeatureStorageTests: XCTestCase { XCTAssertEqual(batch.events.count, 1) storage.reader.markBatchAsRead(batch) - XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batche") + XCTAssertNil(storage.reader.readNextBatch(context: .mockAny()), "There must be no other batches") } // MARK: - Behaviours on tracking consent diff --git a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift index fe02188bdd..545112a88e 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/DataUploadWorkerTests.swift @@ -533,7 +533,7 @@ class DataUploadWorkerTests: XCTestCase { worker.cancelSynchronously() } - func testItTriggersBackgroundTaskRegistration() { + func testItTriggersBackgroundTaskBeginEndForSuccessfulUpload() { let expectTaskRegistered = expectation(description: "task should be registered") let expectTaskEnded = expectation(description: "task should be ended") let backgroundTaskCoordinator = SpyBackgroundTaskCoordinator( @@ -561,6 +561,61 @@ class DataUploadWorkerTests: XCTestCase { wait(for: [expectTaskRegistered, expectTaskEnded]) } } + + func testItTriggersBackgroundTaskBeginEndWhenBlockerOccurs() { + let expectTaskRegistered = expectation(description: "task should be registered") + let expectTaskEnded = expectation(description: "task should be ended") + let backgroundTaskCoordinator = SpyBackgroundTaskCoordinator( + beginBackgroundTaskCalled: { + expectTaskRegistered.fulfill() + }, endBackgroundTaskCalled: { + expectTaskEnded.fulfill() + } + ) + let worker = DataUploadWorker( + queue: uploaderQueue, + fileReader: reader, + dataUploader: DataUploaderMock(uploadStatus: .mockWith()), + contextProvider: .mockAny(), + uploadConditions: .neverUpload(), + delay: DataUploadDelay(performance: UploadPerformanceMock.veryQuick), + featureName: .mockAny(), + telemetry: NOPTelemetry(), + backgroundTaskCoordinator: backgroundTaskCoordinator + ) + writer.write(value: ["k1": "v1"]) + + // Then + withExtendedLifetime(worker) { + wait(for: [expectTaskRegistered, expectTaskEnded]) + } + } + + func testItTriggersBackgroundTaskEndWhenThereIsNothingToUpload() { + let expectTaskEnded = expectation(description: "task should be ended") + let backgroundTaskCoordinator = SpyBackgroundTaskCoordinator( + beginBackgroundTaskCalled: { + XCTFail("begin background task should not be called") + }, endBackgroundTaskCalled: { + expectTaskEnded.fulfill() + } + ) + let worker = DataUploadWorker( + queue: uploaderQueue, + fileReader: reader, + dataUploader: DataUploaderMock(uploadStatus: .mockWith()), + contextProvider: .mockAny(), + uploadConditions: .neverUpload(), + delay: DataUploadDelay(performance: UploadPerformanceMock.veryQuickInitialUpload), + featureName: .mockAny(), + telemetry: NOPTelemetry(), + backgroundTaskCoordinator: backgroundTaskCoordinator + ) + // Then + withExtendedLifetime(worker) { + wait(for: [expectTaskEnded]) + } + } } private extension DataUploadConditions { @@ -589,7 +644,7 @@ private class SpyBackgroundTaskCoordinator: BackgroundTaskCoordinator { beginBackgroundTaskCalled() } - func endCurrentBackgroundTaskIfActive() { + func endBackgroundTask() { endBackgroundTaskCalled() } } diff --git a/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift b/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift index b638579339..c827f48b67 100644 --- a/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Upload/UIKitBackgroundTaskCoordinatorTests.swift @@ -29,14 +29,14 @@ class UIKitBackgroundTaskCoordinatorTests: XCTestCase { func testEndBackgroundTask() throws { coordinator?.beginBackgroundTask() - coordinator?.endCurrentBackgroundTaskIfActive() + coordinator?.endBackgroundTask() XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, true) XCTAssertEqual(appSpy?.endBackgroundTaskCalled, true) } func testEndBackgroundTaskNotCalledWhenNotBegan() throws { - coordinator?.endCurrentBackgroundTaskIfActive() + coordinator?.endBackgroundTask() XCTAssertEqual(appSpy?.beginBackgroundTaskCalled, false) XCTAssertEqual(appSpy?.endBackgroundTaskCalled, false) From 6ab4e7bb1c26d495d6ac8e4e283fed377acca245 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Tue, 12 Sep 2023 10:06:35 +0100 Subject: [PATCH 14/15] REPLAY-1963 PR fixes --- .../Sources/Core/Storage/FilesOrchestrator.swift | 2 +- .../FilesOrchestrator+MetricsTests.swift | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift index d6eb0a2da4..fefe2e8087 100644 --- a/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift +++ b/DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift @@ -280,7 +280,7 @@ internal class FilesOrchestrator: FilesOrchestratorType { BatchDeletedMetric.uploaderWindowKey: performance.uploaderWindow.toMilliseconds, BatchDeletedMetric.batchAgeKey: batchAge.toMilliseconds, BatchDeletedMetric.batchRemovalReasonKey: deletionReason.toString(), - BatchDeletedMetric.inBackgroundKey: inBackground, + BatchDeletedMetric.inBackgroundKey: inBackground ] ) } diff --git a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift index a5b318f148..3066d3f74b 100644 --- a/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift +++ b/DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift @@ -127,6 +127,21 @@ class FilesOrchestrator_MetricsTests: XCTestCase { ]) } + func testWhenAppIsInBackground_itSendsBatchInBackgroundMetric() throws { + // Given + let orchestrator = createOrchestrator() + let context: DatadogContext = .mockWith(applicationStateHistory: .mockAppInBackground()) + let file = try XCTUnwrap(orchestrator.getWritableFile(writeSize: 1, context: context) as? ReadableFile) + + // When: + orchestrator.delete(readableFile: file, deletionReason: .intakeCode(responseCode: 202), context: context) + + // Then + let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Deleted")) + let inBackground = try XCTUnwrap(metric.attributes["in_background"] as? Bool) + XCTAssertTrue(inBackground) + } + // MARK: - "Batch Closed" Metric func testWhenNewBatchIsStarted_itSendsBatchClosedMetric() throws { From 275dec674270e4bcc3692d046222ac14505cb517 Mon Sep 17 00:00:00 2001 From: Maciej Burda Date: Tue, 12 Sep 2023 12:56:22 +0100 Subject: [PATCH 15/15] REPLAY-1963 Rephrase docs --- .../Sources/Core/Upload/BackgroundTaskCoordinator.swift | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift index 54a8f4e3ef..561d70d61d 100644 --- a/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift +++ b/DatadogCore/Sources/Core/Upload/BackgroundTaskCoordinator.swift @@ -8,12 +8,11 @@ import Foundation /// The `BackgroundTaskCoordinator` protocol provides an abstraction for managing background tasks and includes methods for registering and ending background tasks. internal protocol BackgroundTaskCoordinator { - /// Requests additional background execution time for the app. + /// Begins a background task, requesting additional background execution time for the app. + /// Calling it multiple times will end the previous background task and start a new one. + /// It internally implements system handler for background task expiration which will end current background task. func beginBackgroundTask() /// Marks the end of a background task. - /// - /// You must call this method to end a task that was started using the `beginBackgroundTask()` method. - /// If you do not, the system may terminate your app. func endBackgroundTask() }