Merge pull request #837 from DataDog/jward/RUMM-2157-first-party-hosts

Allow manually tracked resources to detect first party hosts

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ### Changes
 
+* [IMPROVEMENT] Allow manually tracked resources in RUM Sessions to detect first party hosts. 
+
 # 1.11.0-beta2 / 05-04-2022
 
 ### Changes

Sources/Datadog/Core/FeaturesConfiguration.swift

@@ -62,6 +62,7 @@ internal struct FeaturesConfiguration {
         let instrumentation: Instrumentation?
         let backgroundEventTrackingEnabled: Bool
         let onSessionStart: RUMSessionListener?
+        let firstPartyHosts: Set<String>
     }
 
     struct URLSessionAutoInstrumentation {
@@ -184,6 +185,14 @@ extension FeaturesConfiguration {
             )
         }
 
+        var sanitizedHosts: Set<String> = []
+        if let firstPartyHosts = configuration.firstPartyHosts {
+            sanitizedHosts = hostsSanitizer.sanitized(
+                hosts: firstPartyHosts,
+                warningMessage: "The first party host configured for Datadog SDK is not valid"
+            )
+        }
+
         if configuration.rumEnabled {
             let instrumentation = RUM.Instrumentation(
                 uiKitRUMViewsPredicate: configuration.rumUIKitViewsPredicate,
@@ -207,7 +216,8 @@ extension FeaturesConfiguration {
                     longTaskEventMapper: configuration.rumLongTaskEventMapper,
                     instrumentation: instrumentation,
                     backgroundEventTrackingEnabled: configuration.rumBackgroundEventTrackingEnabled,
-                    onSessionStart: configuration.rumSessionsListener
+                    onSessionStart: configuration.rumSessionsListener,
+                    firstPartyHosts: sanitizedHosts
                 )
             } else {
                 let error = ProgrammerError(
@@ -220,13 +230,10 @@ extension FeaturesConfiguration {
             }
         }
 
-        if let firstPartyHosts = configuration.firstPartyHosts {
+        if configuration.firstPartyHosts != nil {
             if configuration.tracingEnabled || configuration.rumEnabled {
                 urlSessionAutoInstrumentation = URLSessionAutoInstrumentation(
-                    userDefinedFirstPartyHosts: hostsSanitizer.sanitized(
-                        hosts: firstPartyHosts,
-                        warningMessage: "The first party host configured for Datadog SDK is not valid"
-                    ),
+                    userDefinedFirstPartyHosts: sanitizedHosts,
                     sdkInternalURLs: [
                         logsEndpoint.url,
                         tracesEndpoint.url,

Sources/Datadog/RUM/Instrumentation/Resources/URLSessionRUMResourcesHandler.swift

@@ -42,7 +42,6 @@ internal class URLSessionRUMResourcesHandler: URLSessionInterceptionHandler, RUM
                 url: url,
                 httpMethod: RUMMethod(httpMethod: interception.request.httpMethod),
                 kind: RUMResourceType(request: interception.request),
-                isFirstPartyRequest: interception.isFirstPartyRequest,
                 spanContext: interception.spanContext.flatMap { spanContext in
                     .init(
                         traceID: String(spanContext.traceID.rawValue),

Sources/Datadog/RUM/RUMMonitor/RUMCommand.swift

@@ -157,8 +157,6 @@ internal struct RUMStartResourceCommand: RUMResourceCommand {
     let httpMethod: RUMMethod
     /// A type of the Resource if it's possible to determine on start (when the response MIME is not yet known).
     let kind: RUMResourceType?
-    /// Whether or not the resource url targets a first party host, if that information is available.
-    let isFirstPartyRequest: Bool?
     /// Span context passed to the RUM backend in order to generate the APM span for underlying resource.
     let spanContext: RUMSpanContext?
 }

Sources/Datadog/RUM/RUMMonitor/Scopes/RUMResourceScope.swift

@@ -28,7 +28,7 @@ internal class RUMResourceScope: RUMScope {
     /// The HTTP method used to load this Resource.
     private var resourceHTTPMethod: RUMMethod
     /// Whether or not the Resource is provided by a first party host, if that information is available.
-    private let isFirstPartyResource: Bool?
+    private let isFirstPartyResource: Bool
     /// The Resource kind captured when starting the `URLRequest`.
     /// It may be `nil` if it's not possible to predict the kind from resource and the response MIME type is needed.
     private var resourceKindBasedOnRequest: RUMResourceType?
@@ -54,7 +54,6 @@ internal class RUMResourceScope: RUMScope {
         dateCorrection: DateCorrection,
         url: String,
         httpMethod: RUMMethod,
-        isFirstPartyResource: Bool?,
         resourceKindBasedOnRequest: RUMResourceType?,
         spanContext: RUMSpanContext?,
         onResourceEventSent: @escaping () -> Void,
@@ -69,7 +68,7 @@ internal class RUMResourceScope: RUMScope {
         self.resourceLoadingStartTime = startTime
         self.dateCorrection = dateCorrection
         self.resourceHTTPMethod = httpMethod
-        self.isFirstPartyResource = isFirstPartyResource
+        self.isFirstPartyResource = dependencies.firstPartyURLsFilter.isFirstParty(string: url)
         self.resourceKindBasedOnRequest = resourceKindBasedOnRequest
         self.spanContext = spanContext
         self.onResourceEventSent = onResourceEventSent

Sources/Datadog/RUM/RUMMonitor/Scopes/RUMScopeDependencies.swift

@@ -23,6 +23,7 @@ internal struct RUMScopeDependencies {
     let applicationVersion: String
     let sdkVersion: String
     let source: String
+    let firstPartyURLsFilter: FirstPartyURLsFilter
     let eventBuilder: RUMEventBuilder
     let eventOutput: RUMEventOutput
     let rumUUIDGenerator: RUMUUIDGenerator
@@ -61,6 +62,7 @@ internal extension RUMScopeDependencies {
             applicationVersion: rumFeature.configuration.common.applicationVersion,
             sdkVersion: rumFeature.configuration.common.sdkVersion,
             source: rumFeature.configuration.common.source,
+            firstPartyURLsFilter: FirstPartyURLsFilter(hosts: rumFeature.configuration.firstPartyHosts),
             eventBuilder: RUMEventBuilder(
                 eventsMapper: rumFeature.eventsMapper
             ),

Sources/Datadog/RUM/RUMMonitor/Scopes/RUMViewScope.swift

@@ -234,7 +234,6 @@ internal class RUMViewScope: RUMScope, RUMContextProvider {
             dateCorrection: dateCorrection,
             url: command.url,
             httpMethod: command.httpMethod,
-            isFirstPartyResource: command.isFirstPartyRequest,
             resourceKindBasedOnRequest: command.kind,
             spanContext: command.spanContext,
             onResourceEventSent: { [weak self] in

Sources/Datadog/RUMMonitor.swift

@@ -349,7 +349,6 @@ public class RUMMonitor: DDRUMMonitor, RUMCommandSubscriber {
                 url: request.url?.absoluteString ?? "unknown_url",
                 httpMethod: RUMMethod(httpMethod: request.httpMethod),
                 kind: RUMResourceType(request: request),
-                isFirstPartyRequest: nil,
                 spanContext: nil
             )
         )
@@ -368,7 +367,6 @@ public class RUMMonitor: DDRUMMonitor, RUMCommandSubscriber {
                 url: url.absoluteString,
                 httpMethod: .get,
                 kind: nil,
-                isFirstPartyRequest: nil,
                 spanContext: nil
             )
         )
@@ -388,7 +386,6 @@ public class RUMMonitor: DDRUMMonitor, RUMCommandSubscriber {
                 url: urlString,
                 httpMethod: httpMethod,
                 kind: nil,
-                isFirstPartyRequest: nil,
                 spanContext: nil
             )
         )

Sources/Datadog/URLSessionAutoInstrumentation/Interception/URLFiltering/FirstPartyURLsFilter.swift

@@ -32,4 +32,15 @@ internal struct FirstPartyURLsFilter {
         }
         return host.range(of: regex, options: .regularExpression) != nil
     }
+
+    // Returns `true` if given `String` can be parsed as a URL and matches the first
+    // party hosts defined by the user; `false` otherwise
+    func isFirstParty(string: String) -> Bool {
+        guard let url = URL(string: string),
+              let regex = self.regex,
+              let host = url.host else {
+            return false
+        }
+        return host.range(of: regex, options: .regularExpression) != nil
+    }
 }

Tests/DatadogTests/Datadog/Mocks/CoreMocks.swift

@@ -268,7 +268,8 @@ extension FeaturesConfiguration.RUM {
         longTaskEventMapper: RUMLongTaskEventMapper? = nil,
         instrumentation: FeaturesConfiguration.RUM.Instrumentation? = nil,
         backgroundEventTrackingEnabled: Bool = false,
-        onSessionStart: @escaping RUMSessionListener = mockNoOpSessionListener()
+        onSessionStart: @escaping RUMSessionListener = mockNoOpSessionListener(),
+        firstPartyHosts: Set<String> = []
     ) -> Self {
         return .init(
             common: common,
@@ -285,7 +286,8 @@ extension FeaturesConfiguration.RUM {
             longTaskEventMapper: longTaskEventMapper,
             instrumentation: instrumentation,
             backgroundEventTrackingEnabled: backgroundEventTrackingEnabled,
-            onSessionStart: onSessionStart
+            onSessionStart: onSessionStart,
+            firstPartyHosts: firstPartyHosts
         )
     }
 }

Tests/DatadogTests/Datadog/Mocks/RUMFeatureMocks.swift

@@ -362,7 +362,6 @@ extension RUMStartResourceCommand: AnyMockable, RandomMockable {
             url: url,
             httpMethod: httpMethod,
             kind: kind,
-            isFirstPartyRequest: isFirstPartyRequest,
             spanContext: spanContext
         )
     }
@@ -664,6 +663,7 @@ extension RUMScopeDependencies {
         applicationVersion: String = .mockAny(),
         sdkVersion: String = .mockAny(),
         source: String = "ios",
+        firstPartyURLsFilter: FirstPartyURLsFilter = FirstPartyURLsFilter(hosts: []),
         eventBuilder: RUMEventBuilder = RUMEventBuilder(eventsMapper: .mockNoOp()),
         eventOutput: RUMEventOutput = RUMEventOutputMock(),
         rumUUIDGenerator: RUMUUIDGenerator = DefaultRUMUUIDGenerator(),
@@ -686,6 +686,7 @@ extension RUMScopeDependencies {
             applicationVersion: applicationVersion,
             sdkVersion: sdkVersion,
             source: source,
+            firstPartyURLsFilter: firstPartyURLsFilter,
             eventBuilder: eventBuilder,
             eventOutput: eventOutput,
             rumUUIDGenerator: rumUUIDGenerator,
@@ -714,6 +715,7 @@ extension RUMScopeDependencies {
         applicationVersion: String? = nil,
         sdkVersion: String? = nil,
         source: String? = nil,
+        firstPartyUrls: Set<String>? = nil,
         eventBuilder: RUMEventBuilder? = nil,
         eventOutput: RUMEventOutput? = nil,
         rumUUIDGenerator: RUMUUIDGenerator? = nil,
@@ -736,6 +738,7 @@ extension RUMScopeDependencies {
             applicationVersion: applicationVersion ?? self.applicationVersion,
             sdkVersion: sdkVersion ?? self.sdkVersion,
             source: source ?? self.source,
+            firstPartyURLsFilter: firstPartyUrls.map { .init(hosts: $0) } ?? self.firstPartyURLsFilter,
             eventBuilder: eventBuilder ?? self.eventBuilder,
             eventOutput: eventOutput ?? self.eventOutput,
             rumUUIDGenerator: rumUUIDGenerator ?? self.rumUUIDGenerator,
@@ -868,7 +871,6 @@ extension RUMResourceScope {
             dateCorrection: dateCorrection,
             url: url,
             httpMethod: httpMethod,
-            isFirstPartyResource: isFirstPartyResource,
             resourceKindBasedOnRequest: resourceKindBasedOnRequest,
             spanContext: spanContext,
             onResourceEventSent: onResourceEventSent,

Tests/DatadogTests/Datadog/RUM/Instrumentation/Resources/URLSessionRUMResourcesHandlerTests.swift

@@ -45,40 +45,6 @@ class URLSessionRUMResourcesHandlerTests: XCTestCase {
         XCTAssertNil(resourceStartCommand.spanContext)
     }
 
-    func testGivenTaskInterceptionForFirstPartyHost_whenInterceptionStarts_itStartsRUMResourceForFirstPartyHost() throws {
-        let receiveCommand = expectation(description: "Receive RUM command")
-        commandSubscriber.onCommandReceived = { _ in receiveCommand.fulfill() }
-
-        // Given
-        let taskInterception = TaskInterception(request: .mockAny(), isFirstParty: true)
-
-        // When
-        handler.notify_taskInterceptionStarted(interception: taskInterception)
-
-        // Then
-        waitForExpectations(timeout: 0.5, handler: nil)
-
-        let resourceStartCommand = try XCTUnwrap(commandSubscriber.lastReceivedCommand as? RUMStartResourceCommand)
-        XCTAssertTrue(resourceStartCommand.isFirstPartyRequest!)
-    }
-
-    func testGivenTaskInterceptionForThirdPartyHost_whenInterceptionStarts_itStartsRUMResourceForThirdPartyHost() throws {
-        let receiveCommand = expectation(description: "Receive RUM command")
-        commandSubscriber.onCommandReceived = { _ in receiveCommand.fulfill() }
-
-        // Given
-        let taskInterception = TaskInterception(request: .mockAny(), isFirstParty: false)
-
-        // When
-        handler.notify_taskInterceptionStarted(interception: taskInterception)
-
-        // Then
-        waitForExpectations(timeout: 0.5, handler: nil)
-
-        let resourceStartCommand = try XCTUnwrap(commandSubscriber.lastReceivedCommand as? RUMStartResourceCommand)
-        XCTAssertFalse(resourceStartCommand.isFirstPartyRequest!)
-    }
-
     func testGivenTaskInterceptionWithSpanContext_whenInterceptionStarts_itStartsRUMResource() throws {
         let receiveCommand = expectation(description: "Receive RUM command")
         commandSubscriber.onCommandReceived = { _ in receiveCommand.fulfill() }

Tests/DatadogTests/Datadog/RUM/RUMMonitor/Scopes/RUMResourceScopeTests.swift

@@ -12,6 +12,7 @@ class RUMResourceScopeTests: XCTestCase {
     private let randomServiceName: String = .mockRandom()
     private lazy var dependencies: RUMScopeDependencies = .mockWith(
         serviceName: randomServiceName,
+        firstPartyURLsFilter: FirstPartyURLsFilter(hosts: ["firstparty.com"]),
         eventOutput: output
     )
     private let context = RUMContext.mockWith(
@@ -53,7 +54,6 @@ class RUMResourceScopeTests: XCTestCase {
             dateCorrection: .zero,
             url: "https://foo.com/resource/1",
             httpMethod: .post,
-            isFirstPartyResource: nil,
             resourceKindBasedOnRequest: nil,
             spanContext: .init(traceID: "100", spanID: "200")
         )
@@ -560,7 +560,7 @@ class RUMResourceScopeTests: XCTestCase {
             attributes: [:],
             startTime: currentTime,
             dateCorrection: .zero,
-            url: "https://foo.com/resource/1",
+            url: "https://firstparty.com/resource/1",
             httpMethod: .post,
             isFirstPartyResource: true,
             resourceKindBasedOnRequest: nil,
@@ -581,7 +581,7 @@ class RUMResourceScopeTests: XCTestCase {
         let providerType = try XCTUnwrap(event.resource.provider?.type)
         let providerDomain = try XCTUnwrap(event.resource.provider?.domain)
         XCTAssertEqual(providerType, .firstParty)
-        XCTAssertEqual(providerDomain, "foo.com")
+        XCTAssertEqual(providerDomain, "firstparty.com")
     }
 
     func testGivenStartedThirdartyResource_whenResourceLoadingEnds_itSendsResourceEventWithoutResourceProvider() throws {
@@ -627,7 +627,7 @@ class RUMResourceScopeTests: XCTestCase {
             attributes: [:],
             startTime: currentTime,
             dateCorrection: .zero,
-            url: "https://foo.com/resource/1",
+            url: "https://firstparty.com/resource/1",
             httpMethod: .post,
             isFirstPartyResource: true
         )
@@ -646,7 +646,7 @@ class RUMResourceScopeTests: XCTestCase {
         let providerType = try XCTUnwrap(event.error.resource?.provider?.type)
         let providerDomain = try XCTUnwrap(event.error.resource?.provider?.domain)
         XCTAssertEqual(providerType, .firstParty)
-        XCTAssertEqual(providerDomain, "foo.com")
+        XCTAssertEqual(providerDomain, "firstparty.com")
         XCTAssertEqual(event.error.sourceType, .ios)
     }
 

Tests/DatadogTests/Datadog/RUMMonitorTests.swift

@@ -262,6 +262,7 @@ class RUMMonitorTests: XCTestCase {
         let resourceEvent = session.viewVisits[0].resourceEvents[0]
         XCTAssertEqual(resourceEvent.resource.url, url.absoluteString)
         XCTAssertEqual(resourceEvent.resource.statusCode, 200)
+        XCTAssertNil(resourceEvent.resource.provider?.type)
     }
 
     func testStartingView_thenLoadingResourceWithURLString() throws {
@@ -284,6 +285,57 @@ class RUMMonitorTests: XCTestCase {
         XCTAssertEqual(resourceEvent.resource.statusCode, 333)
         XCTAssertEqual(resourceEvent.resource.type, .beacon)
         XCTAssertEqual(resourceEvent.resource.method, .post)
+        XCTAssertNil(resourceEvent.resource.provider?.type)
+    }
+
+    func testLoadingResourceWithURL_thenMarksFirstPartyURLs() throws {
+        RUMFeature.instance = .mockByRecordingRUMEventMatchers(
+            directories: temporaryFeatureDirectories,
+            configuration: .mockWith(
+                // .mockRandom always uses foo.com
+                firstPartyHosts: ["foo.com"]
+            )
+        )
+        defer { RUMFeature.instance?.deinitialize() }
+
+        let monitor = try createTestableRUMMonitor()
+        setGlobalAttributes(of: monitor)
+
+        let url: URL = .mockRandom()
+        monitor.startView(viewController: mockView)
+        monitor.startResourceLoading(resourceKey: "/resource/1", url: url)
+        monitor.stopResourceLoading(resourceKey: "/resource/1", response: .mockWith(statusCode: 200, mimeType: "image/png"))
+
+        let rumEventMatchers = try RUMFeature.waitAndReturnRUMEventMatchers(count: 4)
+        verifyGlobalAttributes(in: rumEventMatchers)
+
+        let session = try XCTUnwrap(try RUMSessionMatcher.groupMatchersBySessions(rumEventMatchers).first)
+        let resourceEvent = session.viewVisits[0].resourceEvents[0]
+        XCTAssertEqual(resourceEvent.resource.provider?.type, RUMResourceEvent.Resource.Provider.ProviderType.firstParty)
+    }
+
+    func testLoadingResourceWithURLString_thenMarksFirstPartyURLs() throws {
+        RUMFeature.instance = .mockByRecordingRUMEventMatchers(
+            directories: temporaryFeatureDirectories,
+            configuration: .mockWith(
+                firstPartyHosts: ["foo.com"]
+            )
+        )
+        defer { RUMFeature.instance?.deinitialize() }
+
+        let monitor = try createTestableRUMMonitor()
+        setGlobalAttributes(of: monitor)
+
+        monitor.startView(viewController: mockView)
+        monitor.startResourceLoading(resourceKey: "/resource/1", httpMethod: .post, urlString: "http://www.foo.com/some/url/string", attributes: [:])
+        monitor.stopResourceLoading(resourceKey: "/resource/1", statusCode: 333, kind: .beacon)
+
+        let rumEventMatchers = try RUMFeature.waitAndReturnRUMEventMatchers(count: 4)
+        verifyGlobalAttributes(in: rumEventMatchers)
+
+        let session = try XCTUnwrap(try RUMSessionMatcher.groupMatchersBySessions(rumEventMatchers).first)
+        let resourceEvent = session.viewVisits[0].resourceEvents[0]
+        XCTAssertEqual(resourceEvent.resource.provider?.type, RUMResourceEvent.Resource.Provider.ProviderType.firstParty)
     }
 
     func testStartingView_thenTappingButton() throws {