⚗ Collect scroll metrics (#2180)

* Initial poc

* Fix conflict

* Nudge CI

* Fix event listener

* Collect scroll metrics

* remove test

* Move scrollX/scrollY to browser-rum=core

* address code review comments

* Fix duration

* rename max_depth_timestamp to max_depth_time

* fix failing unit test

* generate type

* Empty commit

* fix failing ts job

* remove initialization & add passive listener

* rebase rum-events-format

* Initialize scroll metrics on page load

* rename scrollTop to maxScrollTop

* fix unit test

* Empty commit

* Empty commit

* submodule ref

* add timestamp to initial scroll event

* update unit tests

* Add experimental flag

* address code review comments

* update rum-events-format

* remove min when calculating scroll depth

* non optional params in scroll metrics

* rum events format update

* address code review

* address code review comments

* update schema

* update rum events format

* update rum-events-format

* update .d.ts file

* Fix flaky unit test

* update submodule

* update submodule

* renaming nits

* sync event format

* change naming

packages/core/src/tools/experimentalFeatures.ts

@@ -17,6 +17,7 @@ export enum ExperimentalFeature {
   PAGE_STATES = 'page_states',
   COLLECT_FLUSH_REASON = 'collect_flush_reason',
   NO_RESOURCE_DURATION_FROZEN_STATE = 'no_resource_duration_frozen_state',
+  SCROLLMAP = 'scrollmap',
 }
 
 const enabledExperimentalFeatures: Set<ExperimentalFeature> = new Set()

packages/rum/src/domain/record/viewports.spec.ts → packages/rum-core/src/browser/scroll.spec.ts

@@ -1,11 +1,11 @@
 import { addEventListener, DOM_EVENT, isIE } from '@datadog/browser-core'
-import { getScrollX, getScrollY } from './viewports'
+import { getScrollX, getScrollY } from './scroll'
 
 function isMobileSafari12() {
   return /iPhone OS 12.* like Mac OS.* Version\/12.* Mobile.*Safari/.test(navigator.userAgent)
 }
 
-describe('layout viewport', () => {
+describe('scroll', () => {
   let shouldWaitForWindowScrollEvent: boolean
 
   const addVerticalScrollBar = () => {

packages/rum-core/src/browser/scroll.ts

@@ -0,0 +1,25 @@
+export function getScrollX() {
+  let scrollX
+  const visual = window.visualViewport
+  if (visual) {
+    scrollX = visual.pageLeft - visual.offsetLeft
+  } else if (window.scrollX !== undefined) {
+    scrollX = window.scrollX
+  } else {
+    scrollX = window.pageXOffset || 0
+  }
+  return Math.round(scrollX)
+}
+
+export function getScrollY() {
+  let scrollY
+  const visual = window.visualViewport
+  if (visual) {
+    scrollY = visual.pageTop - visual.offsetTop
+  } else if (window.scrollY !== undefined) {
+    scrollY = window.scrollY
+  } else {
+    scrollY = window.pageYOffset || 0
+  }
+  return Math.round(scrollY)
+}

packages/rum-core/src/domain/rumEventsCollection/view/trackViewMetrics.spec.ts

@@ -1,5 +1,14 @@
-import type { Context, RelativeTime, Duration } from '@datadog/browser-core'
-import { addDuration, relativeNow } from '@datadog/browser-core'
+import type { Context, RelativeTime, TimeStamp, Duration } from '@datadog/browser-core'
+import {
+  ExperimentalFeature,
+  addExperimentalFeatures,
+  resetExperimentalFeatures,
+  DOM_EVENT,
+  addDuration,
+  relativeNow,
+} from '@datadog/browser-core'
+import type { Clock } from '@datadog/browser-core/test'
+import { createNewEvent, mockClock } from '@datadog/browser-core/test'
 import type { RumEvent } from '../../../rumEvent.types'
 import type { TestSetupBuilder } from '../../../../test'
 import { setup } from '../../../../test'
@@ -11,6 +20,8 @@ import { PAGE_ACTIVITY_END_DELAY, PAGE_ACTIVITY_VALIDATION_DELAY } from '../../w
 import { THROTTLE_VIEW_UPDATE_PERIOD } from './trackViews'
 import type { ViewTest } from './setupViewTest.specHelper'
 import { setupViewTest } from './setupViewTest.specHelper'
+import type { ScrollMetrics } from './trackViewMetrics'
+import { THROTTLE_SCROLL_DURATION, trackScrollMetrics } from './trackViewMetrics'
 
 const BEFORE_PAGE_ACTIVITY_VALIDATION_DELAY = (PAGE_ACTIVITY_VALIDATION_DELAY * 0.8) as Duration
 
@@ -519,4 +530,161 @@ describe('rum track view metrics', () => {
       expect(getViewUpdate(2).cumulativeLayoutShift).toBe(0.5)
     })
   })
+
+  describe('scroll metrics', () => {
+    describe('on scroll', () => {
+      let scrollMetrics: ScrollMetrics | undefined
+      let stopTrackScrollMetrics: () => void
+      let clock: Clock
+      const getMetrics = jasmine.createSpy('getMetrics')
+
+      const newScroll = (scrollParams: { scrollHeight: number; scrollDepth: number; scrollTop: number }) => {
+        getMetrics.and.returnValue(scrollParams)
+
+        window.dispatchEvent(createNewEvent(DOM_EVENT.SCROLL))
+
+        clock.tick(THROTTLE_SCROLL_DURATION)
+      }
+      describe('with flag enabled', () => {
+        beforeEach(() => {
+          clock = mockClock()
+          addExperimentalFeatures([ExperimentalFeature.SCROLLMAP])
+          stopTrackScrollMetrics = trackScrollMetrics(
+            { relative: 0 as RelativeTime, timeStamp: 0 as TimeStamp },
+            (s) => (scrollMetrics = s),
+            getMetrics
+          ).stop
+        })
+
+        afterEach(() => {
+          stopTrackScrollMetrics()
+          resetExperimentalFeatures()
+          scrollMetrics = undefined
+          clock.cleanup()
+        })
+
+        it('should update scroll metrics when scrolling the first time', () => {
+          newScroll({ scrollHeight: 1000, scrollDepth: 500, scrollTop: 100 })
+
+          expect(scrollMetrics).toEqual({
+            maxDepthScrollHeight: 1000,
+            maxDepth: 500,
+            maxDepthScrollTop: 100,
+            maxDepthTime: 1000 as Duration,
+          })
+        })
+
+        it('should update scroll metrics when scroll depth has increased', () => {
+          newScroll({ scrollHeight: 1000, scrollDepth: 500, scrollTop: 100 })
+
+          newScroll({ scrollHeight: 1000, scrollDepth: 600, scrollTop: 200 })
+
+          expect(scrollMetrics).toEqual({
+            maxDepthScrollHeight: 1000,
+            maxDepth: 600,
+            maxDepthScrollTop: 200,
+            maxDepthTime: 2000 as Duration,
+          })
+        })
+
+        it('should NOT update scroll metrics when scroll depth has not increased', () => {
+          newScroll({ scrollHeight: 1000, scrollDepth: 600, scrollTop: 200 })
+
+          newScroll({ scrollHeight: 1000, scrollDepth: 450, scrollTop: 50 })
+
+          expect(scrollMetrics).toEqual({
+            maxDepthScrollHeight: 1000,
+            maxDepth: 600,
+            maxDepthScrollTop: 200,
+            maxDepthTime: 1000 as Duration,
+          })
+        })
+      })
+
+      describe('with flag disabled', () => {
+        beforeEach(() => {
+          clock = mockClock()
+          stopTrackScrollMetrics = trackScrollMetrics(
+            { relative: 1 as RelativeTime, timeStamp: 2 as TimeStamp },
+            (s) => (scrollMetrics = s),
+            getMetrics
+          ).stop
+        })
+
+        afterEach(() => {
+          stopTrackScrollMetrics()
+          resetExperimentalFeatures()
+          scrollMetrics = undefined
+          clock.cleanup()
+        })
+        it('should NOT update scroll metrics when scrolling the first time', () => {
+          newScroll({ scrollHeight: 1000, scrollDepth: 500, scrollTop: 100 })
+
+          expect(scrollMetrics).toEqual(undefined)
+        })
+      })
+    })
+
+    describe('on load', () => {
+      beforeEach(() => {
+        setupBuilder.withFakeClock()
+      })
+
+      describe('with flag enabled', () => {
+        beforeEach(() => {
+          addExperimentalFeatures([ExperimentalFeature.SCROLLMAP])
+        })
+
+        afterEach(() => {
+          resetExperimentalFeatures()
+        })
+        it('should have an undefined loading time and empty scroll metrics if there is no activity on a route change', () => {
+          const { clock } = setupBuilder.build()
+          const { getViewUpdate, getViewUpdateCount, startView } = viewTest
+
+          startView()
+          clock.tick(THROTTLE_VIEW_UPDATE_PERIOD)
+
+          expect(getViewUpdateCount()).toEqual(3)
+          expect(getViewUpdate(2).loadingTime).toBeUndefined()
+          expect(getViewUpdate(2).scrollMetrics).toEqual(undefined)
+        })
+
+        it('should have a loading time equal to the activity time and scroll metrics if there is a unique activity on a route change', () => {
+          const { domMutationObservable, clock } = setupBuilder.build()
+          const { getViewUpdate, startView } = viewTest
+
+          startView()
+          clock.tick(BEFORE_PAGE_ACTIVITY_VALIDATION_DELAY)
+          domMutationObservable.notify()
+          clock.tick(AFTER_PAGE_ACTIVITY_END_DELAY)
+          clock.tick(THROTTLE_VIEW_UPDATE_PERIOD)
+
+          expect(getViewUpdate(3).loadingTime).toEqual(BEFORE_PAGE_ACTIVITY_VALIDATION_DELAY)
+          expect(getViewUpdate(3).scrollMetrics).toEqual({
+            maxDepthScrollHeight: jasmine.any(Number),
+            maxDepth: jasmine.any(Number),
+            maxDepthTime: jasmine.any(Number),
+            maxDepthScrollTop: jasmine.any(Number),
+          })
+        })
+      })
+
+      describe('with flag disabled', () => {
+        it('should have a loading time equal to the activity time but no scroll metrics if there is a unique activity on a route change', () => {
+          const { domMutationObservable, clock } = setupBuilder.build()
+          const { getViewUpdate, startView } = viewTest
+
+          startView()
+          clock.tick(BEFORE_PAGE_ACTIVITY_VALIDATION_DELAY)
+          domMutationObservable.notify()
+          clock.tick(AFTER_PAGE_ACTIVITY_END_DELAY)
+          clock.tick(THROTTLE_VIEW_UPDATE_PERIOD)
+
+          expect(getViewUpdate(3).loadingTime).toEqual(BEFORE_PAGE_ACTIVITY_VALIDATION_DELAY)
+          expect(getViewUpdate(3).scrollMetrics).toEqual(undefined)
+        })
+      })
+    })
+  })
 })

packages/rum-core/src/domain/rumEventsCollection/view/trackViewMetrics.ts

@@ -1,5 +1,16 @@
-import type { Duration, RelativeTime, Observable, ClocksState } from '@datadog/browser-core'
-import { noop, round, ONE_SECOND, elapsed } from '@datadog/browser-core'
+import type { ClocksState, Duration, Observable, RelativeTime } from '@datadog/browser-core'
+import {
+  ExperimentalFeature,
+  isExperimentalFeatureEnabled,
+  DOM_EVENT,
+  ONE_SECOND,
+  addEventListener,
+  elapsed,
+  noop,
+  relativeNow,
+  round,
+  throttle,
+} from '@datadog/browser-core'
 import type { RumLayoutShiftTiming } from '../../../browser/performanceCollection'
 import { supportPerformanceTimingEvent } from '../../../browser/performanceCollection'
 import { ViewLoadingType } from '../../../rawRumEvent.types'
@@ -8,6 +19,19 @@ import type { LifeCycle } from '../../lifeCycle'
 import { LifeCycleEventType } from '../../lifeCycle'
 import { waitPageActivityEnd } from '../../waitPageActivityEnd'
 
+import { getScrollY } from '../../../browser/scroll'
+import { getViewportDimension } from '../../../browser/viewportObservable'
+
+export interface ScrollMetrics {
+  maxDepth: number
+  maxDepthScrollHeight: number
+  maxDepthScrollTop: number
+  maxDepthTime: Duration
+}
+
+/** Arbitrary scroll throttle duration */
+export const THROTTLE_SCROLL_DURATION = ONE_SECOND
+
 export interface ViewMetrics {
   loadingTime?: Duration
   cumulativeLayoutShift?: number
@@ -23,6 +47,8 @@ export function trackViewMetrics(
 ) {
   const viewMetrics: ViewMetrics = {}
 
+  let scrollMetrics: ScrollMetrics | undefined
+
   const { stop: stopLoadingTimeTracking, setLoadEvent } = trackLoadingTime(
     lifeCycle,
     domMutationObservable,
@@ -31,10 +57,31 @@ export function trackViewMetrics(
     viewStart,
     (newLoadingTime) => {
       viewMetrics.loadingTime = newLoadingTime
+
+      // We compute scroll metrics at loading time to ensure we have scroll data when loading the view initially
+      // This is to ensure that we have the depth data even if the user didn't scroll or if the view is not scrollable.
+      if (isExperimentalFeatureEnabled(ExperimentalFeature.SCROLLMAP)) {
+        const { scrollHeight, scrollDepth, scrollTop } = computeScrollValues()
+
+        scrollMetrics = {
+          maxDepth: scrollDepth,
+          maxDepthScrollHeight: scrollHeight,
+          maxDepthTime: newLoadingTime,
+          maxDepthScrollTop: scrollTop,
+        }
+      }
       scheduleViewUpdate()
     }
   )
 
+  const { stop: stopScrollMetricsTracking } = trackScrollMetrics(
+    viewStart,
+    (newScrollMetrics) => {
+      scrollMetrics = newScrollMetrics
+    },
+    computeScrollValues
+  )
+
   let stopCLSTracking: () => void
   if (isLayoutShiftSupported()) {
     viewMetrics.cumulativeLayoutShift = 0
@@ -45,13 +92,70 @@ export function trackViewMetrics(
   } else {
     stopCLSTracking = noop
   }
+
   return {
     stop: () => {
       stopLoadingTimeTracking()
       stopCLSTracking()
+      stopScrollMetricsTracking()
     },
     setLoadEvent,
     viewMetrics,
+    getScrollMetrics: () => scrollMetrics,
+  }
+}
+
+export function trackScrollMetrics(
+  viewStart: ClocksState,
+  callback: (scrollMetrics: ScrollMetrics) => void,
+  getScrollValues = computeScrollValues
+) {
+  if (!isExperimentalFeatureEnabled(ExperimentalFeature.SCROLLMAP)) {
+    return { stop: noop }
+  }
+  let maxDepth = 0
+  const handleScrollEvent = throttle(
+    () => {
+      const { scrollHeight, scrollDepth, scrollTop } = getScrollValues()
+
+      if (scrollDepth > maxDepth) {
+        const now = relativeNow()
+        const maxDepthTime = elapsed(viewStart.relative, now)
+        maxDepth = scrollDepth
+        callback({
+          maxDepth,
+          maxDepthScrollHeight: scrollHeight,
+          maxDepthTime,
+          maxDepthScrollTop: scrollTop,
+        })
+      }
+    },
+    THROTTLE_SCROLL_DURATION,
+    { leading: false, trailing: true }
+  )
+
+  const { stop } = addEventListener(window, DOM_EVENT.SCROLL, handleScrollEvent.throttled, { passive: true })
+
+  return {
+    stop: () => {
+      handleScrollEvent.cancel()
+      stop()
+    },
+  }
+}
+
+function computeScrollValues() {
+  const scrollTop = getScrollY()
+
+  const { height } = getViewportDimension()
+
+  const scrollHeight = Math.round((document.scrollingElement || document.documentElement).scrollHeight)
+  const scrollDepth = Math.round(height + scrollTop)
+
+  return {
+    scrollHeight,
+    scrollDepth,
+    scrollTop,
   }
 }