⚗[REPLAY-341] Add VisualViewport tracking (Pinch Zoom)

packages/rum/src/domain/record/observer.ts

@@ -7,6 +7,8 @@ import {
   addEventListener,
   includes,
   DefaultPrivacyLevel,
+  isExperimentalFeatureEnabled,
+  noop,
 } from '@datadog/browser-core'
 import { NodePrivacyLevel } from '../../constants'
 import { getNodePrivacyLevel, shouldMaskNode } from './privacy'
@@ -28,12 +30,25 @@ import {
   ScrollCallback,
   StyleSheetRuleCallback,
   ViewportResizeCallback,
+  VisualViewportResizeCallback,
+  MousePosition,
+  MouseInteractionParam,
 } from './types'
-import { forEach, getWindowHeight, getWindowWidth, hookSetter, isTouchEvent } from './utils'
+import { forEach, hookSetter, isTouchEvent } from './utils'
 import { startMutationObserver, MutationController } from './mutationObserver'
 
+import {
+  getVisualViewport,
+  getWindowHeight,
+  getWindowWidth,
+  getScrollX,
+  getScrollY,
+  convertMouseEventToLayoutCoordinates,
+} from './viewports'
+
 const MOUSE_MOVE_OBSERVER_THRESHOLD = 50
 const SCROLL_OBSERVER_THRESHOLD = 100
+const VISUAL_VIEWPORT_OBSERVER_THRESHOLD = 200
 
 export function initObservers(o: ObserverParam): ListenerHandler {
   const mutationHandler = initMutationObserver(o.mutationController, o.mutationCb, o.defaultPrivacyLevel)
@@ -46,6 +61,10 @@ export function initObservers(o: ObserverParam): ListenerHandler {
   const styleSheetObserver = initStyleSheetObserver(o.styleSheetRuleCb)
   const focusHandler = initFocusObserver(o.focusCb)
 
+  const visualViewportResizeHandler = isExperimentalFeatureEnabled('visualviewport')
+    ? initVisualViewportResizeObserver(o.visualViewportResizeCb)
+    : noop
+
   return () => {
     mutationHandler()
     mousemoveHandler()
@@ -56,6 +75,7 @@ export function initObservers(o: ObserverParam): ListenerHandler {
     mediaInteractionHandler()
     styleSheetObserver()
     focusHandler()
+    visualViewportResizeHandler()
   }
 }
 
@@ -73,12 +93,17 @@ function initMoveObserver(cb: MousemoveCallBack): ListenerHandler {
       const target = event.target as Node
       if (hasSerializedNode(target)) {
         const { clientX, clientY } = isTouchEvent(event) ? event.changedTouches[0] : event
-        const position = {
+        const position: MousePosition = {
           id: getSerializedNodeId(target),
           timeOffset: 0,
           x: clientX,
           y: clientY,
         }
+        if (isExperimentalFeatureEnabled('visualviewport') && window.visualViewport) {
+          const { visualViewportX, visualViewportY } = convertMouseEventToLayoutCoordinates(clientX, clientY)
+          position.x = visualViewportX
+          position.y = visualViewportY
+        }
         cb([position], isTouchEvent(event) ? IncrementalSource.TouchMove : IncrementalSource.MouseMove)
       }
     }),
@@ -115,12 +140,18 @@ function initMouseInteractionObserver(
       return
     }
     const { clientX, clientY } = isTouchEvent(event) ? event.changedTouches[0] : event
-    cb({
+    const position: MouseInteractionParam = {
       id: getSerializedNodeId(target),
       type: eventTypeToMouseInteraction[event.type as keyof typeof eventTypeToMouseInteraction],
       x: clientX,
       y: clientY,
-    })
+    }
+    if (isExperimentalFeatureEnabled('visualviewport') && window.visualViewport) {
+      const { visualViewportX, visualViewportY } = convertMouseEventToLayoutCoordinates(clientX, clientY)
+      position.x = visualViewportX
+      position.y = visualViewportY
+    }
+    cb(position)
   }
   return addEventListeners(document, Object.keys(eventTypeToMouseInteraction) as DOM_EVENT[], handler, {
     capture: true,
@@ -141,12 +172,20 @@ function initScrollObserver(cb: ScrollCallback, defaultPrivacyLevel: DefaultPriv
       }
       const id = getSerializedNodeId(target)
       if (target === document) {
-        const scrollEl = (document.scrollingElement || document.documentElement)!
-        cb({
-          id,
-          x: scrollEl.scrollLeft,
-          y: scrollEl.scrollTop,
-        })
+        if (isExperimentalFeatureEnabled('visualviewport')) {
+          cb({
+            id,
+            x: getScrollX(),
+            y: getScrollY(),
+          })
+        } else {
+          const scrollEl = (document.scrollingElement || document.documentElement)!
+          cb({
+            id,
+            x: scrollEl.scrollLeft,
+            y: scrollEl.scrollTop,
+          })
+        }
       } else {
         cb({
           id,
@@ -338,3 +377,32 @@ function initFocusObserver(focusCb: FocusCallback): ListenerHandler {
     focusCb({ has_focus: document.hasFocus() })
   }).stop
 }
+
+function initVisualViewportResizeObserver(cb: VisualViewportResizeCallback): ListenerHandler {
+  if (!window.visualViewport) {
+    return noop
+  }
+  const { throttled: updateDimension, cancel: cancelThrottle } = throttle(
+    monitor(() => {
+      cb(getVisualViewport())
+    }),
+    VISUAL_VIEWPORT_OBSERVER_THRESHOLD,
+    {
+      trailing: false,
+    }
+  )
+  const removeListener = addEventListeners(
+    window.visualViewport,
+    [DOM_EVENT.RESIZE, DOM_EVENT.SCROLL],
+    updateDimension,
+    {
+      capture: true,
+      passive: true,
+    }
+  ).stop
+
+  return function stop() {
+    removeListener()
+    cancelThrottle()
+  }
+}

packages/rum/src/domain/record/record.ts

@@ -1,9 +1,11 @@
+import { isExperimentalFeatureEnabled } from '@datadog/browser-core'
 import { RecordType } from '../../types'
 import { serializeDocument } from './serialize'
 import { initObservers } from './observer'
 import { IncrementalSource, RecordAPI, RecordOptions } from './types'
-import { getWindowHeight, getWindowWidth } from './utils'
+
 import { MutationController } from './mutationObserver'
+import { getVisualViewport, getScrollX, getScrollY, getWindowHeight, getWindowWidth } from './viewports'
 
 export function record(options: RecordOptions): RecordAPI {
   const { emit } = options
@@ -37,26 +39,21 @@ export function record(options: RecordOptions): RecordAPI {
       data: {
         node: serializeDocument(document, options.defaultPrivacyLevel),
         initialOffset: {
-          left:
-            window.pageXOffset !== undefined
-              ? window.pageXOffset
-              : document?.documentElement.scrollLeft ||
-                document?.body?.parentElement?.scrollLeft ||
-                document?.body.scrollLeft ||
-                0,
-          top:
-            window.pageYOffset !== undefined
-              ? window.pageYOffset
-              : document?.documentElement.scrollTop ||
-                document?.body?.parentElement?.scrollTop ||
-                document?.body.scrollTop ||
-                0,
+          left: getScrollX(),
+          top: getScrollY(),
         },
       },
       type: RecordType.FullSnapshot,
     })
   }
 
+  if (isExperimentalFeatureEnabled('visualviewport') && window.visualViewport) {
+    emit({
+      data: getVisualViewport(),
+      type: RecordType.VisualViewport,
+    })
+  }
+
   takeFullSnapshot()
 
   const stopObservers = initObservers({
@@ -131,6 +128,12 @@ export function record(options: RecordOptions): RecordAPI {
         type: RecordType.Focus,
         data,
       }),
+    visualViewportResizeCb: (data) => {
+      emit({
+        data,
+        type: RecordType.VisualViewport,
+      })
+    },
   })
 
   return {

packages/rum/src/domain/record/types.ts

@@ -1,5 +1,5 @@
 import { DefaultPrivacyLevel } from '@datadog/browser-core'
-import { FocusRecord, RawRecord } from '../../types'
+import { FocusRecord, RawRecord, VisualViewportRecord } from '../../types'
 import { MutationController } from './mutationObserver'
 
 export enum IncrementalSource {
@@ -79,6 +79,7 @@ export interface ObserverParam {
   mouseInteractionCb: MouseInteractionCallBack
   scrollCb: ScrollCallback
   viewportResizeCb: ViewportResizeCallback
+  visualViewportResizeCb: VisualViewportResizeCallback
   inputCb: InputCallback
   mediaInteractionCb: MediaInteractionCallback
   styleSheetRuleCb: StyleSheetRuleCallback
@@ -181,7 +182,7 @@ export const MouseInteractions = {
 
 export type MouseInteractions = typeof MouseInteractions[keyof typeof MouseInteractions]
 
-interface MouseInteractionParam {
+export interface MouseInteractionParam {
   type: MouseInteractions
   id: number
   x: number
@@ -242,6 +243,8 @@ export type MediaInteractionCallback = (p: MediaInteractionParam) => void
 
 export type FocusCallback = (data: FocusRecord['data']) => void
 
+export type VisualViewportResizeCallback = (data: VisualViewportRecord['data']) => void
+
 export type ListenerHandler = () => void
 export type HookResetter = () => void
 

packages/rum/src/domain/record/utils.ts

@@ -22,22 +22,6 @@ export function hookSetter<T>(
   }
 }
 
-export function getWindowHeight(): number {
-  return (
-    window.innerHeight ||
-    (document.documentElement && document.documentElement.clientHeight) ||
-    (document.body && document.body.clientHeight)
-  )
-}
-
-export function getWindowWidth(): number {
-  return (
-    window.innerWidth ||
-    (document.documentElement && document.documentElement.clientWidth) ||
-    (document.body && document.body.clientWidth)
-  )
-}
-
 export function isTouchEvent(event: MouseEvent | TouchEvent): event is TouchEvent {
   return Boolean((event as TouchEvent).changedTouches)
 }

packages/rum/src/domain/record/viewports.spec.ts

@@ -0,0 +1,48 @@
+import { getScrollX, getScrollY, getWindowWidth, getWindowHeight } from './viewports'
+
+function isMobileSafari12() {
+  return /iPhone OS 12.* like Mac OS.* Version\/12.* Mobile.*Safari/.test(navigator.userAgent)
+}
+
+describe('layout viewport', () => {
+  beforeEach(() => {
+    document.body.style.setProperty('margin-bottom', '2000px')
+  })
+
+  afterEach(() => {
+    document.body.style.removeProperty('margin-bottom')
+    window.scrollTo(0, 0)
+  })
+
+  describe('get window width and height', () => {
+    it('normalized scroll matches native behaviour', () => {
+      const initialInnerWidth = getWindowWidth()
+      const initialInnerHeight = getWindowHeight()
+      expect(initialInnerWidth).toBe(window.innerWidth)
+      expect(initialInnerHeight).toBe(window.innerHeight)
+    })
+  })
+
+  describe('getScrollX/Y', () => {
+    it('normalized scroll matches initial behaviour', () => {
+      expect(getScrollX()).toBe(0)
+      expect(getScrollY()).toBe(0)
+      expect(getScrollX()).toBe(window.scrollX || window.pageXOffset)
+      expect(getScrollY()).toBe(window.scrollY || window.pageYOffset)
+    })
+    it('normalized scroll updates when scrolled', () => {
+      if (isMobileSafari12()) {
+        // Mobile Safari 12 doesn't support scrollTo() within an iframe
+        // Karma is evaluating some tests in an iframe
+        // https://coderwall.com/p/c-aqqw/scrollable-iframe-on-mobile-safari
+        pending('Mobile Safari 12 not supported')
+      }
+      const SCROLL_DOWN_PX = 100
+      window.scrollTo(0, SCROLL_DOWN_PX)
+      expect(getScrollX()).toBe(0)
+      expect(getScrollY()).toBe(100)
+      expect(getScrollX()).toBe(window.scrollX || window.pageXOffset)
+      expect(getScrollY()).toBe(window.scrollY || window.pageYOffset)
+    })
+  })
+})