⚗️ Add canvas change detection for Session Replay

packages/browser-rum/src/domain/record/canvas/canvasManager.spec.ts

@@ -0,0 +1,74 @@
+import { registerCleanupTask } from '@datadog/browser-core/test'
+import { createCanvasManager } from './canvasManager'
+
+describe('CanvasManager', () => {
+  it('tracks whether a canvas is dirty', () => {
+    const canvasManager = createCanvasManager()
+    const canvas = appendCanvas()
+
+    expect(canvasManager.isCanvasDirty(canvas)).toBeFalse()
+
+    canvasManager.markCanvasDirty(canvas)
+    expect(canvasManager.isCanvasDirty(canvas)).toBeTrue()
+
+    canvasManager.markCanvasClean(canvas)
+    expect(canvasManager.isCanvasDirty(canvas)).toBeFalse()
+
+    canvasManager.markCanvasDirty(canvas)
+    expect(canvasManager.isCanvasDirty(canvas)).toBeTrue()
+  })
+
+  it('tracks canvases independently', () => {
+    const canvasManager = createCanvasManager()
+    const dirtyCanvas = appendCanvas()
+    const cleanCanvas = appendCanvas()
+
+    canvasManager.markCanvasDirty(dirtyCanvas)
+
+    expect(canvasManager.isCanvasDirty(dirtyCanvas)).toBeTrue()
+    expect(canvasManager.isCanvasDirty(cleanCanvas)).toBeFalse()
+  })
+
+  it('returns connected dirty canvases', () => {
+    const canvasManager = createCanvasManager()
+    const canvas = appendCanvas()
+
+    canvasManager.markCanvasDirty(canvas)
+
+    expect(canvasManager.getDirtyCanvases()).toEqual([canvas])
+
+    canvasManager.markCanvasClean(canvas)
+    expect(canvasManager.getDirtyCanvases()).toEqual([])
+  })
+
+  it('does not retain detached canvases', () => {
+    const canvasManager = createCanvasManager()
+    const canvas = document.createElement('canvas')
+
+    canvasManager.markCanvasDirty(canvas)
+
+    expect(canvasManager.getDirtyCanvases()).toEqual([])
+
+    document.body.appendChild(canvas)
+    registerCleanupTask(() => canvas.remove())
+    expect(canvasManager.getDirtyCanvases()).toEqual([])
+  })
+
+  it('clears dirty canvases', () => {
+    const canvasManager = createCanvasManager()
+    const canvas = appendCanvas()
+    canvasManager.markCanvasDirty(canvas)
+
+    canvasManager.clearDirtyCanvases()
+
+    expect(canvasManager.getDirtyCanvases()).toEqual([])
+    expect(canvasManager.isCanvasDirty(canvas)).toBeFalse()
+  })
+})
+
+function appendCanvas(): HTMLCanvasElement {
+  const canvas = document.createElement('canvas')
+  document.body.appendChild(canvas)
+  registerCleanupTask(() => canvas.remove())
+  return canvas
+}

packages/browser-rum/src/domain/record/canvas/canvasManager.ts

@@ -0,0 +1,35 @@
+export interface CanvasManager {
+  clearDirtyCanvases: () => void
+  getDirtyCanvases: () => HTMLCanvasElement[]
+  isCanvasDirty: (canvas: HTMLCanvasElement) => boolean
+  markCanvasClean: (canvas: HTMLCanvasElement) => void
+  markCanvasDirty: (canvas: HTMLCanvasElement) => void
+}
+
+export function createCanvasManager(): CanvasManager {
+  const dirtyCanvases = new Set<HTMLCanvasElement>()
+
+  return {
+    clearDirtyCanvases: () => dirtyCanvases.clear(),
+    getDirtyCanvases: () => {
+      const connectedCanvases: HTMLCanvasElement[] = []
+
+      dirtyCanvases.forEach((canvas) => {
+        if (canvas.isConnected) {
+          connectedCanvases.push(canvas)
+        } else {
+          dirtyCanvases.delete(canvas)
+        }
+      })
+
+      return connectedCanvases
+    },
+    isCanvasDirty: (canvas) => dirtyCanvases.has(canvas),
+    markCanvasClean: (canvas) => dirtyCanvases.delete(canvas),
+    markCanvasDirty: (canvas) => {
+      if (canvas.isConnected) {
+        dirtyCanvases.add(canvas)
+      }
+    },
+  }
+}

packages/browser-rum/src/domain/record/canvas/canvasUtils.spec.ts

@@ -0,0 +1,15 @@
+import { isCanvasElement, isCanvasSizeAttribute } from './canvasUtils'
+
+describe('canvasUtils', () => {
+  it('identifies only canvas elements', () => {
+    expect(isCanvasElement(document.createElement('canvas'))).toBeTrue()
+    expect(isCanvasElement(document.createElement('div'))).toBeFalse()
+    expect(isCanvasElement(document.createTextNode('canvas'))).toBeFalse()
+  })
+
+  it('identifies canvas size attributes', () => {
+    expect(isCanvasSizeAttribute('width')).toBeTrue()
+    expect(isCanvasSizeAttribute('HEIGHT')).toBeTrue()
+    expect(isCanvasSizeAttribute('class')).toBeFalse()
+  })
+})

packages/browser-rum/src/domain/record/canvas/canvasUtils.ts

@@ -0,0 +1,11 @@
+import { isElementNode } from '@datadog/browser-rum-core'
+
+const CANVAS_SIZE_ATTRIBUTES = ['width', 'height']
+
+export function isCanvasElement(node: Node): node is HTMLCanvasElement {
+  return isElementNode(node) && node.tagName === 'CANVAS'
+}
+
+export function isCanvasSizeAttribute(attributeName: string): boolean {
+  return CANVAS_SIZE_ATTRIBUTES.includes(attributeName.toLowerCase())
+}

packages/browser-rum/src/domain/record/internalApi.ts

@@ -10,6 +10,7 @@ import { createElementsScrollPositions } from './elementsScrollPositions'
 import type { EmitRecordCallback } from './record.types'
 import type { SerializationTransaction } from './serialization'
 import { createRootInsertionCursor, SerializationKind, serializeInTransaction, serializeNode } from './serialization'
+import { createCanvasManager } from './canvas/canvasManager'
 
 /**
  * Take a full snapshot of the document, generating the same records that the browser SDK
@@ -74,6 +75,7 @@ export function takeNodeSnapshot(
 
 function createTemporaryRecordingScope(configuration?: Partial<RumConfiguration>): RecordingScope {
   return createRecordingScope(
+    createCanvasManager(),
     {
       defaultPrivacyLevel: NodePrivacyLevel.ALLOW,
       ...configuration,

packages/browser-rum/src/domain/record/record.spec.ts

@@ -73,6 +73,38 @@ describe('record', () => {
     ])
   })
 
+  describe('canvas mutation tracking', () => {
+    it('instruments canvas drawing when canvas recording is enabled', () => {
+      const originalFillRect = Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value
+
+      startRecording({ sessionReplayCanvasRecording: { enable: true, maxFramesPerSecond: 1 } })
+
+      expect(Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value).not.toBe(
+        originalFillRect
+      )
+    })
+
+    it('does not instrument canvas drawing when canvas recording is disabled', () => {
+      const originalFillRect = Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value
+
+      startRecording()
+
+      expect(Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value).toBe(
+        originalFillRect
+      )
+    })
+
+    it('does not instrument canvas drawing when the maximum frame rate is zero', () => {
+      const originalFillRect = Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value
+
+      startRecording({ sessionReplayCanvasRecording: { enable: true, maxFramesPerSecond: 0 } })
+
+      expect(Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value).toBe(
+        originalFillRect
+      )
+    })
+  })
+
   it('flushes pending mutation records before taking a full snapshot', async () => {
     startRecording()
 
@@ -374,12 +406,12 @@ describe('record', () => {
     })
   })
 
-  function startRecording() {
+  function startRecording(configuration: Partial<RumConfiguration> = {}) {
     lifeCycle = new LifeCycle()
     recordApi = record({
       emitRecord: emitSpy,
       emitStats: noop,
-      configuration: { defaultPrivacyLevel: DefaultPrivacyLevel.ALLOW } as RumConfiguration,
+      configuration: { defaultPrivacyLevel: DefaultPrivacyLevel.ALLOW, ...configuration } as RumConfiguration,
       lifeCycle,
       viewHistory: {
         findView: () => ({ id: FAKE_VIEW_ID, startClocks: {} }),

packages/browser-rum/src/domain/record/record.ts

@@ -15,13 +15,15 @@ import {
   trackViewEnd,
   trackViewportResize,
   trackVisualViewportResize,
+  trackCanvasContent,
 } from './trackers'
 import { createElementsScrollPositions } from './elementsScrollPositions'
 import type { ShadowRootsController } from './shadowRootsController'
 import { initShadowRootsController } from './shadowRootsController'
 import { startFullSnapshots } from './startFullSnapshots'
 import type { EmitRecordCallback, EmitStatsCallback } from './record.types'
 import { createRecordingScope } from './recordingScope'
+import { createCanvasManager } from './canvas/canvasManager'
 
 export interface RecordOptions {
   emitRecord: EmitRecordCallback
@@ -51,8 +53,14 @@ export function record(options: RecordOptions): RecordAPI {
     replayStats.addRecord(view.id)
   }
 
+  const canvasManager = createCanvasManager()
   const shadowRootsController = initShadowRootsController(processRecord, emitStats)
-  const scope = createRecordingScope(configuration, createElementsScrollPositions(), shadowRootsController)
+  const scope = createRecordingScope(
+    canvasManager,
+    configuration,
+    createElementsScrollPositions(),
+    shadowRootsController
+  )
 
   const { stop: stopFullSnapshots } = startFullSnapshots(lifeCycle, processRecord, emitStats, flushMutations, scope)
 
@@ -74,6 +82,7 @@ export function record(options: RecordOptions): RecordAPI {
     trackFocus(processRecord),
     trackVisualViewportResize(processRecord),
     trackViewEnd(lifeCycle, processRecord, flushMutations),
+    trackCanvasContent(scope),
   ]
 
   return {

packages/browser-rum/src/domain/record/recordingScope.ts

@@ -4,6 +4,7 @@ import type { ElementsScrollPositions } from './elementsScrollPositions'
 import { createEventIds, createNodeIds, createStringIds, createStyleSheetIds } from './itemIds'
 import type { EventIds, NodeIds, StringIds, StyleSheetIds } from './itemIds'
 import type { ShadowRootsController } from './shadowRootsController'
+import type { CanvasManager } from './canvas/canvasManager'
 
 /**
  * State associated with a stream of session replay records. When a new stream of records
@@ -14,6 +15,7 @@ import type { ShadowRootsController } from './shadowRootsController'
 export interface RecordingScope {
   resetIds(): void
 
+  canvasManager: CanvasManager
   configuration: RumConfiguration
   elementsScrollPositions: ElementsScrollPositions
   eventIds: EventIds
@@ -24,6 +26,7 @@ export interface RecordingScope {
 }
 
 export function createRecordingScope(
+  canvasManager: CanvasManager,
   configuration: RumConfiguration,
   elementsScrollPositions: ElementsScrollPositions,
   shadowRootsController: ShadowRootsController
@@ -41,6 +44,7 @@ export function createRecordingScope(
       scope.styleSheetIds.clear()
     },
 
+    canvasManager,
     configuration,
     elementsScrollPositions,
     eventIds,

packages/browser-rum/src/domain/record/serialization/serializeMutations.ts

@@ -12,6 +12,7 @@ import type { AttributeChange } from '../../../types'
 import type { RecordingScope } from '../recordingScope'
 import type { EmitRecordCallback, EmitStatsCallback } from '../record.types'
 import type { NodeId, NodeIds } from '../itemIds'
+import { isCanvasElement, isCanvasSizeAttribute } from '../canvas/canvasUtils'
 import type { SerializationTransaction } from './serializationTransaction'
 import { SerializationKind, serializeInTransaction } from './serializationTransaction'
 import { serializeNode } from './serializeNode'
@@ -111,6 +112,10 @@ function processRemovedNodes(nodes: Set<Node>, transaction: SerializationTransac
     }
 
     forNodeAndDescendants(node, (node: Node) => {
+      if (isCanvasElement(node)) {
+        transaction.scope.canvasManager.markCanvasClean(node)
+      }
+
       if (isNodeShadowHost(node)) {
         transaction.scope.shadowRootsController.removeShadowRoot(node.shadowRoot)
       }
@@ -257,6 +262,10 @@ function processAttributeMutations(
         continue // No change since the last snapshot.
       }
 
+      if (isCanvasElement(node) && isCanvasSizeAttribute(attributeName)) {
+        transaction.scope.canvasManager.markCanvasDirty(node)
+      }
+
       if (attributeName === 'value') {
         const attributeValue = getElementInputValue(node, privacyLevel)
         if (attributeValue !== undefined) {

packages/browser-rum/src/domain/record/serialization/serializeNode.node.spec.ts

@@ -5,6 +5,7 @@ import { ChangeType, PlaybackState } from '../../../types'
 import type { RecordingScope } from '../recordingScope'
 import type { ScrollPositions } from '../elementsScrollPositions'
 import { serializeHtml } from '../test/serializeHtml.specHelper'
+import { createRecordingScopeForTesting } from '../test/recordingScope.specHelper'
 import { SerializationKind } from './serializationTransaction'
 
 describe('serializeNode for DOM nodes', () => {
@@ -118,6 +119,19 @@ describe('serializeNode for DOM nodes', () => {
         ],
       ])
     })
+
+    it('marks nested canvases dirty when their subtree is serialized', async () => {
+      const scope = createRecordingScopeForTesting()
+
+      await serializeHtml('<div><canvas></canvas><div><canvas></canvas></div></div>', {
+        scope,
+        after: (target) => {
+          expect(scope.canvasManager.getDirtyCanvases()).toEqual(
+            Array.from((target as Element).querySelectorAll('canvas'))
+          )
+        },
+      })
+    })
   })
 
   describe('for SVG elements', () => {

packages/browser-rum/src/domain/record/serialization/serializeNode.ts

@@ -13,6 +13,7 @@ import {
 } from '@datadog/browser-rum-core'
 import { MediaInteractionType } from '../../../types'
 import type { NodeId, StyleSheetId } from '../itemIds'
+import { isCanvasElement } from '../canvas/canvasUtils'
 import type { InsertionCursor } from './insertionCursor'
 import type { SerializationTransaction } from './serializationTransaction'
 import { serializeDOMAttributes, serializeVirtualAttributes } from './serializeAttributes'
@@ -138,6 +139,10 @@ function serializeElementNode(
   const domAttributes = Object.entries(serializeDOMAttributes(element, privacyLevel, transaction))
   transaction.addNode(insertionPoint, encodedElementName(element), ...domAttributes)
 
+  if (isCanvasElement(element)) {
+    transaction.scope.canvasManager.markCanvasDirty(element)
+  }
+
   const {
     _cssText: cssText,
     rr_mediaState: mediaState,

packages/browser-rum/src/domain/record/test/recordingScope.specHelper.ts

@@ -3,20 +3,25 @@ import { noop } from '@datadog/browser-core'
 import { createElementsScrollPositions } from '../elementsScrollPositions'
 import type { RecordingScope } from '../recordingScope'
 import { createRecordingScope } from '../recordingScope'
+import type { CanvasManager } from '../canvas/canvasManager'
 import type { AddShadowRootCallBack, RemoveShadowRootCallBack } from '../shadowRootsController'
+import { createCanvasManager } from '../canvas/canvasManager'
 import { DEFAULT_CONFIGURATION } from './rumConfiguration.specHelper'
 import { DEFAULT_SHADOW_ROOT_CONTROLLER } from './shadowRootsController.specHelper'
 
 export function createRecordingScopeForTesting({
   configuration,
   addShadowRoot,
   removeShadowRoot,
+  canvasManager = createCanvasManager(),
 }: {
   configuration?: Partial<RumConfiguration>
   addShadowRoot?: AddShadowRootCallBack
   removeShadowRoot?: RemoveShadowRootCallBack
+  canvasManager?: CanvasManager
 } = {}): RecordingScope {
   return createRecordingScope(
+    canvasManager,
     {
       ...DEFAULT_CONFIGURATION,
       ...configuration,

packages/browser-rum/src/domain/record/trackers/index.ts

@@ -8,4 +8,5 @@ export { trackFocus } from './trackFocus'
 export { trackViewEnd } from './trackViewEnd'
 export { trackInput } from './trackInput'
 export { trackMutation } from './trackMutation'
+export { trackCanvasContent } from './trackCanvasContent'
 export type { Tracker } from './tracker.types'