Add WebGL support (#756)

* Add very basic webgl support * document the default * only capture rr_dataURL in 2d canvas contexts * rr_dataURL no longer part of webgl snapshot * ignore __diff_output__ from jest-image-snapshot * Rename generic "Monorepo" to "RRWeb Monorepo" * Serialize WebGL variables * Move rrweb test port number to unique port rrweb-snapshot uses 3030, rrweb uses 3031 * Prepare for WebGL2 * Split up canvas replay and record webgl vars * fix typo * fix typo part 2 * fix typo * Handle non-variables too * provide correct context for warning * (De)Serialize a lot of different objects * monorepo root should be the first in the list * Upgrade puppeteer to 11.x * Correctly de-serialize webgl variables * Encode arrayBuffers contents to base64 * rename contents to base64 * add webgl2 support and serialize HTMLImageElements * Support serializing ImageData * Correctly classify WebGL2 events * Serialize format changed * check if canvas has contents before we save the dataURL * Remove blank dataURL * reference original file not type defintion file * update types * rename code worspace * update dependencies * add spector to inspect webgl * remove live server settings from code workspace * Save canvas context in the node Prevents from saving webgl canvases as 2d dataUrls * remove extra braces * add ICanvas type * use ICanvas from rrweb-snapshot in rrweb instead of OgmentedCanvas * add snapshots and webgl 2 tests * Upgrade to puppeteer 12.0.1 * Revert back to puppeteer 9.1.1 * Keep index order consistent between replay and record * keep correct index order in webgl2 * fixed forgotten import * buffer up pending canvas mutations * unify the way webgl and webgl2 get patched * fix parsing error * Add types for serialize-args * Add debugging for webgl replay * Move start-server to utils * turn off debug mode by default * Move pendingCanvasMutations to local object and fix if/else statement * Always save pending mutations * only use assert snapshot as it's clearer whats going on * Ugly fix for now * Making the tests more DRY * flush at the end of each request animation frame * Looks like the promise made this test more predictable * add waitForRAF * Make nested iframe recording robust no matter the test speed * mute noisy error in test * force a requestAnimationFrame * Bundle events within one frame together as much as possible WebGL events need to be bundled together as much as possible so they don't accidentally get split over multiple animation frames. `newFrame: true` is used to indicate the start of an new animation frame in the recording, and that the event shouldn't be bundled with the previous events. * Rename RafStamps * Override event.delay * cleanup * Add tests for addDelay * Add webgl e2e test * Remove settimeout * DRY-up test * Preload images in webgl * Add e2e test for webgl image preloading * don't turn on devtools by default! * Remove spector * close server after use * Add imageMap parameter * Make e2e image test more robust * document debug mode * cleanup * WebGL recording in iframes & Safari 14 support * fix tests * don't save null objects as WebGLVar * group (de)serialized webgl variables by context * Fix test * fix tests * bundle webgl mutations on request animation frame Instead of fixing it on the replay side we buffer up webgl canvas mutations and wait for a new RAF to flush them. This allows us to remove `newFrame` from the events and simplify things a little * Add canvas element to mutation observer file * Add Canvas (Mutation) Manager Allows you to do `record.freezePage()` and canvas events will get paused. Based on #756 (comment) * cleanup * Make sure the correct </body> gets replaced * Perf: Speed up check to see if canvas is blank * Access unpatched getImageData * Use is2DCanvasBlank only for 2d context
rrweb-io · Feb 3, 2022 · ab9fed0 · ab9fed0
1 parent 5ec7d9e
commit ab9fed0
Show file tree

Hide file tree

Showing 63 changed files with 5,677 additions and 310 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,5 @@
 .vscode/*
-!/.vscode/monorepo.code-workspace
+!/.vscode/rrweb-monorepo.code-workspace
 .idea
 node_modules
 package-lock.json

diff --git a/.vscode/monorepo.code-workspace → .vscode/rrweb-monorepo.code-workspace b/.vscode/monorepo.code-workspace → .vscode/rrweb-monorepo.code-workspace
@@ -1,26 +1,31 @@
 {
   "folders": [
     {
-      "name": "Monorepo",
+      "name": " rrweb monorepo", // added a space to bump it to the top
       "path": ".."
     },
     {
+      "name": "rrdom (package)",
       "path": "../packages/rrdom"
     },
     {
+      "name": "rrweb (package)",
       "path": "../packages/rrweb"
     },
     {
+      "name": "rrweb-player (package)",
       "path": "../packages/rrweb-player"
     },
     {
+      "name": "rrweb-snapshot (package)",
       "path": "../packages/rrweb-snapshot"
     }
   ],
   "settings": {
     "jest.disabledWorkspaceFolders": [
-      "Monorepo",
-      "rrweb-player"
+      " rrweb monorepo",
+      "rrweb-player (package)",
+      "rrdom (package)"
     ]
   }
 }
diff --git a/packages/rrweb-snapshot/src/snapshot.ts b/packages/rrweb-snapshot/src/snapshot.ts
@@ -10,8 +10,14 @@ import {
   MaskTextFn,
   MaskInputFn,
   KeepIframeSrcFn,
+  ICanvas,
 } from './types';
-import { isElement, isShadowRoot, maskInputValue } from './utils';
+import {
+  is2DCanvasBlank,
+  isElement,
+  isShadowRoot,
+  maskInputValue,
+} from './utils';
 
 let _id = 1;
 const tagNameRegex = new RegExp('[^a-z0-9-_:]');
@@ -504,7 +510,26 @@ function serializeNode(
       }
       // canvas image data
       if (tagName === 'canvas' && recordCanvas) {
-        attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL();
+        if ((n as ICanvas).__context === '2d') {
+          // only record this on 2d canvas
+          if (!is2DCanvasBlank(n as HTMLCanvasElement)) {
+            attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL();
+          }
+        } else if (!('__context' in n)) {
+          // context is unknown, better not call getContext to trigger it
+          const canvasDataURL = (n as HTMLCanvasElement).toDataURL();
+
+          // create blank canvas of same dimensions
+          const blankCanvas = document.createElement('canvas');
+          blankCanvas.width = (n as HTMLCanvasElement).width;
+          blankCanvas.height = (n as HTMLCanvasElement).height;
+          const blankCanvasDataURL = blankCanvas.toDataURL();
+
+          // no need to save dataURL if it's the same as blank canvas
+          if (canvasDataURL !== blankCanvasDataURL) {
+            attributes.rr_dataURL = canvasDataURL;
+          }
+        }
       }
       // save image offline
       if (tagName === 'img' && inlineImages) {
@@ -592,7 +617,10 @@ function serializeNode(
             );
           }
         } catch (err) {
-          console.warn(`Cannot get CSS styles from text's parentNode. Error: ${err}`, n);
+          console.warn(
+            `Cannot get CSS styles from text's parentNode. Error: ${err}`,
+            n,
+          );
         }
         textContent = absoluteToStylesheet(textContent, getHref());
       }

diff --git a/packages/rrweb-snapshot/src/types.ts b/packages/rrweb-snapshot/src/types.ts
@@ -71,6 +71,10 @@ export interface INode extends Node {
   __sn: serializedNodeWithId;
 }
 
+export interface ICanvas extends HTMLCanvasElement {
+  __context: string;
+}
+
 export type idNodeMap = {
   [key: number]: INode;
 };

diff --git a/packages/rrweb-snapshot/src/utils.ts b/packages/rrweb-snapshot/src/utils.ts
@@ -35,3 +35,40 @@ export function maskInputValue({
   }
   return text;
 }
+
+const ORIGINAL_ATTRIBUTE_NAME = '__rrweb_original__';
+type PatchedGetImageData = {
+  [ORIGINAL_ATTRIBUTE_NAME]: CanvasImageData['getImageData'];
+} & CanvasImageData['getImageData'];
+
+export function is2DCanvasBlank(canvas: HTMLCanvasElement): boolean {
+  const ctx = canvas.getContext('2d');
+  if (!ctx) return true;
+
+  const chunkSize = 50;
+
+  // get chunks of the canvas and check if it is blank
+  for (let x = 0; x < canvas.width; x += chunkSize) {
+    for (let y = 0; y < canvas.height; y += chunkSize) {
+      const getImageData = ctx.getImageData as PatchedGetImageData;
+      const originalGetImageData =
+        ORIGINAL_ATTRIBUTE_NAME in getImageData
+          ? getImageData[ORIGINAL_ATTRIBUTE_NAME]
+          : getImageData;
+      // by getting the canvas in chunks we avoid an expensive
+      // `getImageData` call that retrieves everything
+      // even if we can already tell from the first chunk(s) that
+      // the canvas isn't blank
+      const pixelBuffer = new Uint32Array(
+        originalGetImageData(
+          x,
+          y,
+          Math.min(chunkSize, canvas.width - x),
+          Math.min(chunkSize, canvas.height - y),
+        ).data.buffer,
+      );
+      if (pixelBuffer.some((pixel) => pixel !== 0)) return false;
+    }
+  }
+  return true;
+}
diff --git a/packages/rrweb-snapshot/typings/types.d.ts b/packages/rrweb-snapshot/typings/types.d.ts
@@ -55,6 +55,9 @@ export declare type tagMap = {
 export interface INode extends Node {
     __sn: serializedNodeWithId;
 }
+export interface ICanvas extends HTMLCanvasElement {
+    __context: string;
+}
 export declare type idNodeMap = {
     [key: number]: INode;
 };

diff --git a/packages/rrweb-snapshot/typings/utils.d.ts b/packages/rrweb-snapshot/typings/utils.d.ts
@@ -8,3 +8,4 @@ export declare function maskInputValue({ maskInputOptions, tagName, type, value,
     value: string | null;
     maskInputFn?: MaskInputFn;
 }): string;
+export declare function is2DCanvasBlank(canvas: HTMLCanvasElement): boolean;
diff --git a/packages/rrweb/.gitignore b/packages/rrweb/.gitignore
@@ -13,3 +13,4 @@ temp
 *.log
 
 .env
+__diff_output__
diff --git a/packages/rrweb/jest.config.js b/packages/rrweb/jest.config.js
@@ -3,4 +3,7 @@ module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
   testMatch: ['**/**.test.ts'],
+  moduleNameMapper: {
+    '\\.css$': 'identity-obj-proxy',
+  },
 };
diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json
@@ -46,15 +46,18 @@
     "@types/chai": "^4.1.6",
     "@types/inquirer": "0.0.43",
     "@types/jest": "^27.0.2",
+    "@types/jest-image-snapshot": "^4.3.1",
     "@types/jsdom": "^16.2.12",
     "@types/node": "^12.20.16",
     "@types/prettier": "^2.3.2",
-    "@types/puppeteer": "^5.4.3",
+    "@types/puppeteer": "^5.4.4",
     "cross-env": "^5.2.0",
     "fast-mhtml": "^1.1.9",
+    "identity-obj-proxy": "^3.0.0",
     "ignore-styles": "^5.0.1",
     "inquirer": "^6.2.1",
     "jest": "^27.2.4",
+    "jest-image-snapshot": "^4.5.1",
     "jest-snapshot": "^23.6.0",
     "jsdom": "^17.0.0",
     "jsdom-global": "^3.0.2",
@@ -73,6 +76,7 @@
   "dependencies": {
     "@types/css-font-loading-module": "0.0.7",
     "@xstate/fsm": "^1.4.0",
+    "base64-arraybuffer": "^1.0.1",
     "fflate": "^0.4.4",
     "mitt": "^1.1.3",
     "rrweb-snapshot": "^1.1.12"

diff --git a/packages/rrweb/src/record/index.ts b/packages/rrweb/src/record/index.ts
@@ -18,9 +18,11 @@ import {
   listenerHandler,
   mutationCallbackParam,
   scrollCallback,
+  canvasMutationParam,
 } from '../types';
 import { IframeManager } from './iframe-manager';
 import { ShadowDomManager } from './shadow-dom-manager';
+import { CanvasManager } from './observers/canvas/canvas-manager';
 
 function wrapEvent(e: event): eventWithTime {
   return {
@@ -180,11 +182,28 @@ function record<T = eventWithTime>(
         },
       }),
     );
+  const wrappedCanvasMutationEmit = (p: canvasMutationParam) =>
+    wrappedEmit(
+      wrapEvent({
+        type: EventType.IncrementalSnapshot,
+        data: {
+          source: IncrementalSource.CanvasMutation,
+          ...p,
+        },
+      }),
+    );
 
   const iframeManager = new IframeManager({
     mutationCb: wrappedMutationEmit,
   });
 
+  const canvasManager = new CanvasManager({
+    mutationCb: wrappedCanvasMutationEmit,
+    win: window,
+    blockClass,
+    mirror,
+  });
+
   const shadowDomManager = new ShadowDomManager({
     mutationCb: wrappedMutationEmit,
     scrollCb: wrappedScrollEmit,
@@ -202,6 +221,7 @@ function record<T = eventWithTime>(
       sampling,
       slimDOMOptions,
       iframeManager,
+      canvasManager,
     },
     mirror,
   });
@@ -365,16 +385,7 @@ function record<T = eventWithTime>(
                 },
               }),
             ),
-          canvasMutationCb: (p) =>
-            wrappedEmit(
-              wrapEvent({
-                type: EventType.IncrementalSnapshot,
-                data: {
-                  source: IncrementalSource.CanvasMutation,
-                  ...p,
-                },
-              }),
-            ),
+          canvasMutationCb: wrappedCanvasMutationEmit,
           fontCb: (p) =>
             wrappedEmit(
               wrapEvent({
@@ -404,6 +415,7 @@ function record<T = eventWithTime>(
           mirror,
           iframeManager,
           shadowDomManager,
+          canvasManager,
           plugins:
             plugins?.map((p) => ({
               observer: p.observer,

diff --git a/packages/rrweb/src/record/mutation.ts b/packages/rrweb/src/record/mutation.ts
@@ -31,6 +31,7 @@ import {
   hasShadowRoot,
 } from '../utils';
 import { IframeManager } from './iframe-manager';
+import { CanvasManager } from './observers/canvas/canvas-manager';
 import { ShadowDomManager } from './shadow-dom-manager';
 
 type DoubleLinkedListNode = {
@@ -179,6 +180,7 @@ export default class MutationBuffer {
   private mirror: Mirror;
   private iframeManager: IframeManager;
   private shadowDomManager: ShadowDomManager;
+  private canvasManager: CanvasManager;
 
   public init(
     cb: mutationCallBack,
@@ -197,6 +199,7 @@ export default class MutationBuffer {
     mirror: Mirror,
     iframeManager: IframeManager,
     shadowDomManager: ShadowDomManager,
+    canvasManager: CanvasManager,
   ) {
     this.blockClass = blockClass;
     this.blockSelector = blockSelector;
@@ -214,14 +217,17 @@ export default class MutationBuffer {
     this.mirror = mirror;
     this.iframeManager = iframeManager;
     this.shadowDomManager = shadowDomManager;
+    this.canvasManager = canvasManager;
   }
 
   public freeze() {
     this.frozen = true;
+    this.canvasManager.freeze();
   }
 
   public unfreeze() {
     this.frozen = false;
+    this.canvasManager.unfreeze();
     this.emit();
   }
 
@@ -231,16 +237,22 @@ export default class MutationBuffer {
 
   public lock() {
     this.locked = true;
+    this.canvasManager.lock();
   }
 
   public unlock() {
     this.locked = false;
+    this.canvasManager.unlock();
     this.emit();
   }
 
+  public reset() {
+    this.canvasManager.reset();
+  }
+
   public processMutations = (mutations: mutationRecord[]) => {
-    mutations.forEach(this.processMutation);
-    this.emit();
+    mutations.forEach(this.processMutation); // adds mutations to the buffer
+    this.emit(); // clears buffer if not locked/frozen
   };
 
   public emit = () => {