fix(🖼️): improved scene graph model (RN Web only for now) (#2863)

This is a new model where use a Picture recorder that supports animation values.
Consider the following deeply nested tree:
```jsx
<Image x={0} y={0} width={width} height={height} image={oslo} fit="cover">
  <ColorMatrix matrix={matrix} />
  <LinearToSRGBGamma>
    <Lerp t={t}>
      <ColorMatrix matrix={purple} />
      <ColorMatrix matrix={blackAndWhite} />
    </Lerp>
  </LinearToSRGBGamma>
</Image>
```

This will generate the following instruction set:

```
[
  { type: 'SaveCTM', props: { transform: [Array] } },
  { type: 'SavePaint', props: {} },
  {
    type: 'PushColorFilter',
    colorFilterType: 'skMatrixColorFilter',
    props: { matrix: [Array] }
  },
  {
    type: 'PushColorFilter',
    colorFilterType: 'skMatrixColorFilter',
    props: { matrix: [Array] }
  },
  {
    type: 'PushColorFilter',
    colorFilterType: 'skMatrixColorFilter',
    props: { matrix: [Array] }
  },
  {
    type: 'PushColorFilter',
    colorFilterType: 'skLerpColorFilter',
    props: { t: 0.5 }
  },
  {
    type: 'PushColorFilter',
    colorFilterType: 'skLinearToSRGBGammaColorFilter',
    props: {}
  },
  { type: 'ComposeColorFilter' },
  { type: 'MaterializePaint' },
  {
    type: 'DrawImage',
    props: {
      x: 0,
      y: 0,
      width: 256,
      height: 256,
      image: [JsiSkImage],
      fit: 'cover'
    }
  },
  { type: 'RestorePaint' },
  { type: 'RestoreCTM' }
]
```

apps/paper/src/Examples/Matrix/Matrix.tsx

@@ -44,7 +44,7 @@ export const Matrix = () => {
     <Canvas style={{ flex: 1 }} opaque>
       <Fill color="black" />
       <Group>
-        <BlurMask blur={8} style="solid" />
+        <BlurMask blur={4} style="solid" />
         {cols.map((_i, i) =>
           rows.map((_j, j) => (
             <Symbol

apps/paper/src/Examples/Matrix/Symbol.tsx

@@ -38,7 +38,7 @@ export const Symbol = ({
   }, [timestamp]);
 
   const opacity = useDerivedValue(() => {
-    const idx = Math.round(timestamp.value / 100);
+    const idx = Math.round(timestamp.value / 75);
     return stream[(stream.length - j + idx) % stream.length];
   }, [timestamp]);
 

packages/skia/cpp/api/JsiSkPaint.h

@@ -27,6 +27,12 @@ class JsiSkPaint : public JsiSkWrappingSharedPtrHostObject<SkPaint> {
 public:
   EXPORT_JSI_API_TYPENAME(JsiSkPaint, Paint)
 
+  JSI_HOST_FUNCTION(assign) {
+    SkPaint* paint = JsiSkPaint::fromValue(runtime, arguments[0]).get();
+    *getObject() = *paint;
+    return jsi::Value::undefined();
+  }
+
   JSI_HOST_FUNCTION(copy) {
     const auto *paint = getObject().get();
     return jsi::Object::createFromHostObject(
@@ -163,7 +169,8 @@ class JsiSkPaint : public JsiSkWrappingSharedPtrHostObject<SkPaint> {
     return jsi::Value::undefined();
   }
 
-  JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkPaint, copy),
+  JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkPaint, assign),
+                       JSI_EXPORT_FUNC(JsiSkPaint, copy),
                        JSI_EXPORT_FUNC(JsiSkPaint, reset),
                        JSI_EXPORT_FUNC(JsiSkPaint, getAlphaf),
                        JSI_EXPORT_FUNC(JsiSkPaint, getColor),

packages/skia/src/dom/types/ImageFilters.ts

@@ -24,8 +24,9 @@ export interface RuntimeShaderImageFilterProps extends ChildrenProps {
   uniforms?: Uniforms;
 }
 
+// TODO: delete
 export interface BlendImageFilterProps extends ChildrenProps {
-  mode: BlendMode;
+  mode: SkEnum<typeof BlendMode>;
 }
 
 export interface MorphologyImageFilterProps extends ChildrenProps {

packages/skia/src/dom/types/Node.ts

@@ -1,6 +1,5 @@
 import type { GroupProps } from "./Common";
 import type { NodeType } from "./NodeType";
-import type { DeclarationContext } from "../../sksg/DeclarationContext";
 
 export interface Node<P> {
   type: NodeType;
@@ -18,9 +17,6 @@ export interface Node<P> {
 export type Invalidate = () => void;
 
 export interface DeclarationNode<P> extends Node<P> {
-  //declarationType: DeclarationType;
-  decorate(ctx: DeclarationContext): void;
-
   setInvalidate(invalidate: Invalidate): void;
 }
 

packages/skia/src/renderer/__tests__/e2e/ColorFilters.spec.tsx

@@ -14,6 +14,7 @@ import {
 import { docPath, checkImage, processResult } from "../../../__tests__/setup";
 import { setupSkia } from "../../../skia/__tests__/setup";
 import { fitRects } from "../../../dom/nodes";
+import { BlendMode } from "../../../skia/types";
 
 const blackAndWhite = [
   0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0,
@@ -57,6 +58,20 @@ describe("Color Filters", () => {
     );
     checkImage(img, docPath("color-filters/color-blend.png"));
   });
+  it("should build the reference result for should use composition", async () => {
+    const { surface: ckSurface, Skia, canvas } = setupSkia(wWidth, wHeight);
+    const paint = Skia.Paint();
+    const outer = Skia.ColorFilter.MakeSRGBToLinearGamma();
+    const inner = Skia.ColorFilter.MakeBlend(
+      Skia.Color("lightblue"),
+      BlendMode.SrcIn
+    );
+    paint.setColorFilter(Skia.ColorFilter.MakeCompose(outer, inner));
+    const r = (surface.width * 3) / 2;
+    canvas.drawCircle(r, r, r, paint);
+    canvas.drawCircle(r * 2, r, r, paint);
+    processResult(ckSurface, docPath("color-filters/composition.png"));
+  });
   it("should use composition", async () => {
     const { width } = surface;
     const r = width / 2;

packages/skia/src/renderer/__tests__/e2e/Composition/ColorFilterComposition.spec.tsx

@@ -7,12 +7,20 @@ import {
   height as wHeight,
 } from "../../setup";
 import {
+  BlendColor,
+  Circle,
   ColorMatrix,
+  Group,
   Image,
   Lerp,
   LinearToSRGBGamma,
+  SRGBToLinearGamma,
 } from "../../../components";
-import { checkImage, processResult } from "../../../../__tests__/setup";
+import {
+  checkImage,
+  docPath,
+  processResult,
+} from "../../../../__tests__/setup";
 import { setupSkia } from "../../../../skia/__tests__/setup";
 import { fitRects } from "../../../../dom/nodes";
 
@@ -60,7 +68,6 @@ describe("Color Filter Composition", () => {
       "snapshots/color-filter/color-filter-composition.png"
     );
   });
-  // TODO: a bug should be reported here
   it("should apply a color matrix to an image", async () => {
     const { oslo } = images;
     const { width, height } = surface;
@@ -78,4 +85,18 @@ describe("Color Filter Composition", () => {
     );
     checkImage(image, "snapshots/color-filter/color-filter-composition.png");
   });
+  it("should use composition", async () => {
+    const { width } = surface;
+    const r = width / 2;
+    const img = await surface.draw(
+      <Group>
+        <SRGBToLinearGamma>
+          <BlendColor color="lightblue" mode="srcIn" />
+        </SRGBToLinearGamma>
+        <Circle cx={r} cy={r} r={r} />
+        <Circle cx={2 * r} cy={r} r={r} color="red" />
+      </Group>
+    );
+    checkImage(img, docPath("color-filters/composition.png"));
+  });
 });

packages/skia/src/renderer/__tests__/e2e/ImageFilters.spec.tsx

@@ -6,8 +6,15 @@ import {
   itRunsNodeOnly,
   itRunsE2eOnly,
   CI,
+  processResult,
 } from "../../../__tests__/setup";
-import { fonts, images, surface } from "../setup";
+import {
+  fonts,
+  images,
+  surface,
+  width as wWidth,
+  height as wHeight,
+} from "../setup";
 import {
   Fill,
   Image,
@@ -20,6 +27,51 @@ import {
   DisplacementMap,
   Turbulence,
 } from "../../components";
+import { setupSkia } from "../../../skia/__tests__/setup";
+import {
+  BlendMode,
+  TileMode,
+  type SkColor,
+  type Skia,
+  type SkImageFilter,
+} from "../../../skia/types";
+
+const Black = Float32Array.of(0, 0, 0, 1);
+
+const MakeInnerShadow = (
+  Skia: Skia,
+  shadowOnly: boolean | undefined,
+  dx: number,
+  dy: number,
+  sigmaX: number,
+  sigmaY: number,
+  color: SkColor,
+  input: SkImageFilter | null
+) => {
+  "worklet";
+  const sourceGraphic = Skia.ImageFilter.MakeColorFilter(
+    Skia.ColorFilter.MakeBlend(Black, BlendMode.Dst),
+    null
+  );
+  const sourceAlpha = Skia.ImageFilter.MakeColorFilter(
+    Skia.ColorFilter.MakeBlend(Black, BlendMode.SrcIn),
+    null
+  );
+  const f1 = Skia.ImageFilter.MakeColorFilter(
+    Skia.ColorFilter.MakeBlend(color, BlendMode.SrcOut),
+    null
+  );
+  const f2 = Skia.ImageFilter.MakeOffset(dx, dy, f1);
+  const f3 = Skia.ImageFilter.MakeBlur(sigmaX, sigmaY, TileMode.Decal, f2);
+  const f4 = Skia.ImageFilter.MakeBlend(BlendMode.SrcIn, sourceAlpha, f3);
+  if (shadowOnly) {
+    return f4;
+  }
+  return Skia.ImageFilter.MakeCompose(
+    input,
+    Skia.ImageFilter.MakeBlend(BlendMode.SrcOver, sourceGraphic, f4)
+  );
+};
 
 describe("Test Image Filters", () => {
   itRunsNodeOnly(
@@ -191,6 +243,41 @@ describe("Test Image Filters", () => {
       threshold: 0.05,
     });
   });
+  it("should build reference result outer and inner shadows on text", () => {
+    const { surface: ckSurface, canvas, Skia } = setupSkia(wWidth, wHeight);
+    const path = Skia.Path.MakeFromSVGString(
+      // eslint-disable-next-line max-len
+      "M62.477 2.273V75h-8.522L14.324 17.898h-.71V75H4.807V2.273h8.522l39.773 57.244h.71V2.273h8.665ZM78.963 75V20.454h8.381V75h-8.38Zm4.262-63.636c-1.634 0-3.042-.557-4.226-1.67-1.16-1.112-1.74-2.45-1.74-4.012 0-1.563.58-2.9 1.74-4.013C80.183.556 81.59 0 83.225 0c1.633 0 3.03.556 4.19 1.669 1.184 1.113 1.776 2.45 1.776 4.013 0 1.562-.592 2.9-1.776 4.013-1.16 1.112-2.557 1.669-4.19 1.669ZM124.853 76.136c-5.114 0-9.517-1.207-13.21-3.622-3.693-2.415-6.534-5.74-8.523-9.978-1.989-4.238-2.983-9.08-2.983-14.525 0-5.54 1.018-10.428 3.054-14.666 2.06-4.261 4.924-7.587 8.594-9.979 3.693-2.414 8.002-3.622 12.926-3.622 3.835 0 7.292.71 10.369 2.131 3.078 1.42 5.599 3.41 7.564 5.966 1.965 2.557 3.184 5.54 3.658 8.949h-8.381c-.639-2.486-2.059-4.688-4.261-6.605-2.178-1.942-5.114-2.912-8.807-2.912-3.267 0-6.132.852-8.594 2.556-2.438 1.681-4.344 4.06-5.717 7.138-1.35 3.054-2.024 6.641-2.024 10.76 0 4.214.663 7.884 1.988 11.009 1.35 3.125 3.244 5.551 5.682 7.28 2.462 1.728 5.351 2.592 8.665 2.592 2.178 0 4.155-.379 5.93-1.136a12.23 12.23 0 0 0 4.51-3.267c1.231-1.42 2.107-3.125 2.628-5.114h8.381c-.474 3.22-1.646 6.12-3.516 8.7-1.846 2.557-4.297 4.593-7.351 6.108-3.03 1.492-6.557 2.237-10.582 2.237ZM181.423 76.136c-5.256 0-9.79-1.16-13.601-3.48-3.788-2.344-6.712-5.61-8.772-9.8-2.036-4.215-3.054-9.116-3.054-14.703s1.018-10.511 3.054-14.772c2.06-4.285 4.925-7.623 8.594-10.015 3.693-2.414 8.002-3.622 12.926-3.622 2.841 0 5.647.474 8.417 1.42 2.769.948 5.291 2.487 7.563 4.617 2.273 2.107 4.084 4.9 5.434 8.38 1.349 3.481 2.024 7.766 2.024 12.856v3.551h-42.046v-7.244h33.523c0-3.078-.615-5.824-1.847-8.239-1.207-2.415-2.935-4.32-5.184-5.717-2.226-1.397-4.853-2.095-7.884-2.095-3.338 0-6.226.828-8.664 2.486a16.35 16.35 0 0 0-5.576 6.392c-1.302 2.627-1.953 5.445-1.953 8.451v4.83c0 4.12.71 7.611 2.131 10.476 1.444 2.84 3.444 5.007 6.001 6.498 2.557 1.468 5.528 2.202 8.914 2.202 2.201 0 4.19-.308 5.965-.923 1.8-.64 3.35-1.587 4.652-2.841 1.303-1.279 2.309-2.865 3.019-4.759l8.097 2.273a17.965 17.965 0 0 1-4.297 7.244c-2.013 2.06-4.498 3.67-7.458 4.83-2.959 1.136-6.285 1.704-9.978 1.704ZM32.648 123.273h8.806v51.988c0 4.641-.852 8.582-2.556 11.826-1.705 3.243-4.108 5.705-7.21 7.386-3.1 1.681-6.758 2.521-10.972 2.521-3.977 0-7.517-.722-10.618-2.166-3.101-1.468-5.54-3.551-7.315-6.25-1.776-2.699-2.664-5.907-2.664-9.623h8.665c0 2.059.51 3.858 1.527 5.397 1.042 1.515 2.462 2.699 4.261 3.551 1.8.853 3.848 1.279 6.144 1.279 2.533 0 4.687-.533 6.463-1.598 1.776-1.066 3.125-2.628 4.048-4.688.947-2.083 1.42-4.628 1.42-7.635v-51.988ZM80.126 197.136c-4.924 0-9.244-1.172-12.961-3.515-3.693-2.344-6.582-5.623-8.665-9.837-2.06-4.214-3.09-9.138-3.09-14.773 0-5.681 1.03-10.641 3.09-14.879 2.083-4.238 4.972-7.528 8.665-9.872 3.717-2.344 8.037-3.516 12.961-3.516 4.925 0 9.233 1.172 12.927 3.516 3.716 2.344 6.605 5.634 8.664 9.872 2.084 4.238 3.125 9.198 3.125 14.879 0 5.635-1.041 10.559-3.125 14.773-2.06 4.214-4.948 7.493-8.664 9.837-3.694 2.343-8.002 3.515-12.927 3.515Zm0-7.528c3.741 0 6.819-.959 9.233-2.876 2.415-1.918 4.203-4.439 5.363-7.564 1.16-3.125 1.74-6.511 1.74-10.157 0-3.645-.58-7.043-1.74-10.191-1.16-3.149-2.948-5.694-5.363-7.635-2.414-1.942-5.492-2.912-9.233-2.912-3.74 0-6.818.97-9.233 2.912-2.414 1.941-4.202 4.486-5.362 7.635-1.16 3.148-1.74 6.546-1.74 10.191 0 3.646.58 7.032 1.74 10.157 1.16 3.125 2.948 5.646 5.362 7.564 2.415 1.917 5.493 2.876 9.233 2.876ZM118.772 196v-72.727h8.38v26.846h.711c.615-.947 1.467-2.154 2.556-3.622 1.113-1.491 2.699-2.817 4.759-3.977 2.083-1.184 4.9-1.776 8.452-1.776 4.592 0 8.641 1.149 12.145 3.445 3.503 2.296 6.238 5.552 8.203 9.766 1.965 4.214 2.947 9.185 2.947 14.914 0 5.777-.982 10.784-2.947 15.022-1.965 4.214-4.688 7.481-8.168 9.801-3.48 2.296-7.493 3.444-12.038 3.444-3.504 0-6.31-.58-8.417-1.74-2.107-1.183-3.728-2.521-4.865-4.012-1.136-1.516-2.012-2.77-2.627-3.765h-.995V196h-8.096Zm8.238-27.273c0 4.12.604 7.754 1.811 10.902 1.208 3.125 2.971 5.576 5.292 7.351 2.32 1.752 5.161 2.628 8.522 2.628 3.504 0 6.428-.923 8.772-2.77 2.367-1.87 4.143-4.38 5.326-7.528 1.208-3.173 1.811-6.7 1.811-10.583 0-3.835-.591-7.291-1.775-10.369-1.16-3.101-2.924-5.552-5.291-7.351-2.344-1.823-5.292-2.734-8.843-2.734-3.409 0-6.273.864-8.593 2.592-2.321 1.705-4.072 4.096-5.256 7.173-1.184 3.054-1.776 6.617-1.776 10.689ZM190.824 123.273l-.71 52.272h-8.239l-.71-52.272h9.659Zm-4.829 73.295c-1.752 0-3.256-.627-4.51-1.882-1.255-1.255-1.882-2.758-1.882-4.51 0-1.752.627-3.255 1.882-4.51 1.254-1.255 2.758-1.882 4.51-1.882 1.752 0 3.255.627 4.51 1.882 1.254 1.255 1.882 2.758 1.882 4.51a6.02 6.02 0 0 1-.888 3.196 6.634 6.634 0 0 1-2.308 2.344c-.947.568-2.013.852-3.196.852Z"
+    )!;
+
+    const paint = Skia.Paint();
+    paint.setColor(Skia.Color("#add8e6"));
+    canvas.drawPaint(paint);
+    paint.setColor(Skia.Color("red"));
+    const img1 = MakeInnerShadow(
+      Skia,
+      false,
+      0,
+      4,
+      1,
+      1,
+      Skia.Color("#00FF00"),
+      null
+    );
+    const img2 = Skia.ImageFilter.MakeDropShadow(
+      0,
+      4,
+      0,
+      0,
+      Skia.Color("#0000ff"),
+      null
+    );
+    paint.setImageFilter(Skia.ImageFilter.MakeCompose(img1, img2));
+    canvas.scale(3, 3);
+    canvas.drawPath(path, paint);
+    canvas.restore();
+    processResult(ckSurface, "snapshots/image-filter/test-shadow.png");
+  });
   it("should show outer and inner shadows on text", async () => {
     const path =
       // eslint-disable-next-line max-len
@@ -205,7 +292,9 @@ describe("Test Image Filters", () => {
         </Path>
       </>
     );
-    checkImage(img, "snapshots/image-filter/test-shadow.png");
+    checkImage(img, "snapshots/image-filter/test-shadow.png", {
+      maxPixelDiff: 1500,
+    });
   });
   itRunsE2eOnly("use the displacement map as documented", async () => {
     const { oslo } = images;

packages/skia/src/renderer/__tests__/setup.tsx

@@ -14,7 +14,7 @@ import { isPath } from "../../skia/types";
 import { E2E } from "../../__tests__/setup";
 import { LoadSkiaWeb } from "../../web/LoadSkiaWeb";
 import { SkiaSGRoot } from "../../sksg/Reconciler";
-import type { Node } from "../../sksg/nodes";
+import type { Node } from "../../sksg/Node";
 
 import { SkiaObject } from "./e2e/setup";
 

packages/skia/src/skia/types/Paint/Paint.ts

@@ -45,6 +45,8 @@ export interface SkPaint extends SkJSIInstance<"Paint"> {
    */
   reset(): void;
 
+  assign(paint: SkPaint): void;
+
   /**
    * Retrieves the alpha and RGB unpremultiplied. RGB are extended sRGB values
    * (sRGB gamut, and encoded with the sRGB transfer function).

packages/skia/src/skia/web/JsiSkPaint.ts

@@ -34,6 +34,10 @@ export class JsiSkPaint extends HostObject<Paint, "Paint"> implements SkPaint {
     return new JsiSkPaint(this.CanvasKit, this.ref.copy());
   }
 
+  assign(paint: JsiSkPaint) {
+    this.ref = paint.ref.copy();
+  }
+
   reset() {
     this.ref = new this.CanvasKit.Paint();
   }

packages/skia/src/sksg/Container.ts

@@ -3,26 +3,33 @@ import { type SharedValue } from "react-native-reanimated";
 import Rea from "../external/reanimated/ReanimatedProxy";
 import type { Skia, SkCanvas } from "../skia/types";
 
-import { createDrawingContext } from "./DrawingContext";
-import type { Node } from "./nodes";
-import { draw, isSharedValue } from "./nodes";
+import type { Node } from "./Node";
+import { isSharedValue } from "./utils";
+import { Recorder } from "./Recorder/Recorder";
+import { visit } from "./Recorder/Visitor";
+import { replay } from "./Recorder/Player";
+import { createDrawingContext } from "./Recorder/DrawingContext";
+import { createRecording, type Recording } from "./Recorder/Recording";
 
-const drawOnscreen = (Skia: Skia, nativeId: number, root: Node[]) => {
+const drawOnscreen = (Skia: Skia, nativeId: number, recording: Recording) => {
   "worklet";
   const rec = Skia.PictureRecorder();
   const canvas = rec.beginRecording();
-  // TODO: This is only support from 3.15 and above (check the exact version)
-  // This could be polyfilled in C++ if needed (or in JS via functions only?)
-  const ctx = createDrawingContext(Skia, canvas);
-  root.forEach((node) => {
-    draw(ctx, node);
-  });
+  // const start = performance.now();
+
+  // TODO: because the pool is not a shared value here, it is copied on every frame
+  const ctx = createDrawingContext(Skia, recording.paintPool, canvas);
+  //console.log(recording.commands);
+  replay(ctx, recording.commands);
   const picture = rec.finishRecordingAsPicture();
+  //const end = performance.now();
+  //console.log("Recording time: ", end - start);
   SkiaViewApi.setJsiProperty(nativeId, "picture", picture);
 };
 
 export class Container {
-  public _root: Node[] = [];
+  private _root: Node[] = [];
+  private _recording: Recording | null = null;
   public unmounted = false;
 
   private values = new Set<SharedValue<unknown>>();
@@ -40,13 +47,16 @@ export class Container {
       if (this.mapperId !== null) {
         Rea.stopMapper(this.mapperId);
       }
-      const { nativeId, Skia } = this;
+      const { nativeId, Skia, _recording } = this;
       this.mapperId = Rea.startMapper(() => {
         "worklet";
-        drawOnscreen(Skia, nativeId, root);
+        drawOnscreen(Skia, nativeId, _recording!);
       }, Array.from(this.values));
     }
     this._root = root;
+    const recorder = new Recorder();
+    visit(recorder, root);
+    this._recording = createRecording(recorder.commands);
   }
 
   clear() {
@@ -56,9 +66,9 @@ export class Container {
   redraw() {
     const isOnscreen = this.nativeId !== -1;
     if (isOnscreen) {
-      const { nativeId, Skia, root } = this;
+      const { nativeId, Skia, _recording } = this;
       Rea.runOnUI(() => {
-        drawOnscreen(Skia, nativeId, root);
+        drawOnscreen(Skia, nativeId, _recording!);
       })();
     }
   }
@@ -84,9 +94,15 @@ export class Container {
   }
 
   drawOnCanvas(canvas: SkCanvas) {
-    const ctx = createDrawingContext(this.Skia, canvas);
-    this.root.forEach((node) => {
-      draw(ctx, node);
-    });
+    if (!this._recording) {
+      throw new Error("No recording to draw");
+    }
+    const ctx = createDrawingContext(
+      this.Skia,
+      this._recording.paintPool,
+      canvas
+    );
+    //console.log(this._recording);
+    replay(ctx, this._recording.commands);
   }
 }