Merge pull request #1233 from actnwit/feat/vrm-instanced-skinning

feat: support Instanced Skinning of VRM model

samples/test_e2e/GltfImporter-ibl-1/main.ts

@@ -1,4 +1,4 @@
-import Rn from '../../../dist/esm/index.js';
+import Rn from '../../../dist/esmdev/index.js';
 
 declare const window: any;
 const p = document.createElement('p');

samples/test_e2e/VRMAnimation/index.html

@@ -34,6 +34,7 @@ <h2>Correct PNG Image</h2>
       <img src="__image_snapshots__/test-js-regression-test-vrm-animation-1-snap.png" />
     </section>
   </main>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/stats.js/r16/Stats.min.js"></script>
   <script src="../../assets/navigate.js"></script>
   <script type="module" src="./main.js"></script>
 </body>

samples/test_e2e/VRMAnimation/main.ts

@@ -1,18 +1,21 @@
-import Rn from '../../../dist/esm/index.js';
+import Rn from '../../../dist/esmdev/index.js';
 
 let p: any;
 
 declare const window: any;
+declare const Stats: any;
 
 (async () => {
-  Rn.Config.maxEntityNumber = 200;
+  Rn.Config.maxEntityNumber = 40000;
   Rn.Config.maxLightNumberInShader = 1;
   Rn.Config.maxVertexMorphNumberInShader = 1;
   Rn.Config.maxMaterialInstanceForEachType = 30;
   Rn.Config.maxCameraNumber = 3;
   Rn.Config.maxSkeletalBoneNumber = 100;
-  Rn.Config.dataTextureWidth = 2 ** 9;
-  Rn.Config.dataTextureHeight = 2 ** 10;
+  Rn.Config.maxSkeletonNumber = 1504;
+  Rn.Config.maxSkeletalBoneNumberForUniformMode = 100;
+  Rn.Config.dataTextureWidth = 2 ** 12;
+  Rn.Config.dataTextureHeight = 2 ** 11;
   Rn.Config.maxMorphTargetNumber = 1;
   Rn.Config.isUboEnabled = false;
 
@@ -72,6 +75,27 @@ declare const window: any;
     false
   );
 
+  for (let i = 0; i < 1; i++) {
+    for (let j = 0; j < 1; j++) {
+      const vrmRootEntity2nd = Rn.EntityRepository.shallowCopyEntity(
+        vrmRootEntity
+      ) as Rn.ISceneGraphEntity;
+      vrmRootEntity2nd.getTransform().localEulerAngles = Rn.Vector3.fromCopyArray([
+        0,
+        Math.PI,
+        0.0,
+      ]);
+      vrmRootEntity2nd.getTransform().localPosition = Rn.Vector3.fromCopyArray([i, 0, j]);
+      vrmMainRenderPass.addEntities([vrmRootEntity2nd]);
+      animationAssigner.assignAnimation(
+        vrmRootEntity2nd,
+        animGltf2Result.unwrapForce(),
+        vrmModelResult.unwrapForce(),
+        false
+      );
+    }
+  }
+
   //set default camera
   Rn.CameraComponent.current = 0;
 
@@ -82,6 +106,7 @@ declare const window: any;
   const controller = vrmMainCameraControllerComponent.controller as Rn.OrbitCameraController;
   controller.dolly = 0.78;
   controller.setTarget(vrmMainRenderPass.sceneTopLevelGraphComponents[0].entity);
+  // controller.autoUpdate = false;
 
   // Lights
   const lightEntity = Rn.EntityHelper.createLightEntity();
@@ -90,6 +115,10 @@ declare const window: any;
   lightComponent.intensity = Rn.Vector3.fromCopyArray([1.0, 1.0, 1.0]);
   lightEntity.getTransform().localEulerAngles = Rn.Vector3.fromCopyArray([0.0, 0.0, Math.PI / 8]);
 
+  const stats = new Stats();
+  stats.showPanel(0); // 0: fps, 1: ms, 2: mb, 3+: custom
+  document.body.appendChild(stats.domElement);
+
   let count = 0;
   let startTime = Date.now();
   const draw = function () {
@@ -113,7 +142,9 @@ declare const window: any;
       }
     }
 
+    stats.begin();
     Rn.System.process(expressions);
+    stats.end();
 
     count++;
 

src/foundation/cameras/OrbitCameraController.ts

@@ -21,7 +21,9 @@ export class OrbitCameraController extends AbstractCameraController implements I
   public scaleOfLengthCenterToCamera = 1.0;
   public moveSpeed = 1;
   public followTargetAABB = false;
+  public autoUpdate = true;
 
+  private __updated = false;
   private __fixedDolly = false;
   private __fixedLengthOfCenterToEye = 1;
   private __isMouseDown = false;
@@ -116,6 +118,7 @@ export class OrbitCameraController extends AbstractCameraController implements I
   setTarget(targetEntity: ISceneGraphEntity) {
     this.__targetEntity = targetEntity;
     this.__originalTargetAABB = undefined;
+    this.__updated = false;
   }
 
   getTarget(): ISceneGraphEntity | undefined {
@@ -144,6 +147,8 @@ export class OrbitCameraController extends AbstractCameraController implements I
     this.__lastMouseDownTimeStamp = e.timeStamp;
 
     console.log('original', this.__originalX, this.__originalY);
+
+    this.__updated = false;
   }
 
   __mouseMove(e: MouseEvent) {
@@ -189,6 +194,7 @@ export class OrbitCameraController extends AbstractCameraController implements I
     }
     this.__originalX = currentMouseX;
     this.__originalY = currentMouseY;
+    this.__updated = false;
   }
 
   __mouseUp(e: MouseEvent) {
@@ -198,6 +204,7 @@ export class OrbitCameraController extends AbstractCameraController implements I
 
     this.__isMouseDown = false;
     this.__lastMouseUpTimeStamp = e.timeStamp;
+    this.__updated = false;
   }
 
   __touchDown(e: TouchEvent) {
@@ -661,9 +668,12 @@ export class OrbitCameraController extends AbstractCameraController implements I
   }
 
   logic(cameraComponent: CameraComponent) {
-    this.__updateTargeting(cameraComponent);
-    this.__calculateInfluenceOfController();
-    this.__updateCameraComponent(cameraComponent);
+    if (!this.__updated || this.autoUpdate) {
+      this.__updateTargeting(cameraComponent);
+      this.__calculateInfluenceOfController();
+      this.__updateCameraComponent(cameraComponent);
+      this.__updated = true;
+    }
   }
 
   /**

src/foundation/components/Animation/AnimationComponent.ts

@@ -9,7 +9,6 @@ import {
 import { AnimationAttribute } from '../../definitions/AnimationAttribute';
 import { TransformComponent } from '../Transform/TransformComponent';
 import { ProcessStage } from '../../definitions/ProcessStage';
-import { MeshComponent } from '../Mesh/MeshComponent';
 import {
   ComponentTID,
   ComponentSID,

src/foundation/components/CameraController/CameraControllerComponent.ts

@@ -80,7 +80,7 @@ export class CameraControllerComponent extends Component {
     class CameraControllerEntity extends (base.constructor as any) {
       constructor(
         entityUID: EntityUID,
-        isAlive: Boolean,
+        isAlive: boolean,
         components?: Map<ComponentTID, Component>
       ) {
         super(entityUID, isAlive, components);

src/foundation/components/MeshRenderer/MeshRendererComponent.ts

@@ -119,10 +119,10 @@ export class MeshRendererComponent extends Component {
 
     // FrustumCulling
     let primitives: Primitive[] = [];
-    const sceneGraphComponents = renderPass.sceneTopLevelGraphComponents!;
+    const meshComponents = renderPass.meshComponents;
     primitives = MeshRendererComponent.__cullingWithViewFrustum(
       cameraComponent,
-      sceneGraphComponents,
+      meshComponents,
       renderPass
     );
 
@@ -170,65 +170,54 @@ export class MeshRendererComponent extends Component {
 
   private static __cullingWithViewFrustum(
     cameraComponent: CameraComponent,
-    sceneGraphComponents: SceneGraphComponent[],
+    meshComponents: MeshComponent[],
     renderPass: RenderPass
   ) {
-    let meshComponents: MeshComponent[] = [];
+    let filteredMeshComponents: MeshComponent[] = [];
     if (cameraComponent && MeshRendererComponent.isViewFrustumCullingEnabled) {
       cameraComponent.updateFrustum();
 
-      const whetherContainsSkeletal = (sg: SceneGraphComponent): boolean => {
-        const skeletalComponent = sg.entity.tryToGetSkeletal();
-        if (Is.exist(skeletalComponent)) {
-          return true;
-        } else {
-          const children = sg.children;
-          for (const child of children) {
-            return whetherContainsSkeletal(child);
-          }
-          return false;
-        }
-      };
+      // const whetherContainsSkeletal = (sg: SceneGraphComponent): boolean => {
+      //   const skeletalComponent = sg.entity.tryToGetSkeletal();
+      //   if (Is.exist(skeletalComponent)) {
+      //     return true;
+      //   } else {
+      //     const children = sg.children;
+      //     for (const child of children) {
+      //       return whetherContainsSkeletal(child);
+      //     }
+      //     return false;
+      //   }
+      // };
 
       const frustum = cameraComponent.frustum;
-      const doAsVisible = (sg: SceneGraphComponent, meshComponents: MeshComponent[]) => {
-        const sgs = SceneGraphComponent.flattenHierarchy(sg, false);
-        for (const sg of sgs) {
-          const mesh = sg.entity.tryToGetMesh();
-          if (mesh) {
-            meshComponents!.push(mesh);
+      const frustumCulling = (meshComponent: MeshComponent, outMeshComponents: MeshComponent[]) => {
+        const result = frustum.culling(meshComponent);
+        if (result) {
+          outMeshComponents.push(meshComponent);
+          meshComponent.entity.getSceneGraph()._isCulled = false;
+          const skeletal = meshComponent.entity.tryToGetSkeletal();
+          if (skeletal !== undefined) {
+            skeletal._isCulled = false;
           }
-        }
-      };
-      const frustumCullingRecursively = (
-        sg: SceneGraphComponent,
-        meshComponents: MeshComponent[]
-      ) => {
-        const result = frustum.culling(sg);
-        if (result === Visibility.Visible) {
-          doAsVisible(sg, meshComponents);
-        } else if (result === Visibility.Neutral || whetherContainsSkeletal(sg)) {
-          const children = sg.children;
-          const mesh = sg.entity.tryToGetMesh();
-          if (mesh) {
-            meshComponents.push(mesh);
-          }
-          for (const child of children) {
-            frustumCullingRecursively(child, meshComponents);
+        } else {
+          meshComponent.entity.getSceneGraph()._isCulled = true;
+          const skeletal = meshComponent.entity.tryToGetSkeletal();
+          if (skeletal !== undefined) {
+            skeletal._isCulled = true;
           }
         }
       };
-
-      for (const tlsg of sceneGraphComponents) {
-        frustumCullingRecursively(tlsg, meshComponents);
+      for (const meshComponent of meshComponents) {
+        frustumCulling(meshComponent, filteredMeshComponents);
       }
     } else {
-      meshComponents = renderPass!.meshComponents!;
+      filteredMeshComponents = renderPass!.meshComponents!;
     }
 
     const primitives: Primitive[] = [];
-    for (let i = 0; i < meshComponents.length; i++) {
-      const meshComponent = meshComponents[i];
+    for (let i = 0; i < filteredMeshComponents.length; i++) {
+      const meshComponent = filteredMeshComponents[i];
       const viewDepth = meshComponent.calcViewDepth(cameraComponent);
       const mesh = meshComponent.mesh;
       if (mesh !== undefined) {

src/foundation/components/SceneGraph/SceneGraphComponent.ts

@@ -52,13 +52,15 @@ export class SceneGraphComponent extends Component {
   // Skeletal
   public isRootJoint = false;
   public jointIndex = -1;
+  _isCulled = false;
   private static readonly __originVector3 = Vector3.zero();
   private static returnVector3 = MutableVector3.zero();
   private static __sceneGraphs: SceneGraphComponent[] = [];
   private static isJointAABBShouldBeCalculated = false;
   private static invertedMatrix44 = MutableMatrix44.fromCopyArray16ColumnMajor([
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   ]);
+  private static __tmpAABB = new AABB();
 
   constructor(
     entityUid: EntityUID,
@@ -237,6 +239,18 @@ export class SceneGraphComponent extends Component {
     return WellKnownComponentTIDs.SceneGraphComponentTID;
   }
 
+  setWorldMatrixRestDirty() {
+    this.matrixRestInner;
+    this.setWorldMatrixRestDirtyRecursively();
+  }
+
+  setWorldMatrixRestDirtyRecursively() {
+    this.__isWorldMatrixRestUpToDate = false;
+    this.children.forEach((child) => {
+      child.setWorldMatrixRestDirtyRecursively();
+    });
+  }
+
   setWorldMatrixDirty() {
     this.setWorldMatrixDirtyRecursively();
     this.parent?.setWorldAABBDirtyParentRecursively();
@@ -458,6 +472,25 @@ export class SceneGraphComponent extends Component {
 
   calcWorldAABB() {
     this.__worldAABB.initialize();
+
+    // const meshComponent = this.entity.tryToGetMesh();
+    // for (let i = 0; i < this.children.length; i++) {
+    //   this.__worldAABB.mergeAABB(this.children[i].worldAABB);
+    // }
+    // if (meshComponent != null) {
+    //   if (meshComponent.mesh != null) {
+    //     this.__worldAABB.mergeAABB(meshComponent.mesh.AABB);
+    //   }
+    // }
+    // AABB.multiplyMatrixTo(
+    //   this.entity.tryToGetTransform()!.localMatrixInner,
+    //   this.__worldAABB,
+    //   SceneGraphComponent.__tmpAABB
+    // );
+
+    // this.__worldAABB = SceneGraphComponent.__tmpAABB.clone();
+    // return this.__worldAABB;
+
     const aabb = (function mergeAABBRecursively(elem: SceneGraphComponent): AABB {
       const meshComponent = elem.entity.tryToGetMesh();
       if (Is.exist(meshComponent) && Is.exist(meshComponent.mesh)) {
@@ -641,8 +674,7 @@ export class SceneGraphComponent extends Component {
   }
 
   $logic() {
-    this._worldMatrix.copyComponents(this.__calcWorldMatrixRecursively());
-    this._worldMatrixRest.copyComponents(this.__calcWorldMatrixRestRecursively());
+    this.matrixInner;
 
     this.__updateGizmos();
 

src/foundation/components/Skeletal/SkeletalComponent.ts

@@ -41,6 +41,7 @@ export class SkeletalComponent extends Component {
   private __worldMatrix = MutableMatrix44.identity();
   private __isWorldMatrixVanilla = true;
   _animationRetarget?: IAnimationRetarget;
+  _isCulled = false;
   private static __globalDataRepository = GlobalDataRepository.getInstance();
   private static __tookGlobalDataNum = 0;
   private static __tmpVec3_0 = MutableVector3.zero();
@@ -191,7 +192,7 @@ export class SkeletalComponent extends Component {
   }
 
   $logic() {
-    if (!this.isSkinning) {
+    if (!this.isSkinning || this._isCulled) {
       return;
     }