From ba3b7dcef7b403750c10b5593df37f7792182e46 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 25 Oct 2023 14:56:44 +0100 Subject: [PATCH 1/4] store splat instances --- src/splat/sort-manager.ts | 5 ++++ src/splat/splat-resource.ts | 59 ++++++++++++++++++++++++------------- src/splat/splat.ts | 33 +++++++++++---------- 3 files changed, 62 insertions(+), 35 deletions(-) diff --git a/src/splat/sort-manager.ts b/src/splat/sort-manager.ts index 21dd942..e810afe 100644 --- a/src/splat/sort-manager.ts +++ b/src/splat/sort-manager.ts @@ -239,6 +239,11 @@ class SortManager { }; } + destroy() { + this.worker.terminate(); + this.worker = null; + } + sort(vertexBuffer: VertexBuffer, centers: Float32Array, intIndices: boolean, updatedCallback?: () => void) { this.vertexBuffer = vertexBuffer; this.updatedCallback = updatedCallback; diff --git a/src/splat/splat-resource.ts b/src/splat/splat-resource.ts index f92dc13..3bf3f45 100644 --- a/src/splat/splat-resource.ts +++ b/src/splat/splat-resource.ts @@ -19,34 +19,50 @@ const mat = new Mat4(); const pos = new Vec3(); const dir = new Vec3(); +const debugRender = false; const debugRenderBounds = false; class SplatResource extends ContainerResource { device: GraphicsDevice; splatData: SplatData; - sortManager: SortManager; - focalPoint = new Vec3(); - entity: Entity; + customAabb = new BoundingBox(); renders: RenderComponent[] = []; meshes: Mesh[] = []; materials: Material[] = []; textures: Texture[] = []; - handle: any; + // per-instance data (currently only a single instance is supported) + instances: { + splat: Splat, + sortManager: SortManager, + entity: Entity, + callbackHandle: any + }[] = []; constructor(device: GraphicsDevice, splatData: SplatData) { super(); this.device = device; this.splatData = splatData; + + // calculate splat focal point + this.splatData.calcFocalPoint(this.focalPoint); + + // calculate custom aabb + this.splatData.calcAabb(this.customAabb); } destroy() { - this.handle.off(); + this.instances.forEach((instance) => { + instance.splat.destroy(); + instance.sortManager.destroy(); + instance.entity.destroy(); + instance.callbackHandle.off(); + }); - // TODO + this.splatData = null; } instantiateModelEntity(/* options: any */): Entity { @@ -55,7 +71,7 @@ class SplatResource extends ContainerResource { instantiateRenderEntity(options: any): Entity { const splatData = this.splatData; - const splat = new Splat(this.device, splatData.numSplats, false); + const splat = new Splat(this.device, splatData.numSplats, options?.debugRender ?? debugRender); splat.updateColorData(splatData.getProp('f_dc_0'), splatData.getProp('f_dc_1'), splatData.getProp('f_dc_2'), splatData.getProp('opacity')); splat.updateScaleData(splatData.getProp('scale_0'), splatData.getProp('scale_1'), splatData.getProp('scale_2')); @@ -69,14 +85,8 @@ class SplatResource extends ContainerResource { castShadows: false // shadows not supported }); - this.entity = result; - // set custom aabb - const customAabb = new BoundingBox(); - this.splatData.calcAabb(customAabb); - result.render.customAabb = customAabb; - - this.splatData.calcFocalPoint(this.focalPoint); + result.render.customAabb = this.customAabb; // centers - constant buffer that is sent to the worker const x = this.splatData.getProp('x'); @@ -91,8 +101,8 @@ class SplatResource extends ContainerResource { } // initialize sort - this.sortManager = new SortManager(); - this.sortManager.sort( + const sortManager = new SortManager(); + sortManager.sort( splat.meshInstance.instancingData.vertexBuffer, centers, this.device.isWebGPU, @@ -101,17 +111,17 @@ class SplatResource extends ContainerResource { const viewport = [0, 0]; - this.handle = options.app.on('prerender', () => { + const callbackHandle = options.app.on('prerender', () => { const cameraMat = options.camera.getWorldTransform(); cameraMat.getTranslation(pos); cameraMat.getZ(dir); - const modelMat = this.entity.getWorldTransform(); + const modelMat = result.getWorldTransform(); const invModelMat = mat.invert(modelMat); invModelMat.transformPoint(pos, pos); invModelMat.transformVector(dir, dir); - this.sortManager.setCamera(pos, dir); + sortManager.setCamera(pos, dir); viewport[0] = this.device.width; viewport[1] = this.device.height; @@ -123,11 +133,20 @@ class SplatResource extends ContainerResource { } }); + // store instance + this.instances.push({ + splat, + sortManager, + entity: result, + callbackHandle + }); + return result; } getFocalPoint(): Vec3 { - return this.entity.getWorldTransform().transformPoint(this.focalPoint); + const instance = this.instances[0]; + return instance?.entity.getWorldTransform().transformPoint(this.focalPoint); } } diff --git a/src/splat/splat.ts b/src/splat/splat.ts index 957cd72..1a36751 100644 --- a/src/splat/splat.ts +++ b/src/splat/splat.ts @@ -82,7 +82,6 @@ const float2Half = (value: number) => { return bits; }; - const evalTextureSize = (count: number) : Vec2 => { const width = Math.ceil(Math.sqrt(count)); const height = Math.ceil(count / width); @@ -123,7 +122,6 @@ const getTextureFormat = (device: GraphicsDevice, preferHighPrecision: boolean) }; class Splat { - device: GraphicsDevice; numSplats: number; material: Material; mesh: Mesh; @@ -136,7 +134,6 @@ class Splat { centerTexture: Texture; constructor(device: GraphicsDevice, numSplats: number, debugRender = false) { - this.device = device; this.numSplats = numSplats; // material @@ -144,11 +141,11 @@ class Splat { // mesh if (debugRender) { - this.mesh = createBox(this.device, { + this.mesh = createBox(device, { halfExtents: new Vec3(1.0, 1.0, 1.0) }); } else { - this.mesh = new Mesh(this.device); + this.mesh = new Mesh(device); this.mesh.setPositions(new Float32Array([ -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1 ]), 2); @@ -156,12 +153,12 @@ class Splat { } // mesh instance - const vertexFormat = new VertexFormat(this.device, [ - { semantic: SEMANTIC_ATTR13, components: 1, type: this.device.isWebGPU ? TYPE_UINT32 : TYPE_FLOAT32 } + const vertexFormat = new VertexFormat(device, [ + { semantic: SEMANTIC_ATTR13, components: 1, type: device.isWebGPU ? TYPE_UINT32 : TYPE_FLOAT32 } ]); const vertexBuffer = new VertexBuffer( - this.device, + device, vertexFormat, numSplats, BUFFER_DYNAMIC, @@ -175,10 +172,10 @@ class Splat { const size = evalTextureSize(numSplats); this.format = getTextureFormat(device, false); - this.colorTexture = createTexture(this.device, 'splatColor', PIXELFORMAT_RGBA8, size); - this.scaleTexture = createTexture(this.device, 'splatScale', this.format.format, size); - this.rotationTexture = createTexture(this.device, 'splatRotation', this.format.format, size); - this.centerTexture = createTexture(this.device, 'splatCenter', this.format.format, size); + this.colorTexture = createTexture(device, 'splatColor', PIXELFORMAT_RGBA8, size); + this.scaleTexture = createTexture(device, 'splatScale', this.format.format, size); + this.rotationTexture = createTexture(device, 'splatRotation', this.format.format, size); + this.centerTexture = createTexture(device, 'splatCenter', this.format.format, size); this.material.setParameter('splatColor', this.colorTexture); this.material.setParameter('splatScale', this.scaleTexture); @@ -187,6 +184,15 @@ class Splat { this.material.setParameter('tex_params', new Float32Array([size.x, size.y, 1 / size.x, 1 / size.y])); } + destroy() { + this.colorTexture.destroy(); + this.scaleTexture.destroy(); + this.rotationTexture.destroy(); + this.centerTexture.destroy(); + this.material.destroy(); + this.mesh.destroy(); + } + updateColorData(f_dc_0: TypedArray, f_dc_1: TypedArray, f_dc_2: TypedArray, opacity: TypedArray) { const SH_C0 = 0.28209479177387814; const texture = this.colorTexture; @@ -218,7 +224,6 @@ class Splat { } updateScaleData(scale_0: TypedArray, scale_1: TypedArray, scale_2: TypedArray) { - // texture format based vars const { numComponents, isHalf } = this.format; const texture = this.scaleTexture; const data = texture.lock(); @@ -244,7 +249,6 @@ class Splat { } updateRotationData(rot_0: TypedArray, rot_1: TypedArray, rot_2: TypedArray, rot_3: TypedArray) { - // texture format based vars const { numComponents, isHalf } = this.format; const quat = new Quat(); @@ -274,7 +278,6 @@ class Splat { } updateCenterData(x: TypedArray, y: TypedArray, z: TypedArray) { - // texture format based vars const { numComponents, isHalf } = this.format; const texture = this.centerTexture; From 86f8d33de253e953ca9d0715c629528c01430fbc Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 25 Oct 2023 16:05:31 +0100 Subject: [PATCH 2/4] addProp --- src/splat/splat-data.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/splat/splat-data.ts b/src/splat/splat-data.ts index 62758d5..001005f 100644 --- a/src/splat/splat-data.ts +++ b/src/splat/splat-data.ts @@ -115,6 +115,16 @@ class SplatData { return this.vertexElement.properties.find((property: any) => property.name === name && property.storage)?.storage; } + // add a new property + addProp(name: string, storage: Float32Array) { + this.vertexElement.properties.push({ + type: 'float', + name, + storage, + byteSize: 4 + }); + } + // calculate scene aabb taking into account splat size calcAabb(result: BoundingBox) { const x = this.getProp('x'); From df034ecd7a86e7fc9813491ddd7dd311494e94e2 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 25 Oct 2023 16:26:26 +0100 Subject: [PATCH 3/4] debug shader --- src/splat/splat-material.ts | 83 ++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 43 deletions(-) diff --git a/src/splat/splat-material.ts b/src/splat/splat-material.ts index c7d0255..fed0424 100644 --- a/src/splat/splat-material.ts +++ b/src/splat/splat-material.ts @@ -159,57 +159,54 @@ const splatVS = ` return; } - vec3 splat_cova; - vec3 splat_covb; vec3 scale = getScale(); vec3 rotation = getRotation(); - computeCov3d(mat3(matrix_model) * quatToMat3(rotation), scale, splat_cova, splat_covb); - - mat3 Vrk = mat3( - splat_cova.x, splat_cova.y, splat_cova.z, - splat_cova.y, splat_covb.x, splat_covb.y, - splat_cova.z, splat_covb.y, splat_covb.z - ); - - float focal = viewport.x * matrix_projection[0][0]; - - mat3 J = mat3( - focal / splat_cam.z, 0., -(focal * splat_cam.x) / (splat_cam.z * splat_cam.z), - 0., focal / splat_cam.z, -(focal * splat_cam.y) / (splat_cam.z * splat_cam.z), - 0., 0., 0. - ); - - mat3 W = transpose(mat3(matrix_view)); - mat3 T = W * J; - mat3 cov = transpose(T) * Vrk * T; - - float diagonal1 = cov[0][0] + 0.3; - float offDiagonal = cov[0][1]; - float diagonal2 = cov[1][1] + 0.3; - - float mid = 0.5 * (diagonal1 + diagonal2); - float radius = length(vec2((diagonal1 - diagonal2) / 2.0, offDiagonal)); - float lambda1 = mid + radius; - float lambda2 = max(mid - radius, 0.1); - vec2 diagonalVector = normalize(vec2(offDiagonal, lambda1 - diagonal1)); - vec2 v1 = min(sqrt(2.0 * lambda1), 1024.0) * diagonalVector; - vec2 v2 = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x); - - gl_Position = splat_proj + - vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 2.0, - 0.0, 0.0) * splat_proj.w; - - texCoord = vertex_position.xy * 2.0; color = getColor(); #ifdef DEBUG_RENDER - vec3 local = quatToMat3(rotation) * (vertex_position * scale * 2.0) + center; gl_Position = matrix_viewProjection * matrix_model * vec4(local, 1.0); - - color = getColor(); - + #else + vec3 splat_cova; + vec3 splat_covb; + computeCov3d(mat3(matrix_model) * quatToMat3(rotation), scale, splat_cova, splat_covb); + + mat3 Vrk = mat3( + splat_cova.x, splat_cova.y, splat_cova.z, + splat_cova.y, splat_covb.x, splat_covb.y, + splat_cova.z, splat_covb.y, splat_covb.z + ); + + float focal = viewport.x * matrix_projection[0][0]; + + mat3 J = mat3( + focal / splat_cam.z, 0., -(focal * splat_cam.x) / (splat_cam.z * splat_cam.z), + 0., focal / splat_cam.z, -(focal * splat_cam.y) / (splat_cam.z * splat_cam.z), + 0., 0., 0. + ); + + mat3 W = transpose(mat3(matrix_view)); + mat3 T = W * J; + mat3 cov = transpose(T) * Vrk * T; + + float diagonal1 = cov[0][0] + 0.3; + float offDiagonal = cov[0][1]; + float diagonal2 = cov[1][1] + 0.3; + + float mid = 0.5 * (diagonal1 + diagonal2); + float radius = length(vec2((diagonal1 - diagonal2) / 2.0, offDiagonal)); + float lambda1 = mid + radius; + float lambda2 = max(mid - radius, 0.1); + vec2 diagonalVector = normalize(vec2(offDiagonal, lambda1 - diagonal1)); + vec2 v1 = min(sqrt(2.0 * lambda1), 1024.0) * diagonalVector; + vec2 v2 = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x); + + gl_Position = splat_proj + + vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 2.0, + 0.0, 0.0) * splat_proj.w; + + texCoord = vertex_position.xy * 2.0; #endif } `; From ced95a8c6481be763a0ae9e24976b5d3857725fa Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Thu, 26 Oct 2023 14:12:31 +0100 Subject: [PATCH 4/4] whitespace --- src/splat/splat-material.ts | 14 +++++++------- src/viewer.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/splat/splat-material.ts b/src/splat/splat-material.ts index fed0424..f9905ca 100644 --- a/src/splat/splat-material.ts +++ b/src/splat/splat-material.ts @@ -194,13 +194,13 @@ const splatVS = ` float offDiagonal = cov[0][1]; float diagonal2 = cov[1][1] + 0.3; - float mid = 0.5 * (diagonal1 + diagonal2); - float radius = length(vec2((diagonal1 - diagonal2) / 2.0, offDiagonal)); - float lambda1 = mid + radius; - float lambda2 = max(mid - radius, 0.1); - vec2 diagonalVector = normalize(vec2(offDiagonal, lambda1 - diagonal1)); - vec2 v1 = min(sqrt(2.0 * lambda1), 1024.0) * diagonalVector; - vec2 v2 = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x); + float mid = 0.5 * (diagonal1 + diagonal2); + float radius = length(vec2((diagonal1 - diagonal2) / 2.0, offDiagonal)); + float lambda1 = mid + radius; + float lambda2 = max(mid - radius, 0.1); + vec2 diagonalVector = normalize(vec2(offDiagonal, lambda1 - diagonal1)); + vec2 v1 = min(sqrt(2.0 * lambda1), 1024.0) * diagonalVector; + vec2 v2 = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x); gl_Position = splat_proj + vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 2.0, diff --git a/src/viewer.ts b/src/viewer.ts index 17e02cf..00b2352 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -1801,7 +1801,7 @@ class Viewer { rt.resolve(); } - // perform mulitiframe update. returned flag indicates whether more frames + // perform multiframe update. returned flag indicates whether more frames // are needed. this.multiframeBusy = this.multiframe.update(); }