Skip to content

Commit

Permalink
fix(volume_rendering): Volume rendering fixes for max projection - pi…
Browse files Browse the repository at this point in the history 
…cking, blending, and performance (#584)

* fix: ensure picking value from max projection is < 1

* fix: better blending of multiple VR layers with max enabled

* fix: clarify comment

* fix: userIntensity always overwrites default

* fix: no longer double draw opaque layers

* chore: remove log to check fix

* refactor: remove non-functioning continue statement

* feat: emit a single ID for picking in VR layer

* refactor: rename uniform to uPickId

* feat: register VR layer to get pick ID

* refactor: remove unused demo code around pick ID

* refactor: clarify role of copy shaders in perspective panel

* fix: don't copy over the max projection picking buffer for each max layer
Do one copy if there are any max layers instead

* refactor: clarify buffer names

* fix: VR tool from Python
  • Loading branch information
@seankmartin
seankmartin authored May 18, 2024
1 parent 2d7862a commit 3b2a8a3
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 60 deletions.
2 changes: 1 addition & 1 deletion python/neuroglancer/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
PlaceEllipsoidTool, # noqa: F401
BlendTool, # noqa: F401
OpacityTool, # noqa: F401
VolumeRenderingModeTool, # noqa: F401
VolumeRenderingTool, # noqa: F401
VolumeRenderingGainTool, # noqa: F401
VolumeRenderingDepthSamplesTool, # noqa: F401
CrossSectionRenderScaleTool, # noqa: F401
Expand Down
4 changes: 2 additions & 2 deletions python/neuroglancer/viewer_state.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,9 +165,9 @@ class OpacityTool(Tool):


@export_tool
class VolumeRenderingModeTool(Tool):
class VolumeRenderingTool(Tool):
__slots__ = ()
TOOL_TYPE = "volumeRenderingMode"
TOOL_TYPE = "volumeRendering"


@export_tool
Expand Down
127 changes: 74 additions & 53 deletions src/perspective_view/panel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,21 +153,26 @@ export function perspectivePanelEmitOIT(builder: ShaderBuilder) {
}

export function maxProjectionEmit(builder: ShaderBuilder) {
builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1);
builder.addOutputBuffer("highp vec4", "v4f_fragData2", 2);
builder.addOutputBuffer("vec4", "out_color", 0);
builder.addOutputBuffer("highp vec4", "out_z", 1);
builder.addOutputBuffer("highp vec4", "out_intensity", 2);
builder.addOutputBuffer("highp vec4", "out_pickId", 3);
builder.addFragmentCode(`
void emit(vec4 color, float depth, float pick) {
v4f_fragData0 = color;
v4f_fragData1 = vec4(1.0 - depth, 1.0 - depth, 1.0 - depth, 1.0);
v4f_fragData2 = vec4(pick, pick, pick, 1.0);
void emit(vec4 color, float depth, float intensity, highp uint pickId) {
float pickIdFloat = float(pickId);
float bufferDepth = 1.0 - depth;
out_color = color;
out_z = vec4(bufferDepth, bufferDepth, bufferDepth, 1.0);
out_intensity = vec4(intensity, intensity, intensity, 1.0);
out_pickId = vec4(pickIdFloat, pickIdFloat, pickIdFloat, 1.0);
}`);
}

const tempVec3 = vec3.create();
const tempVec4 = vec4.create();
const tempMat4 = mat4.create();

// Copy the OIT values to the main color buffer
function defineTransparencyCopyShader(builder: ShaderBuilder) {
builder.addOutputBuffer("vec4", "v4f_fragColor", null);
builder.setFragmentMain(`
Expand All @@ -180,31 +185,45 @@ v4f_fragColor = vec4(accum.rgb / accum.a, revealage);
`);
}

// Copy the max projection color to the OIT buffer
function defineMaxProjectionColorCopyShader(builder: ShaderBuilder) {
builder.addOutputBuffer("vec4", "v4f_fragColor", null);
builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
builder.addOutputBuffer("vec4", "v4f_fragData1", 1);
builder.addFragmentCode(glsl_perspectivePanelEmitOIT);
builder.setFragmentMain(`
v4f_fragColor = getValue0();
vec4 color = getValue0();
float bufferDepth = getValue1().r;
float weight = computeOITWeight(color.a, 1.0 - bufferDepth);
vec4 accum = color * weight;
float revealage = color.a;
emitAccumAndRevealage(accum, revealage, 0u);
`);
}

// Copy the max projection depth and pick values to the main buffer
function defineMaxProjectionPickCopyShader(builder: ShaderBuilder) {
builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1);
builder.addOutputBuffer("highp vec4", "v4f_fragData2", 2);
builder.addOutputBuffer("vec4", "out_color", 0);
builder.addOutputBuffer("highp vec4", "out_z", 1);
builder.addOutputBuffer("highp vec4", "out_pickId", 2);
builder.setFragmentMain(`
v4f_fragData0 = vec4(0.0);
v4f_fragData1 = getValue0();
v4f_fragData2 = getValue1();
out_color = vec4(0.0);
out_z = getValue0();
out_pickId = getValue1();
`);
}

// Copy the max projection depth and picking to the max projection pick buffer.
// Note that the depth is set as the intensity value from the render layer.
// This is to combine max projection picking data via depth testing
// on the maximum intensity value of the data.
function defineMaxProjectionToPickCopyShader(builder: ShaderBuilder) {
builder.addOutputBuffer("highp vec4", "v4f_fragData0", 0);
builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1);
builder.addOutputBuffer("highp vec4", "out_z", 0);
builder.addOutputBuffer("highp vec4", "out_pickId", 1);
builder.setFragmentMain(`
v4f_fragData0 = getValue0();
v4f_fragData1 = getValue1();
gl_FragDepth = v4f_fragData1.r;
out_z = getValue0();
out_pickId = getValue2();
gl_FragDepth = getValue1().r;
`);
}

Expand Down Expand Up @@ -306,13 +325,13 @@ export class PerspectivePanel extends RenderedDataPanel {
OffscreenCopyHelper.get(this.gl, defineTransparencyCopyShader, 2),
);
protected maxProjectionColorCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 1),
OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 2),
);
protected maxProjectionPickCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(this.gl, defineMaxProjectionPickCopyShader, 2),
);
protected maxProjectionToPickCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(this.gl, defineMaxProjectionToPickCopyShader, 2),
OffscreenCopyHelper.get(this.gl, defineMaxProjectionToPickCopyShader, 3),
);

private sharedObject: PerspectiveViewState;
Expand Down Expand Up @@ -730,6 +749,12 @@ export class PerspectivePanel extends RenderedDataPanel {
WebGL2RenderingContext.RED,
WebGL2RenderingContext.FLOAT,
),
new TextureBuffer(
this.gl,
WebGL2RenderingContext.R32F,
WebGL2RenderingContext.RED,
WebGL2RenderingContext.FLOAT,
),
],
depthBuffer: new DepthStencilRenderbuffer(this.gl),
}),
Expand Down Expand Up @@ -1004,6 +1029,7 @@ export class PerspectivePanel extends RenderedDataPanel {
renderContext.depthBufferTexture =
this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture;
}
// Draw max projection layers
if (
renderLayer.isVolumeRendering &&
isProjectionLayer(renderLayer as VolumeRenderingRenderLayer)
Expand All @@ -1021,21 +1047,26 @@ export class PerspectivePanel extends RenderedDataPanel {
bindMaxProjectionPickingBuffer();
this.maxProjectionToPickCopyHelper.draw(
this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
this.maxProjectionConfiguration.colorBuffers[2 /*pick*/].texture,
this.maxProjectionConfiguration.colorBuffers[2 /*intensity*/]
.texture,
this.maxProjectionConfiguration.colorBuffers[3 /*pick*/].texture,
);

// Copy max projection color result to color only buffer
// Copy max projection color result to the transparent buffer with OIT
// Depth testing off to combine max layers into one color via blend
this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR);
renderContext.bindFramebuffer();
gl.depthMask(false);
gl.disable(WebGL2RenderingContext.DEPTH_TEST);
gl.enable(WebGL2RenderingContext.BLEND);
gl.blendFunc(
gl.blendFuncSeparate(
WebGL2RenderingContext.ONE,
WebGL2RenderingContext.ONE,
WebGL2RenderingContext.ZERO,
WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
);
this.maxProjectionColorCopyHelper.draw(
this.maxProjectionConfiguration.colorBuffers[0 /*color*/].texture,
this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
);

// Reset the max projection buffer
Expand All @@ -1052,19 +1083,15 @@ export class PerspectivePanel extends RenderedDataPanel {
gl.clearDepth(1.0);
gl.clearColor(0.0, 0.0, 0.0, 1.0);
gl.depthMask(false);
gl.blendFuncSeparate(
WebGL2RenderingContext.ONE,
WebGL2RenderingContext.ONE,
WebGL2RenderingContext.ZERO,
WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
);
gl.enable(WebGL2RenderingContext.DEPTH_TEST);
gl.depthFunc(WebGL2RenderingContext.LESS);
renderContext.emitter = perspectivePanelEmitOIT;
renderContext.bindFramebuffer();
continue;
}
renderLayer.draw(renderContext, attachment);
// Draw regular transparent layers
else if (renderLayer.isTransparent) {
renderLayer.draw(renderContext, attachment);
}
}
// Copy transparent rendering result back to primary buffer.
gl.disable(WebGL2RenderingContext.DEPTH_TEST);
Expand Down Expand Up @@ -1109,27 +1136,21 @@ export class PerspectivePanel extends RenderedDataPanel {
/*dppass=*/ WebGL2RenderingContext.REPLACE,
);
gl.stencilMask(2);
if (hasMaxProjection) {
this.maxProjectionPickCopyHelper.draw(
this.maxProjectionPickConfiguration.colorBuffers[0].texture /*depth*/,
this.maxProjectionPickConfiguration.colorBuffers[1].texture /*pick*/,
);
}
for (const [renderLayer, attachment] of visibleLayers) {
if (!renderLayer.isTransparent || !renderLayer.transparentPickEnabled) {
if (
!renderLayer.isTransparent ||
!renderLayer.transparentPickEnabled ||
renderLayer.isVolumeRendering
) {
// Skip non-transparent layers and transparent layers with transparentPickEnabled=false.
// Volume rendering layers are handled separately and are combined in a pick buffer
continue;
}
// For max projection layers, can copy over the pick buffer directly.
if (renderLayer.isVolumeRendering) {
if (isProjectionLayer(renderLayer as VolumeRenderingRenderLayer)) {
this.maxProjectionPickCopyHelper.draw(
this.maxProjectionPickConfiguration.colorBuffers[0]
.texture /*depth*/,
this.maxProjectionPickConfiguration.colorBuffers[1]
.texture /*pick*/,
);
}
// Draw picking for non min/max volume rendering layers
else {
// Currently volume rendering layers have no picking support
// Outside of min/max mode
continue;
}
// other transparent layers are drawn as usual
} else {
renderLayer.draw(renderContext, attachment);
}
Expand Down
16 changes: 12 additions & 4 deletions src/volume_rendering/volume_render_layer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,16 +254,18 @@ void emitIntensity(float value) {
float savedDepth = 0.0;
float savedIntensity = 0.0;
vec4 newColor = vec4(0.0);
float userEmittedIntensity = -100.0;
`);
glsl_emitIntensity = `
float convertIntensity(float value) {
return clamp(${glsl_intensityConversion}, 0.0, 1.0);
}
void emitIntensity(float value) {
defaultMaxProjectionIntensity = value;
userEmittedIntensity = value;
}
float getIntensity() {
return convertIntensity(defaultMaxProjectionIntensity);
float intensity = userEmittedIntensity > -100.0 ? userEmittedIntensity : defaultMaxProjectionIntensity;
return convertIntensity(intensity);
}
`;
glsl_rgbaEmit = `
Expand All @@ -281,8 +283,9 @@ void emitRGBA(vec4 rgba) {
savedIntensity = intensityChanged ? newIntensity : savedIntensity;
savedDepth = intensityChanged ? depthAtRayPosition : savedDepth;
outputColor = intensityChanged ? newColor : outputColor;
emit(outputColor, savedDepth, savedIntensity);
emit(outputColor, savedDepth, savedIntensity, uPickId);
defaultMaxProjectionIntensity = 0.0;
userEmittedIntensity = -100.0;
`;
}
emitter(builder);
Expand All @@ -308,6 +311,7 @@ void emitRGBA(vec4 rgba) {

builder.addUniform("highp float", "uBrightnessFactor");
builder.addUniform("highp float", "uGain");
builder.addUniform("highp uint", "uPickId");
builder.addVarying("highp vec4", "vNormalizedPosition");
builder.addTextureSampler(
"sampler2D",
Expand Down Expand Up @@ -365,7 +369,7 @@ vec2 computeUVFromClipSpace(vec4 clipSpacePosition) {
`;
if (isProjectionMode(shaderParametersState.mode)) {
glsl_emitWireframe = `
emit(outputColor, 1.0, uChunkNumber);
emit(outputColor, 1.0, uChunkNumber, uPickId);
`;
}
builder.setFragmentMainFunction(`
Expand Down Expand Up @@ -621,6 +625,9 @@ void main() {
gl.enable(WebGL2RenderingContext.CULL_FACE);
gl.cullFace(WebGL2RenderingContext.FRONT);

const pickId = isProjectionMode(this.mode.value)
? renderContext.pickIDs.register(this)
: 0;
forEachVisibleVolumeRenderingChunk(
renderContext.projectionParameters,
this.localPosition.value,
Expand Down Expand Up @@ -798,6 +805,7 @@ void main() {
}
newSource = false;
gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition);
gl.uniform1ui(shader.uniform("uPickId"), pickId);
drawBoxes(gl, 1, 1);
++presentCount;
} else {
Expand Down

0 comments on commit 3b2a8a3

Please sign in to comment.