Fixed hang on exit of isolate select with geom subsets, with test.

lib/flowViewport/sceneIndex/fvpIsolateSelectSceneIndex.cpp

@@ -26,6 +26,7 @@
 #include <pxr/imaging/hd/containerDataSourceEditor.h>
 #include <pxr/imaging/hd/instanceSchema.h>
 #include <pxr/imaging/hd/instancerTopologySchema.h>
+#include <pxr/imaging/hd/geomSubsetSchema.h>
 #include <pxr/imaging/hd/tokens.h>
 
 PXR_NAMESPACE_USING_DIRECTIVE
@@ -96,6 +97,17 @@ Dependencies instancedPrim(
         Dependencies());
 }
 
+bool isGeomSubset(const HdSceneIndexPrim& prim) {
+    // HYDRA-1339: PiPrototypePropagatingSceneIndex removes GeomSubset type
+    // from Hydra prims
+#if PXR_VERSION >= 2403
+    return (prim.primType == HdPrimTypeTokens->geomSubset) ||
+        HdGeomSubsetSchema::GetFromParent(prim.dataSource).IsDefined();
+#else
+    return false;
+#endif
+}
+
 }
 
 namespace FVP_NS_DEF {
@@ -164,7 +176,9 @@ HdSceneIndexPrim IsolateSelectSceneIndex::GetPrim(const SdfPath& primPath) const
     TF_DEBUG(FVP_ISOLATE_SELECT_SCENE_INDEX)
         .Msg("    prim path %s is %s isolate select set", primPath.GetText(), (included ? "INCLUDED in" : "EXCLUDED from"));
 
-    if (!included) {
+    // HYDRA-1242: setting visibility on GeomSubset prim causes hang in Hydra
+    // Storm.
+    if (!included && !isGeomSubset(inputPrim)) {
         inputPrim.dataSource = HdContainerDataSourceEditor(inputPrim.dataSource)
             .Set(HdVisibilitySchema::GetDefaultLocator(), visOff)
             .Finish();
@@ -426,6 +440,14 @@ void IsolateSelectSceneIndex::_DirtyVisibilityRecursive(
     TF_DEBUG(FVP_ISOLATE_SELECT_SCENE_INDEX)
         .Msg("            %s: marking %s visibility locator dirty.\n", _viewportId.c_str(), primPath.GetText());
 
+    auto prim = GetInputSceneIndex()->GetPrim(primPath);
+
+    // GeomSubset visibility must not be set (see GetPrim()), so no need to
+    // dirty it.
+    if (isGeomSubset(prim)) {
+      return;
+    }
+
     dirtiedEntries->emplace_back(
         primPath, HdVisibilitySchema::GetDefaultLocator());
 

test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt

@@ -55,6 +55,7 @@ set(INTERACTIVE_TEST_SCRIPT_FILES
     testMayaDefaultMaterial.py
     testMayaLightingModes.py
     testStageInstanceablePrimsSelHighlight.py|skipOnPlatform:osx # HYDRA-1315 : Wire not showing on OSX
+    testIsolateSelectWithGeomSubset.py
     cpp/testColorPreferences.py
     cpp/testCppFramework.py
     cpp/testDataProducerExample.py

test/lib/mayaUsd/render/mayaToHydra/testIsolateSelectWithGeomSubset.py

@@ -0,0 +1,59 @@
+# Copyright 2024 Autodesk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import maya.cmds as cmds
+import maya.mel as mel
+import fixturesUtils
+import mayaUtils
+import mtohUtils
+
+def enableIsolateSelect(modelPanel):
+    # See comments in cpp/testIsolateSelect.py
+    cmds.setFocus(modelPanel)
+    mel.eval("enableIsolateSelect %s 1" % modelPanel)
+
+def disableIsolateSelect(modelPanel):
+    cmds.setFocus(modelPanel)
+    mel.eval("enableIsolateSelect %s 0" % modelPanel)
+
+class TestIsolateSelectWithGeomSubset(mtohUtils.MayaHydraBaseTestCase):
+    # MayaHydraBaseTestCase.setUpClass requirement.
+    _file = __file__
+
+    def test_IsolateSelectWithGeomSubset(self):
+        # HYDRA-1242: hang on exit of isolate select if there is a GeomSubset
+        # in the geometry that is hidden.
+        mayaUtils.openTestScene( 
+                "testIsolateSelectWithGeomSubset",
+                "mayaSpherePlusUSD_Instancer.ma")
+
+        cmds.refresh()
+
+        # Isolate select the Maya sphere.
+        modelPanel = 'modelPanel4'
+        enableIsolateSelect(modelPanel)
+
+        cmds.select('|pSphere1')
+        cmds.editor(modelPanel, edit=True, updateMainConnection=True)
+        cmds.isolateSelect(modelPanel, loadSelected=True)
+
+        cmds.refresh()
+
+        # Disable the isolate selection.
+        disableIsolateSelect(modelPanel)
+
+        cmds.refresh()
+
+if __name__ == '__main__':
+    fixturesUtils.runTests(globals())

test/testSamples/testIsolateSelectWithGeomSubset/rubiksCube/rubiksCube.usda

@@ -0,0 +1,8 @@
+#usda 1.0
+
+def Xform "root" (
+    prepend references = @rubiksCube_A/rubiksCube_A.usda@
+)
+{
+}
+

test/testSamples/testIsolateSelectWithGeomSubset/rubiksCube/rubiksCube_A/bind/rubiksCube_A_bind.usda

@@ -0,0 +1,56 @@
+#usda 1.0
+
+over "root"
+{
+    over "geo"
+    {
+        over "proxy"
+        {
+            over "rubiks01_top_plastic_01" (
+                prepend apiSchemas = ["MaterialBindingAPI"]
+            )
+            {
+                rel material:binding = </root/mtl/rubiksCube_proxy>
+            }
+
+            over "rubiks01_mid_plastic_01" (
+                prepend apiSchemas = ["MaterialBindingAPI"]
+            )
+            {
+                rel material:binding = </root/mtl/rubiksCube_proxy>
+            }
+
+            over "rubiks01_bot_plastic_01" (
+                prepend apiSchemas = ["MaterialBindingAPI"]
+            )
+            {
+                rel material:binding = </root/mtl/rubiksCube_proxy>
+            }
+        }
+
+        over "render"
+        {
+            over "rubiks01_top_plastic_01" (
+                prepend apiSchemas = ["MaterialBindingAPI"]
+            )
+            {
+                rel material:binding = </root/mtl/rubiksCube_proxy>
+            }
+
+            over "rubiks01_mid_plastic_01" (
+                prepend apiSchemas = ["MaterialBindingAPI"]
+            )
+            {
+                rel material:binding = </root/mtl/rubiksCube_proxy>
+            }
+
+            over "rubiks01_bot_plastic_01" (
+                prepend apiSchemas = ["MaterialBindingAPI"]
+            )
+            {
+                rel material:binding = </root/mtl/rubiksCube_proxy>
+            }
+        }
+    }
+}
+

test/testSamples/testIsolateSelectWithGeomSubset/rubiksCube/rubiksCube_A/mtl/rubiksCube_A_mtl.usda

@@ -0,0 +1,129 @@
+#usda 1.0
+
+def Scope "root"
+{
+    def Scope "mtl"
+    {
+        def Material "rubiksCube_proxy" (
+            prepend apiSchemas = ["NodeGraphNodeAPI"]
+            customData = {
+                dictionary Autodesk = {
+                    string ldx_inputPos = "-450 -61"
+                    string ldx_outputPos = "225 -61"
+                }
+            }
+        )
+        {
+            token outputs:surface.connect = </root/mtl/rubiksCube_proxy/UsdPreviewSurface1.outputs:surface>
+            uniform float2 ui:nodegraph:node:pos = (0.055555556, 0.055555556)
+
+            def Shader "UsdPreviewSurface1" (
+                prepend apiSchemas = ["NodeGraphNodeAPI"]
+            )
+            {
+                uniform token info:id = "UsdPreviewSurface"
+                color3f inputs:diffuseColor.connect = </root/mtl/rubiksCube_proxy/base_color.outputs:rgb>
+                normal3f inputs:normal.connect = </root/mtl/rubiksCube_proxy/normal.outputs:rgb>
+                float inputs:roughness.connect = </root/mtl/rubiksCube_proxy/roughness.outputs:r>
+                token outputs:surface
+                uniform float2 ui:nodegraph:node:pos = (-0.44444445, -0.33888888)
+            }
+
+            def Shader "base_color" (
+                prepend apiSchemas = ["NodeGraphNodeAPI"]
+            )
+            {
+                uniform token info:id = "UsdUVTexture"
+                asset inputs:file = @../../../_textures/rubiksCube/rubiks01_var01_basecolor_1k_acescg.exr@
+                float2 inputs:st.connect = </root/mtl/rubiksCube_proxy/UsdPrimvarReader1.outputs:result>
+                float3 outputs:rgb
+                uniform float2 ui:nodegraph:node:pos = (-2.6583333, 1.5277778)
+            }
+
+            def Shader "UsdPrimvarReader1" (
+                prepend apiSchemas = ["NodeGraphNodeAPI"]
+            )
+            {
+                uniform token info:id = "UsdPrimvarReader_float2"
+                string inputs:varname = "st"
+                float2 outputs:result
+                uniform float2 ui:nodegraph:node:pos = (-4.0194445, 1.5277778)
+            }
+
+            def Shader "UsdPrimvarReader2" (
+                prepend apiSchemas = ["NodeGraphNodeAPI"]
+                customData = {
+                    dictionary ClipboardMetadata = {
+                        string materialName = "rubiksCube_proxy"
+                        string shaderName = "UsdPrimvarReader1"
+                        string stagePath = "|world|rubiksCube_A|rubiksCube_AShape"
+                    }
+                }
+            )
+            {
+                uniform token info:id = "UsdPrimvarReader_float2"
+                string inputs:varname = "st"
+                float2 outputs:result
+                uniform float2 ui:nodegraph:node:pos = (-4.0916667, 4.338889)
+            }
+
+            def Shader "roughness" (
+                prepend apiSchemas = ["NodeGraphNodeAPI"]
+                customData = {
+                    dictionary ClipboardMetadata = {
+                        string materialName = "rubiksCube_proxy"
+                        string shaderName = "base_color"
+                        string stagePath = "|world|rubiksCube_A|rubiksCube_AShape"
+                    }
+                }
+            )
+            {
+                uniform token info:id = "UsdUVTexture"
+                asset inputs:file = @../../../_textures/rubiksCube/rubiks01_var01_roughness_1k_raw.exr@
+                float2 inputs:st.connect = </root/mtl/rubiksCube_proxy/UsdPrimvarReader2.outputs:result>
+                float outputs:r
+                float3 outputs:rgb
+                uniform float2 ui:nodegraph:node:pos = (-2.7305555, 4.338889)
+            }
+
+            def Shader "UsdPrimvarReader3" (
+                prepend apiSchemas = ["NodeGraphNodeAPI"]
+                customData = {
+                    dictionary ClipboardMetadata = {
+                        string materialName = "rubiksCube_proxy"
+                        string shaderName = "UsdPrimvarReader2"
+                        string stagePath = "|world|rubiksCube_A|rubiksCube_AShape"
+                    }
+                }
+            )
+            {
+                uniform token info:id = "UsdPrimvarReader_float2"
+                string inputs:varname = "st"
+                float2 outputs:result
+                uniform float2 ui:nodegraph:node:pos = (-4.1138887, 7.5555553)
+            }
+
+            def Shader "normal" (
+                prepend apiSchemas = ["NodeGraphNodeAPI"]
+                customData = {
+                    dictionary ClipboardMetadata = {
+                        string materialName = "rubiksCube_proxy"
+                        string shaderName = "roughness"
+                        string stagePath = "|world|rubiksCube_A|rubiksCube_AShape"
+                    }
+                }
+            )
+            {
+                uniform token info:id = "UsdUVTexture"
+                float4 inputs:bias = (-1, -1, -1, 0)
+                asset inputs:file = @../../../_textures/rubiksCube/rubiks01_var01_normal_1k_raw.exr@
+                float4 inputs:scale = (2, 2, 2, 1)
+                float2 inputs:st.connect = </root/mtl/rubiksCube_proxy/UsdPrimvarReader3.outputs:result>
+                float outputs:r
+                float3 outputs:rgb
+                uniform float2 ui:nodegraph:node:pos = (-2.7527778, 7.5555553)
+            }
+        }
+    }
+}
+

test/testSamples/testIsolateSelectWithGeomSubset/rubiksCube/rubiksCube_A/rubiksCube_A.usda

@@ -0,0 +1,31 @@
+#usda 1.0
+(
+    defaultPrim = "root"
+    metersPerUnit = 0.01
+    subLayers = [
+        @mtl/rubiksCube_A_mtl.usda@,
+        @bind/rubiksCube_A_bind.usda@
+    ]
+    upAxis = "Y"
+)
+
+over "root"
+{
+    def Scope "geo"
+    {
+        def Scope "proxy" (
+            prepend references = @geo/rubiksCube_A_proxy.usd@</rubiks_A_proxy>
+        )
+        {
+            uniform token purpose = "proxy"
+        }
+
+        def Scope "render" (
+            prepend payload = @geo/rubiksCube_A_render.usd@</rubiks_A_render>
+        )
+        {
+            uniform token purpose = "render"
+        }
+    }
+}
+

test/testSamples/testIsolateSelectWithGeomSubset/rubiksCube_scatter.usda

@@ -0,0 +1,28 @@
+#usda 1.0
+(
+    defaultPrim = "rubikCubes"
+    metersPerUnit = 0.01
+    upAxis = "Y"
+)
+
+def "rubikCubes"
+{
+    def PointInstancer "instancer"
+    {
+        quath[] orientations = [(0.830078, 0.546875, -0.304688, -0.0601807), (0.947266, -0.00674438, -0.320801, -0.0022831), (0.995117, 0.0930176, -0.0352783, 0.00329781), (0.79541, 0.554688, 0.200684, -0.140015), (0.776855, 0.582031, -0.192261, 0.144043), (0.980957, 0.104614, 0.162476, -0.017334), (0.916992, 0.296875, 0.253906, -0.0822144), (0.961914, 0.273193, 7.61747e-05, -2.16365e-05), (0.916504, 0.141846, -0.369141, 0.0571289), (0.941895, 0.117126, -0.313232, 0.0389404), (0.943359, 0.314697, -0.0973511, 0.0324707), (0.937012, 0.300293, -0.169434, 0.0543213), (0.980469, -0.0513611, -0.188354, -0.00986481), (0.907715, 0.400391, -0.115601, 0.0509949), (0.785156, 0.617188, -0.0441589, 0.0346985), (0.998047, 0.0215759, -0.054718, 0.00118256), (0.966309, -0.0420532, -0.253662, -0.0110397), (0.834961, 0.437256, 0.295654, -0.154785), (0.964355, 0.248413, 0.086792, -0.0223541), (0.966309, 0.067627, 0.247803, -0.017334), (0.98291, -0.019577, -0.181885, -0.00362015), (0.856445, 0.451416, 0.221558, -0.11676), (0.979004, 0.171997, 0.107605, -0.0189056)]
+        point3f[] positions = [(2.4837434, 0, -18.20366), (-3.7627668, 0, 20.036228), (-17.933372, 0, 6.076911), (41.264294, 0, 2.9849777), (-9.636904, 0, -2.851346), (-9.777513, 0, 11.981253), (23.882774, 0, -18.464333), (1.1104083, 0, 11.133188), (-20.105476, 0, -3.7781515), (19.17274, 0, 6.489833), (-7.739782, 0, -14.802163), (-27.559029, 0, 9.223055), (-26.925617, 0, -17.712435), (19.15774, 0, -8.733529), (0.46824232, 0, -5.369589), (12.41522, 0, 14.0595875), (-26.556774, 0, 20.013483), (12.544684, 0, -16.354366), (12.0251875, 0, -0.82279897), (22.597328, 0, 19.998356), (-16.260296, 0, 19.783886), (27.058168, 0, -0.14280228), (-16.569267, 0, -19.845718)]
+        int[] protoIndices = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+        prepend rel prototypes = </rubikCubes/instancer/prototypes/geom>
+        float3[] scales = [(1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1)]
+
+        def "prototypes"
+        {
+            def Xform "geom" (
+                prepend references = @./rubiksCube/rubiksCube.usda@</root>
+            )
+            {
+            }
+        }
+    }
+}
+