[Canvas] Fix group layering (#114346)

* Fix group layering

* Fixes incorrect z-layers for elements at 90deg

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/canvas/public/components/workpad_page/integration_utils.js

@@ -75,8 +75,40 @@ const missingParentCheck = (groups) => {
 };
 
 export const shapesForNodes = (nodes) => {
+  // For a group, it's z-layer should be the same as the highest of it's children
+  // So we cache every nodes layer in this array so when we get to a group
+  // we can refer back to all of the elements and figure out the appropriate layer
+  // for the group
+  const nodeLayers = nodes.map(() => null);
+
+  const getNodeLayer = (nodeIndex) => {
+    if (nodeLayers[nodeIndex]) {
+      return nodeLayers[nodeIndex];
+    }
+
+    const node = nodes[nodeIndex];
+    const thisId = node.id;
+
+    const childrenIndexesOfThisNode = nodes
+      .map((n, i) => [n, i])
+      .filter(([node]) => node.position.parent === thisId)
+      .map(([, index]) => index);
+
+    if (childrenIndexesOfThisNode.length === 0) {
+      nodeLayers[nodeIndex] = nodeIndex;
+    } else {
+      const layer = Math.max(...childrenIndexesOfThisNode.map(getNodeLayer));
+      nodeLayers[nodeIndex] = layer;
+    }
+
+    return nodeLayers[nodeIndex];
+  };
+
   const rawShapes = nodes
-    .map(elementToShape)
+    .map((node, index) => {
+      const layer = getNodeLayer(index);
+      return elementToShape(node, layer);
+    })
     // filtering to eliminate residual element of a possible group that had been deleted in Redux
     .filter((d, i, a) => !isGroupId(d.id) || a.find((s) => s.parent === d.id))
     .filter(dedupe);

x-pack/plugins/canvas/public/lib/aeroelastic/geometry.js

@@ -73,7 +73,13 @@ const planeTuple = (transformMatrix, x, y) => {
   const x0 = rightPoint[0] - centerPoint[0];
   const y0 = rightPoint[1] - centerPoint[1];
   const x1 = upPoint[0] - centerPoint[0];
-  const y1 = upPoint[1] - centerPoint[1];
+  // If the shape is rotated so it's straight up or down (90 deg or -90 deg) then
+  // y1 is going to be zero here and cause divide by zero issues further down.
+  // The resulting planeVector accounts for this, but it's returning something incorrect that causes z
+  // to be calculated incorrectly.
+  // So instead we're going to set y1 to almost zero so we don't have to worry abou the divide by zeros
+  // and we still end up with an almost correct planeVector
+  const y1 = upPoint[1] - centerPoint[1] || 0.00000000001;
   const rightSlope = y1 ? rightPoint[2] - centerPoint[2] : 0; // handle degenerate case: y1 === 0 (infinite slope)
   const upSlope = y1 ? upPoint[2] - centerPoint[2] : 0; // handle degenerate case: y1 === 0 (infinite slope)
   const inverseProjection = invert(transformMatrix);