feat(video): add support for contour ROI and contour based segmentati…

…on (#988)

* feat: Add spline to video

* Add video spline tools

* Add livewire example

* fix: example was broken

* Fixes to make livewire work on RGB

* Faster convert to grayscale

* Fix the video tools example

* PR review fixes, fix fourth/fifth mouse buttons

* PR fixes

* yarn lock

* feat: Add livewire segmentation on video viewports

* feat: Add spline segmentation for video

* chore(test): Basic changes to have jest running tests (#984)

* Basic changes to have jest running tests

* Fix unit test imports for now

* Fix build issues with recursive imports

* yarn lock

* chore(version): version.json [skip ci]

* chore(version): Update package versions [skip ci]

* Only display on the exact image for the segmentation contour

* Add the planar freehand segmentation

* PR changes

* Fix PR comment

---------

Co-authored-by: ohif-bot <[email protected]>

.gitignore

@@ -47,4 +47,5 @@ temp/
 
 # Report Output
 junit/
+junit.xml
 coverage/

common/reviews/api/core.api.md

@@ -319,6 +319,9 @@ function convertStackToVolumeViewport({ viewport, options, }: {
     };
 }): Promise<IVolumeViewport>;
 
+// @public (undocumented)
+function convertToGrayscale(scalarData: any, width: number, height: number): any;
+
 // @public (undocumented)
 function convertVolumeToStackViewport({ viewport, options, }: {
     viewport: Types.IVolumeViewport;
@@ -3153,6 +3156,7 @@ declare namespace utilities {
         createInt16SharedArray,
         getViewportModality,
         windowLevel,
+        convertToGrayscale,
         getClosestImageId,
         getSpacingInNormalDirection,
         getTargetVolumeAndSpacingInNormalDir,
@@ -3288,7 +3292,7 @@ export class VideoViewport extends Viewport implements IVideoViewport {
     // (undocumented)
     pause(): Promise<void>;
     // (undocumented)
-    play(): void;
+    play(): Promise<void>;
     // (undocumented)
     readonly renderingEngineId: string;
     // (undocumented)

packages/core/src/RenderingEngine/VideoViewport.ts

@@ -156,6 +156,10 @@ class VideoViewport extends Viewport implements IVideoViewport {
       columnCosines[1],
       columnCosines[2]
     );
+
+    const { rows, columns } = imagePlaneModule;
+    this.videoWidth = columns;
+    this.videoHeight = rows;
     const scanAxisNormal = vec3.create();
     vec3.cross(scanAxisNormal, rowCosineVec, colCosineVec);
 
@@ -216,10 +220,14 @@ class VideoViewport extends Viewport implements IVideoViewport {
       this.numberOfFrames = numberOfFrames;
       // 1 based range setting
       this.setFrameRange([1, numberOfFrames]);
-      if (frameNumber !== undefined) {
+      this.play();
+      // This is ugly, but without it, the video often fails to render initially
+      // so having a play, followed by a pause fixes things.
+      // 50 ms is a tested value that seems to work to prevent exceptions
+      window.setTimeout(() => {
         this.pause();
-        this.setFrameNumber(frameNumber);
-      }
+        this.setFrameNumber(frameNumber || 1);
+      }, 50);
     });
   }
 
@@ -258,17 +266,27 @@ class VideoViewport extends Viewport implements IVideoViewport {
     }
   }
 
-  public play() {
-    if (!this.isPlaying) {
-      this.videoElement.play();
-      this.isPlaying = true;
-      this.renderWhilstPlaying();
+  public async play() {
+    try {
+      if (!this.isPlaying) {
+        // Play returns a promise that is true when playing completes.
+        await this.videoElement.play();
+        this.isPlaying = true;
+        this.renderWhilstPlaying();
+      }
+    } catch (e) {
+      // No-op, an exception sometimes gets thrown on the initial play, not
+      // quite sure why.  Catching it prevents displaying an error
     }
   }
 
   public async pause() {
-    await this.videoElement.pause();
-    this.isPlaying = false;
+    try {
+      await this.videoElement.pause();
+      this.isPlaying = false;
+    } catch (e) {
+      // No-op - sometimes this happens on startup
+    }
   }
 
   public async scroll(delta = 1) {
@@ -457,7 +475,7 @@ class VideoViewport extends Viewport implements IVideoViewport {
 
     const spacing = metadata.spacing;
 
-    return {
+    const imageData = {
       dimensions: metadata.dimensions,
       spacing,
       origin: metadata.origin,
@@ -486,6 +504,11 @@ class VideoViewport extends Viewport implements IVideoViewport {
         scaled: false,
       },
     };
+    Object.defineProperty(imageData, 'scalarData', {
+      get: () => this.getScalarData(),
+      enumerable: true,
+    });
+    return imageData;
   }
 
   /**

packages/core/src/utilities/convertToGrayscale.ts

@@ -0,0 +1,27 @@
+export default function convertToGrayscale(
+  scalarData,
+  width: number,
+  height: number
+) {
+  const isRGBA = scalarData.length === width * height * 4;
+  const isRGB = scalarData.length === width * height * 3;
+  if (isRGBA || isRGB) {
+    const newScalarData = new Float32Array(width * height);
+    let offset = 0;
+    let destOffset = 0;
+    const increment = isRGBA ? 4 : 3;
+    for (let x = 0; x < width; x++) {
+      for (let y = 0; y < height; y++) {
+        const r = scalarData[offset];
+        const g = scalarData[offset + 1];
+        const b = scalarData[offset + 2];
+        newScalarData[destOffset] = (r + g + b) / 3;
+        offset += increment;
+        destOffset++;
+      }
+    }
+    return newScalarData;
+  } else {
+    return scalarData;
+  }
+}

packages/core/src/utilities/index.ts

@@ -66,6 +66,7 @@ import { convertStackToVolumeViewport } from './convertStackToVolumeViewport';
 import { convertVolumeToStackViewport } from './convertVolumeToStackViewport';
 import VoxelManager from './VoxelManager';
 import roundNumber, { roundToPrecision } from './roundNumber';
+import convertToGrayscale from './convertToGrayscale';
 
 // name spaces
 import * as planar from './planar';
@@ -97,6 +98,7 @@ export {
   createInt16SharedArray,
   getViewportModality,
   windowLevel,
+  convertToGrayscale,
   getClosestImageId,
   getSpacingInNormalDirection,
   getTargetVolumeAndSpacingInNormalDir,

packages/docs/docs/concepts/cornerstone-core/viewports.md

@@ -42,6 +42,12 @@ Both `StackViewport` and `VolumeViewport`, `VolumeViewport3D` are created via th
 
 :::
 
+## VideoViewport
+
+- Suitable for rendering video data
+- Video can include MPEG 4 encoded vide streams. In theory, MPEG2 is also supported,
+  but practically the browser doesn't support that.
+
 ## Initial Display Area
 
 All viewports inherit from the Viewport class which has a `displayArea` field which can be provided.

packages/docs/docs/tutorials/intro.md

@@ -13,6 +13,7 @@ Tutorials are wholly learning-oriented, and specifically, they are oriented towa
 :::
 
 ## Running a Tutorial Locally
+
 We have included a `tutorial` example in the repo, which you can find at `packages/tools/examples/tutorial/index.ts`. This file contains all the necessary setup code (explained above) for running a tutorial locally. When you open the file, you will see a dedicated place for you to copy and paste and insert the code from the tutorial. So, this way, you don't have to worry about the setup code, and you can focus on the tutorial itself.
 
 How to run it?
@@ -33,20 +34,20 @@ Then open a new tab in your browser and navigate to `http://localhost:3000/`.
 
 For curious learners, here are some components that are used (behind the scene) for each tutorial.
 
-
 ### Image Loaders
+
 `Cornerstone3D` does not deal with loading images. As we will learn later, `Cornerstone3D` also is capable of rendering `Volumes` in any orientation too.
 Therefore, proper image and volume loaders should be registered with `Cornerstone3D` so that it can work as intended. Examples of such loaders are
 
 - imageLoader: `cornerstoneWADOImageLoader`
 - volumeLoader: `cornerstoneStreamingImageVolumeLoader`
 
 ### Metadata Providers
+
 In order for `Cornerstone3D` to properly show the properties of an image such
 as voi, suv values, etc., it needs metadata (in addition to the image data itself).
 Therefore, proper metadata providers should be registered with `Cornerstone3D` so that it can work as intended. Examples of such providers are
 
-
-
 ### Library Initialization
+
 Both `Cornerstone3D` and `Cornerstone3DTools` need to be initialized by calling `.init()` methods.

packages/tools/examples/livewireContour/index.ts

@@ -10,6 +10,7 @@ import {
   setTitleAndDescription,
   createInfoSection,
   setCtTransferFunctionForVolumeActor,
+  addManipulationBindings,
 } from '../../../../utils/demo/helpers';
 import * as cornerstoneTools from '@cornerstonejs/tools';
 
@@ -107,19 +108,13 @@ async function run() {
 
   // Add tools to Cornerstone3D
   cornerstoneTools.addTool(LivewireContourTool);
-  cornerstoneTools.addTool(PanTool);
-  cornerstoneTools.addTool(ZoomTool);
-  cornerstoneTools.addTool(StackScrollMouseWheelTool);
 
   // Define a tool group, which defines how mouse events map to tool commands for
   // Any viewport using the group
   const toolGroup = ToolGroupManager.createToolGroup(toolGroupId);
 
   // Add the tools to the tool group
   toolGroup.addTool(LivewireContourTool.toolName);
-  toolGroup.addTool(PanTool.toolName);
-  toolGroup.addTool(ZoomTool.toolName);
-  toolGroup.addTool(StackScrollMouseWheelTool.toolName);
 
   // Set the initial state of the tools
   toolGroup.setToolActive(LivewireContourTool.toolName, {
@@ -129,26 +124,7 @@ async function run() {
       },
     ],
   });
-
-  toolGroup.setToolActive(PanTool.toolName, {
-    bindings: [
-      {
-        mouseButton: MouseBindings.Auxiliary, // Middle Click
-      },
-    ],
-  });
-
-  toolGroup.setToolActive(ZoomTool.toolName, {
-    bindings: [
-      {
-        mouseButton: MouseBindings.Secondary, // Right Click
-      },
-    ],
-  });
-
-  // As the Stack Scroll mouse wheel is a tool using the `mouseWheelCallback`
-  // hook instead of mouse buttons, it does not need to assign any mouse button.
-  toolGroup.setToolActive(StackScrollMouseWheelTool.toolName);
+  addManipulationBindings(toolGroup);
 
   // Get Cornerstone imageIds and fetch metadata into RAM
   const imageIds = await createImageIdsAndCacheMetaData({
@@ -168,7 +144,7 @@ async function run() {
     type: ViewportType.STACK,
     element: stackViewportElement,
     defaultOptions: {
-      background: <Types.Point3>[0.2, 0, 0.2],
+      background: <Types.Point3>[0.2, 0.2, 0],
     },
   };
 

packages/tools/examples/livewireContourSegmentation/index.ts

@@ -14,6 +14,7 @@ import {
   setTitleAndDescription,
   createInfoSection,
   setCtTransferFunctionForVolumeActor,
+  addManipulationBindings,
 } from '../../../../utils/demo/helpers';
 import * as cornerstoneTools from '@cornerstonejs/tools';
 import type { Types as cstTypes } from '@cornerstonejs/tools';
@@ -37,9 +38,6 @@ const DEFAULT_SEGMENTATION_CONFIG = {
 const {
   SegmentationDisplayTool,
   LivewireContourSegmentationTool,
-  PanTool,
-  ZoomTool,
-  StackScrollMouseWheelTool,
   ToolGroupManager,
   Enums: csToolsEnums,
   segmentation,
@@ -64,16 +62,16 @@ setTitleAndDescription(
 );
 
 const content = document.getElementById('content');
-const viewoprtsContainer = document.createElement('div');
+const viewportsContainer = document.createElement('div');
 
-Object.assign(viewoprtsContainer.style, {
+Object.assign(viewportsContainer.style, {
   display: 'grid',
   height: '500px',
   gridTemplateColumns: '1fr 1fr 1fr',
   gap: '5px',
 });
 
-content.appendChild(viewoprtsContainer);
+content.appendChild(viewportsContainer);
 
 const createViewportElement = (id: string) => {
   const element = document.createElement('div');
@@ -82,7 +80,7 @@ const createViewportElement = (id: string) => {
   element.oncontextmenu = (e) => e.preventDefault();
 
   element.id = id;
-  viewoprtsContainer.appendChild(element);
+  viewportsContainer.appendChild(element);
 
   return element;
 };
@@ -144,7 +142,7 @@ addButtonToToolbar({
 
 addSliderToToolbar({
   id: 'outlineWidthActive',
-  title: 'Segment Thickness',
+  title: 'Outline Thickness',
   range: [0.1, 10],
   step: 0.1,
   defaultValue: 1,
@@ -189,7 +187,7 @@ addSliderToToolbar({
 
 // =============================================================================
 
-const toolGroupId = 'STACK_TOOL_GROUP_ID';
+const toolGroupId = 'DEFAULT_TOOL_GROUP_ID';
 
 function initializeGlobalConfig() {
   const globalSegmentationConfig = segmentation.config.getGlobalConfig();
@@ -304,9 +302,6 @@ async function run() {
   // Add tools to Cornerstone3D
   cornerstoneTools.addTool(SegmentationDisplayTool);
   cornerstoneTools.addTool(LivewireContourSegmentationTool);
-  cornerstoneTools.addTool(PanTool);
-  cornerstoneTools.addTool(ZoomTool);
-  cornerstoneTools.addTool(StackScrollMouseWheelTool);
 
   // Define a tool group, which defines how mouse events map to tool commands for
   // Any viewport using the group
@@ -315,9 +310,7 @@ async function run() {
   // Add the tools to the tool group
   toolGroup.addTool(SegmentationDisplayTool.toolName);
   toolGroup.addTool(LivewireContourSegmentationTool.toolName);
-  toolGroup.addTool(PanTool.toolName);
-  toolGroup.addTool(ZoomTool.toolName);
-  toolGroup.addTool(StackScrollMouseWheelTool.toolName);
+  addManipulationBindings(toolGroup);
 
   // Set the initial state of the tools
   toolGroup.setToolEnabled(SegmentationDisplayTool.toolName);
@@ -330,26 +323,6 @@ async function run() {
     ],
   });
 
-  toolGroup.setToolActive(PanTool.toolName, {
-    bindings: [
-      {
-        mouseButton: MouseBindings.Auxiliary, // Middle Click
-      },
-    ],
-  });
-
-  toolGroup.setToolActive(ZoomTool.toolName, {
-    bindings: [
-      {
-        mouseButton: MouseBindings.Secondary, // Right Click
-      },
-    ],
-  });
-
-  // As the Stack Scroll mouse wheel is a tool using the `mouseWheelCallback`
-  // hook instead of mouse buttons, it does not need to assign any mouse button.
-  toolGroup.setToolActive(StackScrollMouseWheelTool.toolName);
-
   // Get Cornerstone imageIds and fetch metadata into RAM
   const imageIds = await createImageIdsAndCacheMetaData({
     StudyInstanceUID: