Histogram equalization feature

cvat-canvas/src/typescript/canvas.ts

@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -28,7 +28,7 @@ const CanvasVersion = pjson.version;
 
 interface Canvas {
     html(): HTMLDivElement;
-    setup(frameData: any, objectStates: any[], zLayer?: number): void;
+    setup(frameData: any, objectStates: any[], zLayer?: number, forceUpdate?: boolean): void;
     setupIssueRegions(issueRegions: Record<number, number[]>): void;
     activate(clientID: number | null, attributeID?: number): void;
     rotate(rotationAngle: number): void;
@@ -72,8 +72,8 @@ class CanvasImpl implements Canvas {
         return this.view.html();
     }
 
-    public setup(frameData: any, objectStates: any[], zLayer = 0): void {
-        this.model.setup(frameData, objectStates, zLayer);
+    public setup(frameData: any, objectStates: any[], zLayer = 0, forceUpdate = false): void {
+        this.model.setup(frameData, objectStates, zLayer, forceUpdate);
     }
 
     public setupIssueRegions(issueRegions: Record<number, number[]>): void {

cvat-canvas/src/typescript/canvasModel.ts

@@ -183,7 +183,7 @@ export interface CanvasModel {
     zoom(x: number, y: number, direction: number): void;
     move(topOffset: number, leftOffset: number): void;
 
-    setup(frameData: any, objectStates: any[], zLayer: number): void;
+    setup(frameData: any, objectStates: any[], zLayer: number, forceUpdate: boolean): void;
     setupIssueRegions(issueRegions: Record<number, number[]>): void;
     activate(clientID: number | null, attributeID: number | null): void;
     rotate(rotationAngle: number): void;
@@ -386,14 +386,14 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
         this.notify(UpdateReasons.ZOOM_CANVAS);
     }
 
-    public setup(frameData: any, objectStates: any[], zLayer: number): void {
+    public setup(frameData: any, objectStates: any[], zLayer: number, forceUpdate: boolean = false): void {
         if (this.data.imageID !== frameData.number) {
             if ([Mode.EDIT, Mode.DRAG, Mode.RESIZE].includes(this.data.mode)) {
                 throw Error(`Canvas is busy. Action: ${this.data.mode}`);
             }
         }
 
-        if (frameData.number === this.data.imageID) {
+        if (frameData.number === this.data.imageID && !forceUpdate) {
             this.data.zLayer = zLayer;
             this.data.objects = objectStates;
             this.notify(UpdateReasons.OBJECTS_UPDATED);

cvat-core/src/api.js

@@ -20,6 +20,7 @@ function build() {
     const { Project } = require('./project');
     const { Attribute, Label } = require('./labels');
     const MLModel = require('./ml-model');
+    const { FrameData } = require('./frames');
 
     const enums = require('./enums');
 
@@ -765,6 +766,7 @@ function build() {
             Comment,
             Issue,
             Review,
+            FrameData,
         },
     };
 

cvat-core/src/plugins.js

@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx

@@ -6,7 +6,7 @@ import React from 'react';
 import { connect } from 'react-redux';
 import { Row, Col } from 'antd/lib/grid';
 import Popover from 'antd/lib/popover';
-import Icon, { ScissorOutlined } from '@ant-design/icons';
+import Icon, { AreaChartOutlined, ScissorOutlined } from '@ant-design/icons';
 import Text from 'antd/lib/typography/Text';
 import Tabs from 'antd/lib/tabs';
 import Button from 'antd/lib/button';
@@ -29,6 +29,7 @@ import {
 } from 'actions/annotation-actions';
 import LabelSelector from 'components/label-selector/label-selector';
 import CVATTooltip from 'components/common/cvat-tooltip';
+import { HistogramEqualization } from 'utils/opencv-wrapper/histogram-equalization';
 import withVisibilityHandling from './handle-popover-visibility';
 
 interface Props {
@@ -39,6 +40,7 @@ interface Props {
     states: any[];
     frame: number;
     curZOrder: number;
+    data: any;
 }
 
 interface DispatchToProps {
@@ -53,6 +55,7 @@ interface State {
     initializationError: boolean;
     initializationProgress: number;
     activeLabelID: number;
+    activeImageModifiers: string[];
 }
 
 const core = getCore();
@@ -68,7 +71,7 @@ function mapStateToProps(state: CombinedState): Props {
             job: { instance: jobInstance, labels },
             canvas: { activeControl, instance: canvasInstance },
             player: {
-                frame: { number: frame },
+                frame: { number: frame, data },
             },
         },
     } = state;
@@ -81,6 +84,7 @@ function mapStateToProps(state: CombinedState): Props {
         labels,
         states,
         frame,
+        data,
     };
 }
 
@@ -93,22 +97,27 @@ const mapDispatchToProps = {
 
 class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps, State> {
     private activeTool: IntelligentScissors | null;
+    private activeImageModifier: HistogramEqualization | null;
 
     public constructor(props: Props & DispatchToProps) {
         super(props);
         const { labels } = props;
         this.activeTool = null;
+        this.activeImageModifier = null;
+
         this.state = {
             libraryInitialized: openCVWrapper.isInitialized,
             initializationError: false,
             initializationProgress: -1,
             activeLabelID: labels.length ? labels[0].id : null,
+            activeImageModifiers: [],
         };
     }
 
     public componentDidMount(): void {
         const { canvasInstance } = this.props;
         canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener);
+        canvasInstance.html().addEventListener('canvas.setup', this.setupListener);
     }
 
     public componentDidUpdate(prevProps: Props): void {
@@ -124,8 +133,16 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
     public componentWillUnmount(): void {
         const { canvasInstance } = this.props;
         canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener);
+        canvasInstance.html().removeEventListener('canvas.setup', this.setupListener);
     }
 
+    private setupListener = async ():Promise<void> => {
+        const { frame } = this.props;
+        if (this.activeImageModifier && this.activeImageModifier.currentEqualizedNumber !== frame) {
+            this.runImageModifier();
+        }
+    };
+
     private interactionListener = async (e: Event): Promise<void> => {
         const {
             createAnnotations, isActivated, jobInstance, frame, labels, curZOrder, canvasInstance,
@@ -215,6 +232,33 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
         return points;
     }
 
+    private runImageModifier():void {
+        if (this.activeImageModifier) {
+            const {
+                data, states, curZOrder, canvasInstance, frame,
+            } = this.props;
+            const canvas: HTMLCanvasElement | undefined = window.document.getElementById('cvat_canvas_background') as
+                | HTMLCanvasElement
+                | undefined;
+            if (!canvas) {
+                throw new Error('Element #cvat_canvas_background was not found');
+            }
+            const { width, height } = canvas;
+            const context = canvas.getContext('2d');
+            if (!context) {
+                throw new Error('Canvas context is empty');
+            }
+            const imageData = context.getImageData(0, 0, width, height);
+            this.activeImageModifier.equalize(imageData, frame).then((newBitmap:ImageBitmap|undefined) => {
+                if (newBitmap) {
+                    // eslint-disable-next-line no-underscore-dangle
+                    data._data.imageData = newBitmap;
+                    canvasInstance.setup(data, states, curZOrder, true);
+                }
+            });
+        }
+    }
+
     private renderDrawingContent(): JSX.Element {
         const { activeLabelID } = this.state;
         const { labels, canvasInstance, onInteractionStart } = this.props;
@@ -254,6 +298,49 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
         );
     }
 
+    private renderImageContent():JSX.Element {
+        const { activeImageModifiers } = this.state;
+        const histogramActive = activeImageModifiers.includes('histogram');
+        return (
+            <>
+                <Row justify='start'>
+                    <Col>
+                        <CVATTooltip title='Histogram equalization' className='cvat-opencv-image-tool'>
+                            <Button
+                                className={histogramActive ? 'cvat-opencv-image-tool-active' : ''}
+                                onClick={() => {
+                                    if (!this.activeImageModifier) {
+                                        const { hist } = openCVWrapper;
+                                        this.activeImageModifier = hist;
+                                        this.runImageModifier();
+                                        this.setState({
+                                            activeImageModifiers: ['histogram'],
+                                        });
+                                    } else {
+                                        const {
+                                            data, states, curZOrder, canvasInstance,
+                                        } = this.props;
+                                        this.activeImageModifier.restoreImage().then((newBitmap) => {
+                                            // eslint-disable-next-line no-underscore-dangle
+                                            data._data.imageData = newBitmap;
+                                            canvasInstance.setup(data, states, curZOrder, true);
+                                        });
+                                        this.activeImageModifier = null;
+                                        this.setState({
+                                            activeImageModifiers: [],
+                                        });
+                                    }
+                                }}
+                            >
+                                <AreaChartOutlined />
+                            </Button>
+                        </CVATTooltip>
+                    </Col>
+                </Row>
+            </>
+        );
+    }
+
     private renderContent(): JSX.Element {
         const { libraryInitialized, initializationProgress, initializationError } = this.state;
 
@@ -271,7 +358,9 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
                         <Tabs.TabPane key='drawing' tab='Drawing' className='cvat-opencv-control-tabpane'>
                             {this.renderDrawingContent()}
                         </Tabs.TabPane>
-                        <Tabs.TabPane disabled key='image' tab='Image' className='cvat-opencv-control-tabpane' />
+                        <Tabs.TabPane key='image' tab='Image' className='cvat-opencv-control-tabpane'>
+                            {this.renderImageContent()}
+                        </Tabs.TabPane>
                     </Tabs>
                 ) : (
                     <>

cvat-ui/src/components/annotation-page/standard-workspace/styles.scss

@@ -228,6 +228,15 @@
     }
 }
 
+.cvat-opencv-image-tool {
+    @extend .cvat-opencv-drawing-tool;
+}
+
+.cvat-opencv-image-tool-active {
+    color: #40a9ff;
+    border-color: #40a9ff;
+}
+
 .cvat-setup-tag-popover-content,
 .cvat-draw-shape-popover-content {
     padding: $grid-unit-size;