Merge pull request #1 from signatrix/feature/graph-layout

Human Pose Annotation

.gitignore

@@ -29,4 +29,8 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
+# Ignore goturn model
+goturn.caffemodel
+goturn.prototxt
+
 .DS_Store

CONTRIBUTING.md

@@ -150,6 +150,17 @@ curl https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_co
     export AUTO_SEGMENTATION_PATH="/path/to/dir" # dir must contain mask_rcnn_coco.h5 file
 ```
 
+### Tracking
+- Download the GOTURN model if the the OpenCV GOTURN tracker is to be used.
+```sh
+#This downloads it into the cvat folder
+components/tracking/install.sh
+```
+- Add next lines to ``.env/bin/activate``:
+```sh
+    export TRACKING="yes"
+```
+
 ## JavaScript/Typescript coding style
 
 We use the [Airbnb JavaScript Style Guide](https://github.com/airbnb/javascript) for JavaScript code with a

Dockerfile

@@ -104,6 +104,13 @@ RUN if [ "$AUTO_SEGMENTATION" = "yes" ]; then \
     bash -i /tmp/components/auto_segmentation/install.sh; \
     fi
 
+ARG TRACKING
+ENV TRACKING=${TRACKING}
+ENV TRACKING_PATH=${HOME}/tracking
+RUN if [ "$TRACKING" = "yes" ]; then \
+    bash -i /tmp/components/tracking/install.sh; \
+    fi
+
 # Install and initialize CVAT, copy all necessary files
 COPY cvat/requirements/ /tmp/requirements/
 COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/

components/tracking/README.md

@@ -0,0 +1,19 @@
+## Tracking
+This components allows to track bounding boxes in consecutive images.
+
+### Build docker image
+```bash
+# From project root directory
+docker-compose -f docker-compose.yml -f components/tracking/docker-compose.tracking.yml build
+```
+
+### Run docker container
+```bash
+# From project root directory
+docker-compose -f docker-compose.yml -f components/tracking/docker-compose.tracking.yml up -d
+```
+
+### TODO
+* Make API consistent (one request per tracking job)
+* Put jobs into queque
+* Enable by default?

components/tracking/docker-compose.tracking.yml

@@ -0,0 +1,13 @@
+#
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+version: "2.3"
+
+services:
+  cvat:
+    build:
+      context: .
+      args:
+        TRACKING: "yes"

components/tracking/install.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+# Install GOTURN model for OpenCV; It is not in a package and has to be downloaded
+cd "$(dirname "$0")"
+echo "Downloading GOTURN model. This can take a while."
+wget -q 'https://github.com/opencv/opencv_extra/raw/c4219d5eb3105ed8e634278fad312a1a8d2c182d/testdata/tracking/goturn.prototxt'
+wget -q 'https://github.com/opencv/opencv_extra/raw/c4219d5eb3105ed8e634278fad312a1a8d2c182d/testdata/tracking/goturn.caffemodel.zip.001'
+wget -q 'https://github.com/opencv/opencv_extra/raw/c4219d5eb3105ed8e634278fad312a1a8d2c182d/testdata/tracking/goturn.caffemodel.zip.002'
+wget -q 'https://github.com/opencv/opencv_extra/raw/c4219d5eb3105ed8e634278fad312a1a8d2c182d/testdata/tracking/goturn.caffemodel.zip.003'
+wget -q 'https://github.com/opencv/opencv_extra/raw/c4219d5eb3105ed8e634278fad312a1a8d2c182d/testdata/tracking/goturn.caffemodel.zip.004'
+cat goturn.caffemodel.zip* > goturn.caffemodel.zip
+unzip goturn.caffemodel.zip
+mv goturn.caffemodel goturn.prototxt ../..
+rm goturn.caffemodel.zip*

cvat-canvas/src/typescript/canvasModel.ts

@@ -60,13 +60,16 @@ export interface DrawData {
     numberOfPoints?: number;
     initialState?: any;
     crosshair?: boolean;
+    template?: {
+        vertices: Point2D[];
+        labels: number[];
+        edges: Edge[];
+    }
 }
 
-export interface EditData {
-    enabled: boolean;
-    state: any;
-    pointID: number;
-}
+type Point2D = [number, number];
+
+type Edge = [number, number];
 
 export interface GroupData {
     enabled: boolean;
@@ -100,6 +103,7 @@ export enum UpdateReasons {
     FITTED_CANVAS = 'fitted_canvas',
 
     DRAW = 'draw',
+    DRAW_TEMPLATE = 'draw_template',
     MERGE = 'merge',
     SPLIT = 'split',
     GROUP = 'group',

cvat-canvas/src/typescript/canvasView.ts

@@ -1165,6 +1165,8 @@ export class CanvasViewImpl implements CanvasView, Listener {
                     } else if (state.shapeType === 'cuboid') {
                         this.svgShapes[state.clientID] = this
                             .addCuboid(stringified, state);
+                    } else if (state.shapeType === 'template') {
+                        this.svgShapes[state.clientID] = this.addGraph(translatedPoints, state);
                     }
                 }
 
@@ -1630,6 +1632,22 @@ export class CanvasViewImpl implements CanvasView, Listener {
         return cube;
     }
 
+    private addGraph(points: number[], state: any): any {
+        const { edges = [] } = state;
+        const graph = (this.adoptedContent as any).group();
+
+        const points2D = points.reduce((p, c) => {
+            const last = p[p.length - 1]
+
+            return last === undefined || last.length == 2
+                ? p.concat([[c]])
+                : p.slice(0, -1).concat([last.concat(c)]);
+        }
+        , [] as number[][]);
+
+        return graph;
+    }
+
     private setupPoints(basicPolyline: SVG.PolyLine, state: any): any {
         this.selectize(true, basicPolyline);
 

cvat-canvas/src/typescript/drawHandler.ts

@@ -680,6 +680,8 @@ export class DrawHandlerImpl implements DrawHandler {
                 this.drawPoints();
             } else if (this.drawData.shapeType === 'cuboid') {
                 this.drawCuboid();
+            } else if (this.drawData.shapeType === 'template') {
+                this.drawTemplate();
             }
             this.setupDrawEvents();
         }
@@ -688,6 +690,39 @@ export class DrawHandlerImpl implements DrawHandler {
         this.initialized = true;
     }
 
+    public drawTemplate() {
+        this.drawInstance = this.canvas.rect();
+        this.shapeSizeElement = displayShapeSize(this.canvas, this.text);
+
+        this.drawInstance.on('drawstop', (e: Event): void => {
+            const bbox = (e.target as SVGRectElement).getBBox();
+            const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(bbox);
+            const { shapeType, template } = this.drawData;
+
+            const width = xbr - xtl;
+            const height = ybr - ytl;
+
+            this.release();
+
+            if (this.canceled) return;
+            if (width * height >= consts.AREA_THRESHOLD) {
+                const points = template.vertices
+                    .map(([x, y]) => [width * x + xtl, height * y + ytl]);
+
+                this.onDrawDone({
+                    shapeType,
+                    points,
+                    edges: template.edges,
+                    labels: template.labels,
+                }, Date.now() - this.startTimestamp);
+            }
+        }).on('drawupdate', (): void => {
+            this.shapeSizeElement.update(this.drawInstance);
+        }).addClass('cvat_canvas_shape_drawing').attr({
+            'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale,
+        });
+    }
+
     public constructor(
         onDrawDone: (data: object | null, duration?: number, continueDraw?: boolean) => void,
         canvas: SVG.Container,

cvat-core/src/annotations-collection.js

@@ -18,6 +18,7 @@
         PolygonTrack,
         PolylineTrack,
         PointsTrack,
+        PointsTrackWithTracking,
         CuboidTrack,
         Track,
         Shape,
@@ -90,7 +91,7 @@
                 trackModel = new PolylineTrack(trackData, clientID, color, injection);
                 break;
             case 'points':
-                trackModel = new PointsTrack(trackData, clientID, color, injection);
+                trackModel = new PointsTrackWithTracking(trackData, clientID, color, injection);
                 break;
             case 'cuboid':
                 trackModel = new CuboidTrack(trackData, clientID, color, injection);
@@ -124,6 +125,7 @@
             this.shapes = {}; // key is a frame
             this.tags = {}; // key is a frame
             this.tracks = [];
+            this.trackingData = {};
             this.objects = {}; // key is a client id
             this.count = 0;
             this.flush = false;
@@ -139,6 +141,21 @@
             };
         }
 
+        updateTrackingData(tracking) {
+            this.trackingData = tracking;
+
+            const merge = key => {
+                const oldFrames = this.trackingData[key] || {};
+                const newFrames = tracking[key];
+
+                return {...oldFrames, ...newFrames};
+            };
+
+            for (const key in tracking) {
+                this.trackingData[key] = merge(key);
+            }
+        }
+
         import(data) {
             const result = {
                 tags: [],
@@ -219,7 +236,7 @@
                     continue;
                 }
 
-                const stateData = object.get(frame);
+                const stateData = object.get(frame, this.trackingData ? this.trackingData[object.clientID] : {});
                 if (!allTracks && stateData.outside && !stateData.keyframe) {
                     continue;
                 }

cvat-core/src/annotations-objects.js

@@ -696,7 +696,7 @@
         }
 
         // Method is used to construct ObjectState objects
-        get(frame) {
+        get(frame, trackingData) {
             const {
                 prev,
                 next,
@@ -705,7 +705,7 @@
             } = this.boundedKeyframes(frame);
 
             return {
-                ...this.getPosition(frame, prev, next),
+                ...this.getPosition(frame, prev, next, trackingData),
                 attributes: this.getAttributes(frame),
                 group: this.groupObject,
                 objectType: ObjectType.TRACK,
@@ -1967,6 +1967,83 @@
         }
     }
 
+    class PointsTrackWithTracking extends PointsTrack {
+        constructor(data, clientID, color, injection) {
+            super(data, clientID, color, injection);
+            this.shapeType = ObjectShape.POINTS;
+            this.pinned = false;
+            for (const shape of Object.values(this.shapes)) {
+                checkNumberOfPoints(this.shapeType, shape.points);
+            }
+        }
+
+        getPosition(targetFrame, leftKeyframe, rightFrame, trackingData) {
+            const leftFrame = targetFrame in this.shapes ? targetFrame : leftKeyframe;
+            const rightPosition = Number.isInteger(rightFrame) ? this.shapes[rightFrame] : null;
+            const leftPosition = Number.isInteger(leftFrame) ? this.shapes[leftFrame] : null;
+
+            const points = trackingData ? trackingData[targetFrame] : undefined;
+            const isKeyFrame = targetFrame in this.shapes;
+            const targetPositionPoints = isKeyFrame ? this.shapes[targetFrame].points : undefined;
+
+            if (targetPositionPoints) {
+                return {
+                    points: [...targetPositionPoints],
+                    occluded: leftPosition.occluded,
+                    outside: leftPosition.outside,
+                    zOrder: leftPosition.zOrder,
+                    keyframe: targetFrame in this.shapes,
+                }
+            }
+
+            if (points) {
+                return {
+                    points: [...points],
+                    occluded: leftPosition.occluded,
+                    outside: leftPosition.outside,
+                    zOrder: leftPosition.zOrder,
+                    keyframe: targetFrame in this.shapes,
+                }
+            }
+
+            if (leftPosition && rightPosition) {
+                return {
+                    ...this.interpolatePosition(
+                        leftPosition,
+                        rightPosition,
+                        (targetFrame - leftFrame) / (rightFrame - leftFrame),
+                    ),
+                    keyframe: targetFrame in this.shapes,
+                };
+            }
+
+            if (leftPosition) {
+                return {
+                    points: [...leftPosition.points],
+                    occluded: leftPosition.occluded,
+                    outside: leftPosition.outside,
+                    zOrder: leftPosition.zOrder,
+                    keyframe: targetFrame in this.shapes,
+                };
+            }
+
+            if (rightPosition) {
+                return {
+                    points: [...rightPosition.points],
+                    occluded: rightPosition.occluded,
+                    outside: true,
+                    zOrder: rightPosition.zOrder,
+                    keyframe: targetFrame in this.shapes,
+                };
+            }
+
+            throw new DataError(
+                'No one left position or right position was found. '
+                + `Interpolation impossible. Client ID: ${this.clientID}`,
+            );
+        }
+    }
+
     class CuboidTrack extends PolyTrack {
         constructor(data, clientID, color, injection) {
             super(data, clientID, color, injection);
@@ -1982,6 +2059,7 @@
     PolygonTrack.distance = PolygonShape.distance;
     PolylineTrack.distance = PolylineShape.distance;
     PointsTrack.distance = PointsShape.distance;
+    PointsTrackWithTracking.distance = PointsShape.distance;
     CuboidTrack.distance = CuboidShape.distance;
     CuboidTrack.interpolatePosition = RectangleTrack.interpolatePosition;
 
@@ -1995,6 +2073,7 @@
         PolygonTrack,
         PolylineTrack,
         PointsTrack,
+        PointsTrackWithTracking,
         CuboidTrack,
         Track,
         Shape,