From 045700cfbe52677a7617557c2594885c45b71f31 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Thu, 3 Jun 2021 11:36:58 +0300 Subject: [PATCH 1/5] add test for wrong attribute import in tracks --- .../tests/assets/annotations.json | 45 ++++ .../dataset_manager/tests/assets/tasks.json | 15 ++ .../tests/test_rest_api_formats.py | 233 ++++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 cvat/apps/dataset_manager/tests/assets/annotations.json create mode 100644 cvat/apps/dataset_manager/tests/assets/tasks.json create mode 100644 cvat/apps/dataset_manager/tests/test_rest_api_formats.py diff --git a/cvat/apps/dataset_manager/tests/assets/annotations.json b/cvat/apps/dataset_manager/tests/assets/annotations.json new file mode 100644 index 000000000000..204a2721ed55 --- /dev/null +++ b/cvat/apps/dataset_manager/tests/assets/annotations.json @@ -0,0 +1,45 @@ +{ + "CVAT for video 1.1 many jobs": { + "version": 0, + "tags": [], + "shapes": [], + "tracks": [ + { + "frame": 0, + "label_id": null, + "group": 0, + "source": "manual", + "shapes": [ + { + "type": "rectangle", + "occluded": true, + "z_order": 0, + "points": [4.75, 4.8, 13.06, 11.39], + "frame": 1, + "outside": false, + "attributes": [] + } + ], + "attributes": [] + }, + { + "frame": 10, + "label_id": null, + "group": 1, + "source": "manual", + "shapes": [ + { + "type": "polygon", + "occluded": false, + "z_order": 0, + "points": [24.62, 13.01, 34.88, 20.03, 18.14, 18.08], + "frame": 1, + "outside": false, + "attributes": [] + } + ], + "attributes": [] + } + ] + } +} diff --git a/cvat/apps/dataset_manager/tests/assets/tasks.json b/cvat/apps/dataset_manager/tests/assets/tasks.json new file mode 100644 index 000000000000..bbda66a858fe --- /dev/null +++ b/cvat/apps/dataset_manager/tests/assets/tasks.json @@ -0,0 +1,15 @@ +{ + "many jobs": { + "name": "many jobs", + "overlap": 0, + "segment_size": 5, + "owner_id": 1, + "labels": [ + { + "name": "car", + "color": "#2080c0", + "attributes": [] + } + ] + } +} diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py new file mode 100644 index 000000000000..863ba29a9849 --- /dev/null +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -0,0 +1,233 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import copy +import json +import os.path as osp +import random +from io import BytesIO + +from datumaro.components.dataset import Dataset +from datumaro.util.test_utils import TestDir, compare_datasets +from django.contrib.auth.models import Group, User +from PIL import Image +from rest_framework import status +from rest_framework.test import APIClient, APITestCase + +from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, TaskData +from cvat.apps.dataset_manager.task import TaskAnnotation +from cvat.apps.engine.models import Task + +path = osp.join(osp.dirname(__file__), 'assets', 'tasks.json') +with open(path) as f: + tasks = json.load(f) + +path = osp.join(osp.dirname(__file__), 'assets', 'annotations.json') +with open(path) as f: + annotations = json.load(f) + +def generate_image_file(filename, size=(100, 50)): + f = BytesIO() + image = Image.new('RGB', size=size) + image.save(f, 'jpeg') + f.name = filename + f.seek(0) + return f + +class ForceLogin: + def __init__(self, user, client): + self.user = user + self.client = client + + def __enter__(self): + if self.user: + self.client.force_login(self.user, + backend='django.contrib.auth.backends.ModelBackend') + + return self + + def __exit__(self, exception_type, exception_value, traceback): + if self.user: + self.client.logout() + +class _DbTestBase(APITestCase): + def setUp(self): + self.client = APIClient() + + @classmethod + def setUpTestData(cls): + cls.create_db_users() + + @classmethod + def create_db_users(cls): + (group_admin, _) = Group.objects.get_or_create(name="admin") + (group_user, _) = Group.objects.get_or_create(name="user") + + user_admin = User.objects.create_superuser(username="admin", email="", + password="admin") + user_admin.groups.add(group_admin) + user_dummy = User.objects.create_user(username="user", password="user") + user_dummy.groups.add(group_user) + + cls.admin = user_admin + cls.user = user_dummy + + def _put_api_v1_task_id_annotations(self, tid, data): + with ForceLogin(self.admin, self.client): + response = self.client.put("/api/v1/tasks/%s/annotations" % tid, + data=data, format="json") + + return response + + @staticmethod + def _generate_task_images(count): # pylint: disable=no-self-use + images = {"client_files[%d]" % i: generate_image_file("image_%d.jpg" % i) for i in range(count)} + images["image_quality"] = 75 + return images + + def _create_task(self, data, image_data): + with ForceLogin(self.user, self.client): + response = self.client.post('/api/v1/tasks', data=data, format="json") + assert response.status_code == status.HTTP_201_CREATED, response.status_code + tid = response.data["id"] + + response = self.client.post("/api/v1/tasks/%s/data" % tid, + data=image_data) + assert response.status_code == status.HTTP_202_ACCEPTED, response.status_code + + response = self.client.get("/api/v1/tasks/%s" % tid) + task = response.data + + return task + + def _get_request_with_data(self, path, data, user): + with ForceLogin(user, self.client): + response = self.client.get(path, data) + return response + + def _put_request_with_data(self, path, data, user): + with ForceLogin(user, self.client): + response = self.client.put(path, data) + return response + + def _delete_request(self, path, user): + with ForceLogin(user, self.client): + response = self.client.delete(path) + return response + + def _create_annotations(self, task, name_ann, key_get_values): + tmp_annotations = copy.deepcopy(annotations[name_ann]) + + # change attributes in all annotations + for item in tmp_annotations: + if item in ["tags", "shapes", "tracks"]: + for index_elem, _ in enumerate(tmp_annotations[item]): + tmp_annotations[item][index_elem]["label_id"] = task["labels"][0]["id"] + + for index_attribute, attribute in enumerate(task["labels"][0]["attributes"]): + spec_id = task["labels"][0]["attributes"][index_attribute]["id"] + + if key_get_values == "random": + if attribute["input_type"] == "number": + start = int(attribute["values"][0]) + stop = int(attribute["values"][1]) + 1 + step = int(attribute["values"][2]) + value = str(random.randrange(start, stop, step)) + else: + value = random.choice(task["labels"][0]["attributes"][index_attribute]["values"]) + elif key_get_values == "default": + value = attribute["default_value"] + + if item == "tracks" and attribute["mutable"]: + for index_shape, _ in enumerate(tmp_annotations[item][index_elem]["shapes"]): + tmp_annotations[item][index_elem]["shapes"][index_shape]["attributes"].append({ + "spec_id": spec_id, + "value": value, + }) + else: + tmp_annotations[item][index_elem]["attributes"].append({ + "spec_id": spec_id, + "value": value, + }) + + response = self._put_api_v1_task_id_annotations(task["id"], tmp_annotations) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def _download_file(self, url, data, user, file_name): + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + content = BytesIO(b"".join(response.streaming_content)) + with open(file_name, "wb") as f: + f.write(content.getvalue()) + + def _upload_file(self, url, data, user): + response = self._put_request_with_data(url, {"annotation_file": data}, user) + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + response = self._put_request_with_data(url, {}, user) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + def _check_downloaded_file(self, file_name): + if not osp.exists(file_name): + raise FileNotFoundError(f"File '{file_name}' was not downloaded") + + def _generate_url_dump_tasks_annotations(self, task_id): + return f"/api/v1/tasks/{task_id}/annotations" + + def _generate_url_upload_tasks_annotations(self, task_id, upload_format_name): + return f"/api/v1/tasks/{task_id}/annotations?format={upload_format_name}" + + def _remove_annotations(self, url, user): + response = self._delete_request(url, user) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + return response + +class TaskDumpUploadTest(_DbTestBase): + def test_attribute_import_in_tracks(self): + test_name = self._testMethodName + dump_format_name = "CVAT for video 1.1" + upload_format_name = "CVAT 1.1" + + for include_images in (False, True): + with self.subTest(): + # create task with annotations + images = self._generate_task_images(13) + task = self._create_task(tasks["many jobs"], images) + self._create_annotations(task, f'{dump_format_name} many jobs', "dafault") + + task_id = task["id"] + task_ann = TaskAnnotation(task_id) + task_ann.init_from_db() + task_data = TaskData(task_ann.ir_data, Task.objects.get(pk=task_id)) + extractor = CvatTaskDataExtractor(task_data, include_images=include_images) + data_from_task_before_upload = Dataset.from_extractors(extractor) + + # dump annotations + url = self._generate_url_dump_tasks_annotations(task_id) + data = { + "format": dump_format_name, + "action": "download", + } + with TestDir() as test_dir: + file_zip_name = osp.join(test_dir, f'{test_name}_{dump_format_name}.zip') + self._download_file(url, data, self.admin, file_zip_name) + self._check_downloaded_file(file_zip_name) + + # remove annotations + self._remove_annotations(url, self.admin) + + # upload annotations + url = self._generate_url_upload_tasks_annotations(task_id, upload_format_name) + with open(file_zip_name, 'rb') as binary_file: + self._upload_file(url, binary_file, self.admin) + + # equals annotations + task_ann = TaskAnnotation(task_id) + task_ann.init_from_db() + task_data = TaskData(task_ann.ir_data, Task.objects.get(pk=task_id)) + extractor = CvatTaskDataExtractor(task_data, include_images=include_images) + data_from_task_after_upload = Dataset.from_extractors(extractor) + compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) From 05ca6090abddb30b2349ff885f6a25400c85ba4a Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Thu, 3 Jun 2021 11:57:56 +0300 Subject: [PATCH 2/5] fix annotation for test --- .../tests/assets/annotations.json | 16 ++++++++-------- .../tests/test_rest_api_formats.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/assets/annotations.json b/cvat/apps/dataset_manager/tests/assets/annotations.json index 204a2721ed55..22b6a3b4a118 100644 --- a/cvat/apps/dataset_manager/tests/assets/annotations.json +++ b/cvat/apps/dataset_manager/tests/assets/annotations.json @@ -1,5 +1,5 @@ { - "CVAT for video 1.1 many jobs": { + "CVAT for video 1.1 attributes in tracks": { "version": 0, "tags": [], "shapes": [], @@ -11,10 +11,10 @@ "source": "manual", "shapes": [ { - "type": "rectangle", - "occluded": true, + "type": "polygon", + "occluded": false, "z_order": 0, - "points": [4.75, 4.8, 13.06, 11.39], + "points": [25.04, 13.7, 35.85, 20.2, 16.65, 19.8], "frame": 1, "outside": false, "attributes": [] @@ -23,16 +23,16 @@ "attributes": [] }, { - "frame": 10, + "frame": 8, "label_id": null, "group": 1, "source": "manual", "shapes": [ { - "type": "polygon", - "occluded": false, + "type": "rectangle", + "occluded": true, "z_order": 0, - "points": [24.62, 13.01, 34.88, 20.03, 18.14, 18.08], + "points": [5.54, 3.5, 19.64, 11.19], "frame": 1, "outside": false, "attributes": [] diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 863ba29a9849..95b5b99d3113 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -186,7 +186,7 @@ def _remove_annotations(self, url, user): return response class TaskDumpUploadTest(_DbTestBase): - def test_attribute_import_in_tracks(self): + def test_api_v1_check_attribute_import_in_tracks(self): test_name = self._testMethodName dump_format_name = "CVAT for video 1.1" upload_format_name = "CVAT 1.1" @@ -196,7 +196,7 @@ def test_attribute_import_in_tracks(self): # create task with annotations images = self._generate_task_images(13) task = self._create_task(tasks["many jobs"], images) - self._create_annotations(task, f'{dump_format_name} many jobs', "dafault") + self._create_annotations(task, f'{dump_format_name} attributes in tracks', "dafault") task_id = task["id"] task_ann = TaskAnnotation(task_id) From d84f8d8030271fa6b1418e88b73c160c9ece83e3 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Fri, 4 Jun 2021 10:51:07 +0300 Subject: [PATCH 3/5] small fix --- .../dataset_manager/tests/test_rest_api_formats.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 95b5b99d3113..7ff84a5fd3cc 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -19,12 +19,12 @@ from cvat.apps.dataset_manager.task import TaskAnnotation from cvat.apps.engine.models import Task -path = osp.join(osp.dirname(__file__), 'assets', 'tasks.json') -with open(path) as f: +tasks_path = osp.join(osp.dirname(__file__), 'assets', 'tasks.json') +with open(tasks_path) as f: tasks = json.load(f) -path = osp.join(osp.dirname(__file__), 'assets', 'annotations.json') -with open(path) as f: +annotations_path = osp.join(osp.dirname(__file__), 'assets', 'annotations.json') +with open(annotations_path) as f: annotations = json.load(f) def generate_image_file(filename, size=(100, 50)): @@ -196,7 +196,7 @@ def test_api_v1_check_attribute_import_in_tracks(self): # create task with annotations images = self._generate_task_images(13) task = self._create_task(tasks["many jobs"], images) - self._create_annotations(task, f'{dump_format_name} attributes in tracks', "dafault") + self._create_annotations(task, f'{dump_format_name} attributes in tracks', "default") task_id = task["id"] task_ann = TaskAnnotation(task_id) From 7068d29dbc9bb57b449c9b952743944add6ad83a Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 9 Jun 2021 18:08:59 +0300 Subject: [PATCH 4/5] fix annotattion for test --- .../tests/assets/annotations.json | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/assets/annotations.json b/cvat/apps/dataset_manager/tests/assets/annotations.json index 0fd097ef4d26..c70dbcb598b5 100644 --- a/cvat/apps/dataset_manager/tests/assets/annotations.json +++ b/cvat/apps/dataset_manager/tests/assets/annotations.json @@ -59,27 +59,19 @@ "occluded": false, "z_order": 0, "points": [25.04, 13.7, 35.85, 20.2, 16.65, 19.8], - "frame": 1, - "outside": false, + "frame": 0, + "outside": true, "attributes": [] - } - ], - "attributes": [] - }, - { - "frame": 8, - "label_id": null, - "group": 1, - "source": "manual", - "shapes": [ + }, { - "type": "rectangle", - "occluded": true, + "type": "polygon", + "occluded": false, "z_order": 0, - "points": [5.54, 3.5, 19.64, 11.19], + "points": [25.04, 13.7, 35.85, 20.2, 16.65, 19.8], "frame": 1, "outside": false, - "attributes": [] + "attributes": [], + "keyframe": true } ], "attributes": [] From 12faa55eb5382fb8136367d538126544beae9435 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 9 Jun 2021 18:20:25 +0300 Subject: [PATCH 5/5] add function to get data from task --- .../tests/test_rest_api_formats.py | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index f16fd937bfab..14af4154922e 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -103,6 +103,13 @@ def _create_task(self, data, image_data): return task + def _get_data_from_task(self, task_id, include_images): + task_ann = TaskAnnotation(task_id) + task_ann.init_from_db() + task_data = TaskData(task_ann.ir_data, Task.objects.get(pk=task_id)) + extractor = CvatTaskDataExtractor(task_data, include_images=include_images) + return Dataset.from_extractors(extractor) + def _get_request_with_data(self, path, data, user): with ForceLogin(user, self.client): response = self.client.get(path, data) @@ -231,11 +238,7 @@ def test_api_v1_check_widerface_with_all_attributes(self): self._create_annotations(task, f'{dump_format_name}', "random") task_id = task["id"] - task_ann = TaskAnnotation(task_id) - task_ann.init_from_db() - task_data = TaskData(task_ann.ir_data, Task.objects.get(pk=task_id)) - extractor = CvatTaskDataExtractor(task_data, include_images=include_images) - data_from_task_before_upload = Dataset.from_extractors(extractor) + data_from_task_before_upload = self._get_data_from_task(task_id, include_images) # dump annotations url = self._generate_url_dump_tasks_annotations(task_id) @@ -257,11 +260,7 @@ def test_api_v1_check_widerface_with_all_attributes(self): self._upload_file(url, binary_file, self.admin) # equals annotations - task_ann = TaskAnnotation(task_id) - task_ann.init_from_db() - task_data = TaskData(task_ann.ir_data, Task.objects.get(pk=task_id)) - extractor = CvatTaskDataExtractor(task_data, include_images=include_images) - data_from_task_after_upload = Dataset.from_extractors(extractor) + data_from_task_after_upload = self._get_data_from_task(task_id, include_images) compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload)\ def test_api_v1_check_attribute_import_in_tracks(self): @@ -277,11 +276,7 @@ def test_api_v1_check_attribute_import_in_tracks(self): self._create_annotations(task, f'{dump_format_name} attributes in tracks', "default") task_id = task["id"] - task_ann = TaskAnnotation(task_id) - task_ann.init_from_db() - task_data = TaskData(task_ann.ir_data, Task.objects.get(pk=task_id)) - extractor = CvatTaskDataExtractor(task_data, include_images=include_images) - data_from_task_before_upload = Dataset.from_extractors(extractor) + data_from_task_before_upload = self._get_data_from_task(task_id, include_images) # dump annotations url = self._generate_url_dump_tasks_annotations(task_id) @@ -303,9 +298,5 @@ def test_api_v1_check_attribute_import_in_tracks(self): self._upload_file(url, binary_file, self.admin) # equals annotations - task_ann = TaskAnnotation(task_id) - task_ann.init_from_db() - task_data = TaskData(task_ann.ir_data, Task.objects.get(pk=task_id)) - extractor = CvatTaskDataExtractor(task_data, include_images=include_images) - data_from_task_after_upload = Dataset.from_extractors(extractor) + data_from_task_after_upload = self._get_data_from_task(task_id, include_images) compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload)