From 98e5c40f5472369de935c7cde59cef3b2257dc2a Mon Sep 17 00:00:00 2001 From: suttang Date: Fri, 18 Oct 2019 11:25:59 +0900 Subject: [PATCH 1/5] create div2k dataset class --- blueoil/generate_lmnet_config.py | 5 +- lmnet/lmnet/datasets/div2k.py | 51 +++++++++++ .../datasets/DIV2K/DIV2K_train_HR/0001.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_train_HR/0002.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_train_HR/0003.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_train_HR/0004.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_train_HR/0005.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_valid_HR/0001.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_valid_HR/0002.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_valid_HR/0003.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_valid_HR/0004.png | Bin 0 -> 345 bytes .../datasets/DIV2K/DIV2K_valid_HR/0005.png | Bin 0 -> 345 bytes .../lmnet_tests/datasets_tests/test_div2k.py | 83 ++++++++++++++++++ 13 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 lmnet/lmnet/datasets/div2k.py create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0001.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0002.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0003.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0004.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0005.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0001.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0002.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0003.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0004.png create mode 100644 lmnet/tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0005.png create mode 100644 lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py diff --git a/blueoil/generate_lmnet_config.py b/blueoil/generate_lmnet_config.py index b8cc4c255..a4875ebed 100644 --- a/blueoil/generate_lmnet_config.py +++ b/blueoil/generate_lmnet_config.py @@ -76,6 +76,10 @@ "dataset_module": "camvid", "dataset_class": "CamvidCustom", }, + "DIV2K": { + "dataset_module": "div2k", + "dataset_class": "Div2k", + } } @@ -127,7 +131,6 @@ def _blueoil_to_lmnet(blueoil_config): } dataset = {} - model_name = blueoil_config["model_name"] template_file = _TASK_TYPE_TEMPLATE_FILE[blueoil_config["task_type"]] diff --git a/lmnet/lmnet/datasets/div2k.py b/lmnet/lmnet/datasets/div2k.py new file mode 100644 index 000000000..4aa0e4b86 --- /dev/null +++ b/lmnet/lmnet/datasets/div2k.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 The Blueoil Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +import functools +import os +from glob import glob + +from lmnet.datasets.base import Base +from lmnet.utils.image import load_image + + +class Div2k(Base): + classes = [] + num_classes = 0 + extend_dir = "DIV2K" + available_subsets = ["train", "validation"] + + @property + @functools.lru_cache(maxsize=None) + def files(self): + if self.subset == "train": + images_dir = os.path.join(self.data_dir, "DIV2K_train_HR") + else: + images_dir = os.path.join(self.data_dir, "DIV2K_valid_HR") + + return [filepath for filepath in glob(os.path.join(images_dir, "*.png"))] + + @property + def num_per_epoch(self): + return len(self.files) + + def __getitem__(self, i, type=None): + target_file = self.files[i] + image = load_image(target_file) + + return image, None + + def __len__(self): + return self.num_per_epoch diff --git a/lmnet/tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0001.png b/lmnet/tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0001.png new file mode 100644 index 0000000000000000000000000000000000000000..ab1b2b623e9fb09638be3bc6c42d326a190bc436 GIT binary patch literal 345 zcmeAS@N?(olHy`uVBq!ia0vp^DIm``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eB{yR?>$B+ufw-*f=859^;7GD21ck40UfX2;l xdwZ;dT8?mZ3JNK!xOwzUkWe&qN*dw Date: Mon, 28 Oct 2019 13:11:58 +0900 Subject: [PATCH 2/5] fix test for div2k dataset --- .../lmnet_tests/datasets_tests/test_div2k.py | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py b/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py index 2ea070943..8f531e079 100644 --- a/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py +++ b/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py @@ -24,41 +24,54 @@ from lmnet.datasets.div2k import Div2k -@pytest.mark.parametrize("type", [ - {"subset": "train", "dir": "DIV2K_train_HR"}, - {"subset": "validation", "dir": "DIV2K_valid_HR"}, -]) -def test_files(set_test_environment, type): - dataset = Div2k(type["subset"]) +def test_train_files(set_test_environment): + dataset = Div2k("train") files = sorted(dataset.files) - dataset_files = os.path.join(environment.DATA_DIR, "DIV2K/{}/*.png".format(type["dir"])) - expected = sorted(glob(dataset_files)) + expected = [ + "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0001.png", + "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0002.png", + "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0003.png", + "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0004.png", + "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0005.png", + ] assert files == expected -@pytest.mark.parametrize("subset", ["train", "validation"]) -def test_length(set_test_environment, subset): - dataset = Div2k(subset) +def test_validation_files(set_test_environment): + dataset = Div2k("validation") + files = sorted(dataset.files) - assert len(dataset.files) == len(dataset) + expected = [ + "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0001.png", + "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0002.png", + "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0003.png", + "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0004.png", + "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0005.png", + ] + assert files == expected -@pytest.mark.parametrize("subset", ["train", "validation"]) -def test_num_per_epoch(set_test_environment, subset): - dataset = Div2k(subset) - assert len(dataset.files) == dataset.num_per_epoch +def test_length(set_test_environment): + dataset = Div2k("train") + expected = 5 + assert len(dataset) == expected -@pytest.mark.parametrize("subset", ["train", "validation"]) -def test_get_item(set_test_environment, subset): - dataset = Div2k(subset) - for i in range(len(dataset)): - image, _ = dataset[i] - assert isinstance(image, np.ndarray) +def test_num_per_epoch(set_test_environment): + dataset = Div2k("train") + expected = 5 + + assert dataset.num_per_epoch == expected + + +def test_get_item(set_test_environment): + dataset = Div2k("train") + + assert all([isinstance(image, np.ndarray) for image, _ in dataset]) @pytest.mark.parametrize("subset", ["train", "validation"]) From 010d2351b551fee4bc45f0535a8ef7decd461568 Mon Sep 17 00:00:00 2001 From: suttang Date: Mon, 28 Oct 2019 13:48:03 +0900 Subject: [PATCH 3/5] remove unused imports --- lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py b/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py index 8f531e079..f0bdc7d8e 100644 --- a/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py +++ b/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py @@ -13,13 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= -import os -from glob import glob - import numpy as np import pytest -from lmnet import environment from lmnet.datasets.dataset_iterator import DatasetIterator from lmnet.datasets.div2k import Div2k From d73b42f27881c09e36af95833a9592a734f573f4 Mon Sep 17 00:00:00 2001 From: suttang Date: Mon, 28 Oct 2019 13:55:49 +0900 Subject: [PATCH 4/5] remove delete line diff --- blueoil/generate_lmnet_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/blueoil/generate_lmnet_config.py b/blueoil/generate_lmnet_config.py index a4875ebed..4e788a29d 100644 --- a/blueoil/generate_lmnet_config.py +++ b/blueoil/generate_lmnet_config.py @@ -131,6 +131,7 @@ def _blueoil_to_lmnet(blueoil_config): } dataset = {} + model_name = blueoil_config["model_name"] template_file = _TASK_TYPE_TEMPLATE_FILE[blueoil_config["task_type"]] From 0f199365d6cc87b5f60d45b37066c5557b811d1c Mon Sep 17 00:00:00 2001 From: suttang Date: Mon, 28 Oct 2019 13:58:07 +0900 Subject: [PATCH 5/5] made code easy to understand --- .../lmnet_tests/datasets_tests/test_div2k.py | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py b/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py index f0bdc7d8e..4f96588ef 100644 --- a/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py +++ b/lmnet/tests/lmnet_tests/datasets_tests/test_div2k.py @@ -21,9 +21,6 @@ def test_train_files(set_test_environment): - dataset = Div2k("train") - files = sorted(dataset.files) - expected = [ "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0001.png", "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0002.png", @@ -31,14 +28,10 @@ def test_train_files(set_test_environment): "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0004.png", "tests/fixtures/datasets/DIV2K/DIV2K_train_HR/0005.png", ] - - assert files == expected + assert sorted(Div2k("train").files) == expected def test_validation_files(set_test_environment): - dataset = Div2k("validation") - files = sorted(dataset.files) - expected = [ "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0001.png", "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0002.png", @@ -46,28 +39,21 @@ def test_validation_files(set_test_environment): "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0004.png", "tests/fixtures/datasets/DIV2K/DIV2K_valid_HR/0005.png", ] - - assert files == expected + assert sorted(Div2k("validation").files) == expected def test_length(set_test_environment): - dataset = Div2k("train") expected = 5 - - assert len(dataset) == expected + assert len(Div2k("train")) == expected def test_num_per_epoch(set_test_environment): - dataset = Div2k("train") expected = 5 - - assert dataset.num_per_epoch == expected + assert Div2k("train").num_per_epoch == expected def test_get_item(set_test_environment): - dataset = Div2k("train") - - assert all([isinstance(image, np.ndarray) for image, _ in dataset]) + assert all(isinstance(image, np.ndarray) for image, _ in Div2k("train")) @pytest.mark.parametrize("subset", ["train", "validation"])