Skip to content

Commit

Permalink
fix bugs.
Browse files Browse the repository at this point in the history
  • Loading branch information
@donnyyou
donnyyou committed Jul 26, 2019
1 parent 98497db commit 524fda5
Show file tree
Hide file tree
Showing 83 changed files with 307 additions and 309 deletions.
24 changes: 12 additions & 12 deletions datasets/cls/data_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,35 +23,35 @@ class DataLoader(object):
def __init__(self, configer):
self.configer = configer

if self.configer.get('datasets', 'image_tool') == 'pil':
if self.configer.get('data', 'image_tool') == 'pil':
self.aug_train_transform = pil_aug_trans.PILAugCompose(self.configer, split='train')
elif self.configer.get('datasets', 'image_tool') == 'cv2':
elif self.configer.get('data', 'image_tool') == 'cv2':
self.aug_train_transform = cv2_aug_trans.CV2AugCompose(self.configer, split='train')
else:
Log.error('Not support {} image tool.'.format(self.configer.get('datasets', 'image_tool')))
Log.error('Not support {} image tool.'.format(self.configer.get('data', 'image_tool')))
exit(1)

if self.configer.get('datasets', 'image_tool') == 'pil':
if self.configer.get('data', 'image_tool') == 'pil':
self.aug_val_transform = pil_aug_trans.PILAugCompose(self.configer, split='val')
elif self.configer.get('datasets', 'image_tool') == 'cv2':
elif self.configer.get('data', 'image_tool') == 'cv2':
self.aug_val_transform = cv2_aug_trans.CV2AugCompose(self.configer, split='val')
else:
Log.error('Not support {} image tool.'.format(self.configer.get('datasets', 'image_tool')))
Log.error('Not support {} image tool.'.format(self.configer.get('data', 'image_tool')))
exit(1)

self.img_transform = trans.Compose([
trans.ToTensor(),
trans.Normalize(**self.configer.get('datasets', 'normalize')), ])
trans.Normalize(**self.configer.get('data', 'normalize')), ])

def get_trainloader(self):
if not self.configer.exists('train', 'loader') or self.configer.get('train', 'loader') == 'default':
trainloader = data.DataLoader(
DefaultLoader(root_dir=self.configer.get('datasets', 'data_dir'), dataset='train',
DefaultLoader(root_dir=self.configer.get('data', 'data_dir'), dataset='train',
aug_transform=self.aug_train_transform,
img_transform=self.img_transform, configer=self.configer),
batch_size=self.configer.get('train', 'batch_size'), shuffle=True,
num_workers=self.configer.get('datasets', 'workers'), pin_memory=True,
drop_last=self.configer.get('datasets', 'drop_last'),
num_workers=self.configer.get('data', 'workers'), pin_memory=True,
drop_last=self.configer.get('data', 'drop_last'),
collate_fn=lambda *args: collate(
*args, trans_dict=self.configer.get('train', 'data_transformer')
)
Expand All @@ -67,11 +67,11 @@ def get_valloader(self, dataset=None):
dataset = 'val' if dataset is None else dataset
if not self.configer.exists('val', 'loader') or self.configer.get('val', 'loader') == 'default':
valloader = data.DataLoader(
DefaultLoader(root_dir=self.configer.get('datasets', 'data_dir'), dataset=dataset,
DefaultLoader(root_dir=self.configer.get('data', 'data_dir'), dataset=dataset,
aug_transform=self.aug_val_transform,
img_transform=self.img_transform, configer=self.configer),
batch_size=self.configer.get('val', 'batch_size'), shuffle=False,
num_workers=self.configer.get('datasets', 'workers'), pin_memory=True,
num_workers=self.configer.get('data', 'workers'), pin_memory=True,
collate_fn=lambda *args: collate(
*args, trans_dict=self.configer.get('val', 'data_transformer')
)
Expand Down
6 changes: 3 additions & 3 deletions datasets/cls/loader/default_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ def __init__(self, root_dir=None, dataset=None, aug_transform=None, img_transfor

def __getitem__(self, index):
img = ImageHelper.read_image(self.img_list[index],
tool=self.configer.get('datasets', 'image_tool'),
mode=self.configer.get('datasets', 'input_mode'))
tool=self.configer.get('data', 'image_tool'),
mode=self.configer.get('data', 'input_mode'))
label = self.label_list[index]

if self.aug_transform is not None:
Expand Down Expand Up @@ -57,7 +57,7 @@ def __read_json_file(self, root_dir, dataset):
img_list.append(img_path)
label_list.append(item['label'])

if dataset == 'train' and self.configer.get('datasets', 'include_val'):
if dataset == 'train' and self.configer.get('data', 'include_val'):
with open(os.path.join(root_dir, 'val', 'label.json'), 'r') as file_stream:
items = json.load(file_stream)
for item in items:
Expand Down
8 changes: 4 additions & 4 deletions datasets/cls/preprocess/cifar/cifar_cls_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def __init__(self, args, image_dir=IMAGE_DIR):
entry = pickle.load(f)
else:
entry = pickle.load(f, encoding='latin1')
self.train_data.append(entry['datasets'])
self.train_data.append(entry['data'])
if 'labels' in entry:
self.train_targets.extend(entry['labels'])
else:
Expand All @@ -87,7 +87,7 @@ def __init__(self, args, image_dir=IMAGE_DIR):
entry = pickle.load(f)
else:
entry = pickle.load(f, encoding='latin1')
self.test_data.append(entry['datasets'])
self.test_data.append(entry['data'])
if 'labels' in entry:
self.test_targets.extend(entry['labels'])
else:
Expand Down Expand Up @@ -221,9 +221,9 @@ class Cifar100ClsGenerator(Cifar10ClsGenerator):

parser = argparse.ArgumentParser()
parser.add_argument('--save_dir', default=None, type=str,
dest='save_dir', help='The directory to save the datasets.')
dest='save_dir', help='The directory to save the data.')
parser.add_argument('--root_dir', default=None, type=str,
dest='root_dir', help='The directory of the image datasets.')
dest='root_dir', help='The directory of the image data.')
parser.add_argument('--dataset', default="cifar10", type=str,
dest='dataset', help='The dataset name.')
args = parser.parse_args()
Expand Down
6 changes: 3 additions & 3 deletions datasets/cls/preprocess/flower/flower_cls_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,11 @@ def generate_label(self):

parser = argparse.ArgumentParser()
parser.add_argument('--save_dir', default=None, type=str,
dest='save_dir', help='The directory to save the datasets.')
dest='save_dir', help='The directory to save the data.')
parser.add_argument('--ori_img_dir', default=None, type=str,
dest='ori_img_dir', help='The directory of the image datasets.')
dest='ori_img_dir', help='The directory of the image data.')
parser.add_argument('--val_interval', default=10, type=float,
dest='val_interval', help='The ratio of train & val datasets.')
dest='val_interval', help='The ratio of train & val data.')

args = parser.parse_args()

Expand Down
4 changes: 2 additions & 2 deletions datasets/cls/preprocess/imagenet/imagenet_cls_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ def generate_label(self):

parser = argparse.ArgumentParser()
parser.add_argument('--save_dir', default=None, type=str,
dest='save_dir', help='The directory to save the datasets.')
dest='save_dir', help='The directory to save the data.')
parser.add_argument('--ori_img_dir', default=None, type=str,
dest='ori_img_dir', help='The directory of the image datasets.')
dest='ori_img_dir', help='The directory of the image data.')

args = parser.parse_args()

Expand Down
34 changes: 17 additions & 17 deletions datasets/det/data_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,36 +19,36 @@ class DataLoader(object):
def __init__(self, configer):
self.configer = configer

if self.configer.get('datasets', 'image_tool') == 'pil':
if self.configer.get('data', 'image_tool') == 'pil':
self.aug_train_transform = pil_aug_trans.PILAugCompose(self.configer, split='train')
elif self.configer.get('datasets', 'image_tool') == 'cv2':
elif self.configer.get('data', 'image_tool') == 'cv2':
self.aug_train_transform = cv2_aug_trans.CV2AugCompose(self.configer, split='train')
else:
Log.error('Not support {} image tool.'.format(self.configer.get('datasets', 'image_tool')))
Log.error('Not support {} image tool.'.format(self.configer.get('data', 'image_tool')))
exit(1)

if self.configer.get('datasets', 'image_tool') == 'pil':
if self.configer.get('data', 'image_tool') == 'pil':
self.aug_val_transform = pil_aug_trans.PILAugCompose(self.configer, split='val')
elif self.configer.get('datasets', 'image_tool') == 'cv2':
elif self.configer.get('data', 'image_tool') == 'cv2':
self.aug_val_transform = cv2_aug_trans.CV2AugCompose(self.configer, split='val')
else:
Log.error('Not support {} image tool.'.format(self.configer.get('datasets', 'image_tool')))
Log.error('Not support {} image tool.'.format(self.configer.get('data', 'image_tool')))
exit(1)

self.img_transform = trans.Compose([
trans.ToTensor(),
trans.Normalize(**self.configer.get('datasets', 'normalize')), ])
trans.Normalize(**self.configer.get('data', 'normalize')), ])

def get_trainloader(self):
if not self.configer.exists('train', 'loader') or self.configer.get('train', 'loader') == 'default':
trainloader = data.DataLoader(
DefaultLoader(root_dir=self.configer.get('datasets', 'data_dir'), dataset='train',
DefaultLoader(root_dir=self.configer.get('data', 'data_dir'), dataset='train',
aug_transform=self.aug_train_transform,
img_transform=self.img_transform,
configer=self.configer),
batch_size=self.configer.get('train', 'batch_size'), shuffle=True,
num_workers=self.configer.get('datasets', 'workers'), pin_memory=True,
drop_last=self.configer.get('datasets', 'drop_last'),
num_workers=self.configer.get('data', 'workers'), pin_memory=True,
drop_last=self.configer.get('data', 'drop_last'),
collate_fn=lambda *args: collate(
*args, trans_dict=self.configer.get('train', 'data_transformer')
)
Expand All @@ -58,13 +58,13 @@ def get_trainloader(self):

elif self.configer.get('train', 'loader') == 'fasterrcnn':
trainloader = data.DataLoader(
FasterRCNNLoader(root_dir=self.configer.get('datasets', 'data_dir'), dataset='train',
FasterRCNNLoader(root_dir=self.configer.get('data', 'data_dir'), dataset='train',
aug_transform=self.aug_train_transform,
img_transform=self.img_transform,
configer=self.configer),
batch_size=self.configer.get('train', 'batch_size'), shuffle=True,
num_workers=self.configer.get('datasets', 'workers'), pin_memory=True,
drop_last=self.configer.get('datasets', 'drop_last'),
num_workers=self.configer.get('data', 'workers'), pin_memory=True,
drop_last=self.configer.get('data', 'drop_last'),
collate_fn=lambda *args: collate(
*args, trans_dict=self.configer.get('train', 'data_transformer')
)
Expand All @@ -79,12 +79,12 @@ def get_valloader(self, dataset=None):
dataset = 'val' if dataset is None else dataset
if not self.configer.exists('val', 'loader') or self.configer.get('val', 'loader') == 'default':
valloader = data.DataLoader(
DefaultLoader(root_dir=self.configer.get('datasets', 'data_dir'), dataset=dataset,
DefaultLoader(root_dir=self.configer.get('data', 'data_dir'), dataset=dataset,
aug_transform=self.aug_val_transform,
img_transform=self.img_transform,
configer=self.configer),
batch_size=self.configer.get('val', 'batch_size'), shuffle=False,
num_workers=self.configer.get('datasets', 'workers'), pin_memory=True,
num_workers=self.configer.get('data', 'workers'), pin_memory=True,
collate_fn=lambda *args: collate(
*args, trans_dict=self.configer.get('val', 'data_transformer')
)
Expand All @@ -94,12 +94,12 @@ def get_valloader(self, dataset=None):

elif self.configer.get('val', 'loader') == 'fasterrcnn':
valloader = data.DataLoader(
FasterRCNNLoader(root_dir=self.configer.get('datasets', 'data_dir'), dataset=dataset,
FasterRCNNLoader(root_dir=self.configer.get('data', 'data_dir'), dataset=dataset,
aug_transform=self.aug_val_transform,
img_transform=self.img_transform,
configer=self.configer),
batch_size=self.configer.get('val', 'batch_size'), shuffle=False,
num_workers=self.configer.get('datasets', 'workers'), pin_memory=True,
num_workers=self.configer.get('data', 'workers'), pin_memory=True,
collate_fn=lambda *args: collate(
*args, trans_dict=self.configer.get('val', 'data_transformer')
)
Expand Down
8 changes: 4 additions & 4 deletions datasets/det/loader/default_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ def __init__(self, root_dir=None, dataset=None,

def __getitem__(self, index):
img = ImageHelper.read_image(self.img_list[index],
tool=self.configer.get('datasets', 'image_tool'),
mode=self.configer.get('datasets', 'input_mode'))
tool=self.configer.get('data', 'image_tool'),
mode=self.configer.get('data', 'input_mode'))
img_size = ImageHelper.get_size(img)
bboxes, labels = self.__read_json_file(self.json_list[index])
ori_bboxes, ori_labels = bboxes.copy(), labels.copy()
Expand Down Expand Up @@ -73,7 +73,7 @@ def __read_json_file(self, json_file):
bboxes = list()

for object in json_dict['objects']:
if 'difficult' in object and object['difficult'] and not self.configer.get('datasets', 'keep_difficult'):
if 'difficult' in object and object['difficult'] and not self.configer.get('data', 'keep_difficult'):
continue

labels.append(object['label'])
Expand All @@ -98,7 +98,7 @@ def __list_dirs(self, root_dir, dataset):
json_list.append(json_path)
img_list.append(img_path)

if dataset == 'train' and self.configer.get('datasets', 'include_val'):
if dataset == 'train' and self.configer.get('data', 'include_val'):
image_dir = os.path.join(root_dir, 'val/image')
json_dir = os.path.join(root_dir, 'val/json')
for file_name in os.listdir(json_dir):
Expand Down
8 changes: 4 additions & 4 deletions datasets/det/loader/fasterrcnn_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ def __init__(self, root_dir=None, dataset=None,

def __getitem__(self, index):
img = ImageHelper.read_image(self.img_list[index],
tool=self.configer.get('datasets', 'image_tool'),
mode=self.configer.get('datasets', 'input_mode'))
tool=self.configer.get('data', 'image_tool'),
mode=self.configer.get('data', 'input_mode'))

img_size = ImageHelper.get_size(img)
bboxes, labels = self.__read_json_file(self.json_list[index])
Expand Down Expand Up @@ -75,7 +75,7 @@ def __read_json_file(self, json_file):
bboxes = list()

for object in json_dict['objects']:
if 'difficult' in object and object['difficult'] and not self.configer.get('datasets', 'keep_difficult'):
if 'difficult' in object and object['difficult'] and not self.configer.get('data', 'keep_difficult'):
continue

labels.append(object['label'])
Expand All @@ -100,7 +100,7 @@ def __list_dirs(self, root_dir, dataset):
json_list.append(json_path)
img_list.append(img_path)

if dataset == 'train' and self.configer.get('datasets', 'include_val'):
if dataset == 'train' and self.configer.get('data', 'include_val'):
image_dir = os.path.join(root_dir, 'val/image')
json_dir = os.path.join(root_dir, 'val/json')
for file_name in os.listdir(json_dir):
Expand Down
4 changes: 2 additions & 2 deletions datasets/det/preprocess/coco/coco_det_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@ def generate_label(self):

parser = argparse.ArgumentParser()
parser.add_argument('--save_dir', default=None, type=str,
dest='save_dir', help='The directory to save the datasets.')
dest='save_dir', help='The directory to save the data.')
parser.add_argument('--ori_img_dir', default=None, type=str,
dest='ori_img_dir', help='The directory of the image datasets.')
dest='ori_img_dir', help='The directory of the image data.')
parser.add_argument('--anno_file', default=None, type=str,
dest='anno_file', help='The annotation file.')

Expand Down
10 changes: 5 additions & 5 deletions datasets/det/preprocess/fashion/fashion_det_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,15 @@ def generate_label(self):

parser = argparse.ArgumentParser()
parser.add_argument('--save_dir', default=None, type=str,
dest='save_dir', help='The directory to save the datasets.')
dest='save_dir', help='The directory to save the data.')
parser.add_argument('--ori_img_dir', default=None, type=str,
dest='ori_img_dir', help='The directory of the image datasets.')
dest='ori_img_dir', help='The directory of the image data.')
parser.add_argument('--ori_type_file', default=None, type=str,
dest='ori_type_file', help='The directory of the type datasets.')
dest='ori_type_file', help='The directory of the type data.')
parser.add_argument('--ori_bbox_file', default=None, type=str,
dest='ori_bbox_file', help='The directory of the bbox datasets.')
dest='ori_bbox_file', help='The directory of the bbox data.')
parser.add_argument('--val_interval', default=10, type=float,
dest='val_interval', help='The ratio of train & val datasets.')
dest='val_interval', help='The ratio of train & val data.')

args = parser.parse_args()

Expand Down
8 changes: 4 additions & 4 deletions datasets/det/preprocess/kitti/kitti_det_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,13 @@ def generate_label(self):

parser = argparse.ArgumentParser()
parser.add_argument('--save_dir', default=None, type=str,
dest='save_dir', help='The directory to save the datasets.')
dest='save_dir', help='The directory to save the data.')
parser.add_argument('--ori_img_dir', default=None, type=str,
dest='ori_img_dir', help='The directory of the image datasets.')
dest='ori_img_dir', help='The directory of the image data.')
parser.add_argument('--ori_label_dir', default=None, type=str,
dest='ori_label_dir', help='The directory of the label datasets.')
dest='ori_label_dir', help='The directory of the label data.')
parser.add_argument('--val_interval', default=10, type=float,
dest='val_interval', help='The ratio of train & val datasets.')
dest='val_interval', help='The ratio of train & val data.')

args = parser.parse_args()

Expand Down
Loading

0 comments on commit 524fda5

Please sign in to comment.