From 4c92dd6c759eda3038713b414ae8d4c570a6a6b4 Mon Sep 17 00:00:00 2001 From: Takuya Wakisaka Date: Mon, 17 Dec 2018 22:01:57 +0900 Subject: [PATCH 1/7] Change learning rate schdule selections --- blueoil/blueoil_init.py | 26 ++++++++++--------- blueoil/generate_lmnet_config.py | 4 +-- blueoil/templates/blueoil-config.tpl.yml | 12 ++++----- blueoil/templates/lmnet/classification.tpl.py | 12 ++++----- .../templates/lmnet/object_detection.tpl.py | 12 ++++----- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/blueoil/blueoil_init.py b/blueoil/blueoil_init.py index b28d09a7a..e3243dc14 100644 --- a/blueoil/blueoil_init.py +++ b/blueoil/blueoil_init.py @@ -97,6 +97,13 @@ # ] +learning_rate_schedule_map = { + "constant": "'constant' -> constant learning rate.", + "2-step-decay": "'2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1.", + "3-step-decay": "'3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1", + "3-step-decay-with-warmup": "'3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'", +} + def network_name_choices(task_type): if task_type == 'classification': return [definition['name'] for definition in classification_network_definitions] @@ -280,21 +287,16 @@ def ask_questions(): } initial_learning_rate_value = prompt(initial_learning_rate_value_question) - training_learning_rate_question = { + # learning rate schedule + learning_rate_schedule_question = { 'type': 'rawlist', 'name': 'value', - 'message': 'choose learning rate setting(tune1 / tune2 / tune3 / fixed):', - 'choices': ['tune1 -> "2 times decay"', 'tune2 -> "3 times decay"', 'tune3 -> "warm-up and 3 times decay"', 'fixed'], - 'default': 'tune1 -> "2 times decay"', - } - choices_key_map = { - 'tune1 -> "2 times decay"': 'tune1', - 'tune2 -> "3 times decay"': 'tune2', - 'tune3 -> "warm-up and 3 times decay"': 'tune3', - 'fixed': 'fixed', + 'message': 'choose learning rate schedule:', + 'choices': list(learning_rate_schedule_map.values()), + 'default': learning_rate_schedule_map["constant"], } - tmp_learning_rate_setting = prompt(training_learning_rate_question) - training_learning_rate_setting = choices_key_map[tmp_learning_rate_setting] + _tmp_learning_rate_schedule = prompt(learning_rate_schedule_question) + learning_rate_schedule = learning_rate_schedule_map[_tmp_learning_rate_schedule] quantize_first_convolution_question = { 'type': 'rawlist', diff --git a/blueoil/generate_lmnet_config.py b/blueoil/generate_lmnet_config.py index 1f066b8f3..023e05f91 100644 --- a/blueoil/generate_lmnet_config.py +++ b/blueoil/generate_lmnet_config.py @@ -136,7 +136,7 @@ def _blueoil_to_lmnet(blueoil_config): # trainer batch_size = blueoil_config["trainer"]["batch_size"] initial_learning_rate = blueoil_config["trainer"]["initial_learning_rate"] - learning_rate_setting = blueoil_config["trainer"]["learning_rate_setting"] + learning_rate_schedule = blueoil_config["trainer"]["learning_rate_schedule"] # common image_size = blueoil_config["common"]["image_size"] @@ -158,7 +158,7 @@ def _blueoil_to_lmnet(blueoil_config): "max_epochs": "", "max_steps": "", "initial_learning_rate": initial_learning_rate, - "learning_rate_setting": learning_rate_setting, + "learning_rate_schedule": learning_rate_schedule, "image_size": image_size, diff --git a/blueoil/templates/blueoil-config.tpl.yml b/blueoil/templates/blueoil-config.tpl.yml index 9599c94b9..619fbbb0a 100644 --- a/blueoil/templates/blueoil-config.tpl.yml +++ b/blueoil/templates/blueoil-config.tpl.yml @@ -11,12 +11,12 @@ dataset: trainer: batch_size: {{ batch_size }} epochs: {{ training_epochs }} -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: {{ training_learning_rate_setting }} + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: {{ learning_rate_schedule }} initial_learning_rate: {{ initial_learning_rate_value }} network: diff --git a/blueoil/templates/lmnet/classification.tpl.py b/blueoil/templates/lmnet/classification.tpl.py index 732495639..4e19ff094 100644 --- a/blueoil/templates/lmnet/classification.tpl.py +++ b/blueoil/templates/lmnet/classification.tpl.py @@ -82,28 +82,28 @@ NETWORK = EasyDict() NETWORK.OPTIMIZER_CLASS = tf.train.MomentumOptimizer -if '{{learning_rate_setting}}' != 'fixed': +if '{{learning_rate_setting}}' != 'constant': NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9} NETWORK.LEARNING_RATE_FUNC = tf.train.piecewise_constant -if '{{learning_rate_setting}}' == 'tune1': +if '{{learning_rate_setting}}' == '2-step-decay': NETWORK.LEARNING_RATE_KWARGS = { "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100], "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) / 2), int(step_per_epoch * (MAX_EPOCHS - 1))], } -elif '{{learning_rate_setting}}' == 'tune2': +elif '{{learning_rate_setting}}' == '3-step-decay': NETWORK.LEARNING_RATE_KWARGS = { "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], } -elif '{{learning_rate_setting}}' == 'tune3': +elif '{{learning_rate_setting}}' == '3-step-decay-with-warmup': if MAX_EPOCHS < 4: - raise ValueError("epoch number must be >= 4, when tune3 is selected.") + raise ValueError("epoch number must be >= 4, when 3-step-decay-with-warmup is selected.") NETWORK.LEARNING_RATE_KWARGS = { "values": [{{initial_learning_rate}} / 1000, {{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], "boundaries": [int(step_per_epoch * 1), int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], } -elif '{{learning_rate_setting}}' == 'fixed': +elif '{{learning_rate_setting}}' == 'constant': NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9, "learning_rate": {{initial_learning_rate}}} else: raise ValueError diff --git a/blueoil/templates/lmnet/object_detection.tpl.py b/blueoil/templates/lmnet/object_detection.tpl.py index 677d177d7..07229dfaf 100644 --- a/blueoil/templates/lmnet/object_detection.tpl.py +++ b/blueoil/templates/lmnet/object_detection.tpl.py @@ -108,27 +108,27 @@ elif '{{optimizer}}' == 'AdamOptimizer': NETWORK.OPTIMIZER_CLASS = tf.train.AdamOptimizer -if '{{learning_rate_setting}}' != 'fixed': +if '{{learning_rate_setting}}' != 'constant': NETWORK.LEARNING_RATE_FUNC = tf.train.piecewise_constant -if '{{learning_rate_setting}}' == 'tune1': +if '{{learning_rate_setting}}' == '2-step-decay': NETWORK.LEARNING_RATE_KWARGS = { "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100], "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) / 2), int(step_per_epoch * (MAX_EPOCHS - 1))], } -elif '{{learning_rate_setting}}' == 'tune2': +elif '{{learning_rate_setting}}' == '3-step-decay': NETWORK.LEARNING_RATE_KWARGS = { "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], } -elif '{{learning_rate_setting}}' == 'tune3': +elif '{{learning_rate_setting}}' == '3-step-decay-with-warmup': if MAX_EPOCHS < 4: - raise ValueError("epoch number must be >= 4, when tune3 is selected.") + raise ValueError("epoch number must be >= 4, when 3-step-decay-with-warmup is selected.") NETWORK.LEARNING_RATE_KWARGS = { "values": [{{initial_learning_rate}} / 1000, {{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], "boundaries": [int(step_per_epoch * 1), int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], } -elif '{{learning_rate_setting}}' == 'fixed': +elif '{{learning_rate_setting}}' == 'constant': NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9, "learning_rate": {{initial_learning_rate}}} else: raise ValueError From 99e4746604607ad097ecdbe182dccc90ccccec20 Mon Sep 17 00:00:00 2001 From: Takuya Wakisaka Date: Mon, 17 Dec 2018 22:06:18 +0900 Subject: [PATCH 2/7] Change learning rate schdule selections --- tests/config/caltech101_classification.yml | 13 ++++--- ...ltech101_classification_has_validation.yml | 13 ++++--- tests/config/delta_mark_classification.yml | 13 ++++--- ...lta_mark_classification_has_validation.yml | 13 ++++--- tests/config/delta_mark_object_detection.yml | 13 ++++--- ...a_mark_object_detection_has_validation.yml | 13 ++++--- tests/config/make_yml_config.py | 38 ++++++++++--------- .../config/openimagesv4_object_detection.yml | 13 ++++--- ...agesv4_object_detection_has_validation.yml | 13 ++++--- 9 files changed, 76 insertions(+), 66 deletions(-) diff --git a/tests/config/caltech101_classification.yml b/tests/config/caltech101_classification.yml index 714a3d4d3..821fa8c33 100644 --- a/tests/config/caltech101_classification.yml +++ b/tests/config/caltech101_classification.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: diff --git a/tests/config/caltech101_classification_has_validation.yml b/tests/config/caltech101_classification_has_validation.yml index b79c388a6..6462124d7 100644 --- a/tests/config/caltech101_classification_has_validation.yml +++ b/tests/config/caltech101_classification_has_validation.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: diff --git a/tests/config/delta_mark_classification.yml b/tests/config/delta_mark_classification.yml index dfd066123..cfc9ea275 100644 --- a/tests/config/delta_mark_classification.yml +++ b/tests/config/delta_mark_classification.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: diff --git a/tests/config/delta_mark_classification_has_validation.yml b/tests/config/delta_mark_classification_has_validation.yml index c2324bdca..8383df33e 100644 --- a/tests/config/delta_mark_classification_has_validation.yml +++ b/tests/config/delta_mark_classification_has_validation.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: diff --git a/tests/config/delta_mark_object_detection.yml b/tests/config/delta_mark_object_detection.yml index 32fffc191..a177f6beb 100644 --- a/tests/config/delta_mark_object_detection.yml +++ b/tests/config/delta_mark_object_detection.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: diff --git a/tests/config/delta_mark_object_detection_has_validation.yml b/tests/config/delta_mark_object_detection_has_validation.yml index 6cde8673e..14545dad3 100644 --- a/tests/config/delta_mark_object_detection_has_validation.yml +++ b/tests/config/delta_mark_object_detection_has_validation.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: diff --git a/tests/config/make_yml_config.py b/tests/config/make_yml_config.py index 006c7554d..29af38321 100644 --- a/tests/config/make_yml_config.py +++ b/tests/config/make_yml_config.py @@ -113,21 +113,23 @@ ' optimizer: MomentumOptimizer\n', ] -trainer_lr_setting_comment = "# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'.\n\ -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1.\n\ -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1.\n\ -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'.\n\ -# 'fixed' is constant learning rate.\n" - -trainer_lr_settings = [ - ' learning_rate_setting: tune1\n', - ' learning_rate_setting: tune1\n', - ' learning_rate_setting: tune1\n', - ' learning_rate_setting: tune1\n', - ' learning_rate_setting: tune1\n', - ' learning_rate_setting: tune1\n', - ' learning_rate_setting: tune1\n', - ' learning_rate_setting: tune1\n', +trainer_lr_schedule_comment = """ + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' +""" + +trainer_lr_schedules = [ + ' learning_rate_schedule: constant\n', + ' learning_rate_schedule: constant\n', + ' learning_rate_schedule: constant\n', + ' learning_rate_schedule: constant\n', + ' learning_rate_schedule: constant\n', + ' learning_rate_schedule: constant\n', + ' learning_rate_schedule: constant\n', + ' learning_rate_schedule: constant\n', ] trainer_initial_lrs = [ @@ -236,9 +238,9 @@ def learning_settings_to_yaml(index): # trainer optimizer #fp.write(str(trainer_optimizers[index])) # trainer lr setting comment - fp.write(str(trainer_lr_setting_comment)) - # trainer lr setting - fp.write(str(trainer_lr_settings[index])) + fp.write(str(trainer_lr_schedule_comment)) + # trainer lr schedule + fp.write(str(trainer_lr_schedules[index])) # trainer initial lr fp.write(str(trainer_initial_lrs[index])) fp.write('\n') diff --git a/tests/config/openimagesv4_object_detection.yml b/tests/config/openimagesv4_object_detection.yml index 7865028ed..b08450b64 100644 --- a/tests/config/openimagesv4_object_detection.yml +++ b/tests/config/openimagesv4_object_detection.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: diff --git a/tests/config/openimagesv4_object_detection_has_validation.yml b/tests/config/openimagesv4_object_detection_has_validation.yml index ea92d1ca1..0848603aa 100644 --- a/tests/config/openimagesv4_object_detection_has_validation.yml +++ b/tests/config/openimagesv4_object_detection_has_validation.yml @@ -11,12 +11,13 @@ dataset: trainer: batch_size: 1 epochs: 1 -# supported 'learning_rate_setting' is 'tune1', 'tune2', 'tune3', 'fixed'. -# 'tune1' is 2 times decay, learning rate reduce to 1/10 on epoch/2 and epoch-1. -# 'tune2' is 3 times decay, learning rate reduce to 1/10 on epoch/3 and epoch*2/3 and epoch-1. -# 'tune3' is warmup and 3 times decay, warmup learning rate 1/1000 in 1 epoch, then train same as 'tune2'. -# 'fixed' is constant learning rate. - learning_rate_setting: tune1 + + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # 'constant' -> constant learning rate. + # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + learning_rate_schedule: constant initial_learning_rate: 0.001 network: From 8252695896406c66b40292cf15dfe7a8805299e6 Mon Sep 17 00:00:00 2001 From: Takuya Wakisaka Date: Tue, 18 Dec 2018 22:20:11 +0900 Subject: [PATCH 3/7] handle learning rate func, kwargs in generate_lmnet_config --- blueoil/blueoil_init.py | 4 +- blueoil/generate_lmnet_config.py | 85 ++++++++++++++++--- blueoil/templates/lmnet/classification.tpl.py | 38 +-------- .../templates/lmnet/object_detection.tpl.py | 46 ++-------- tests/config/caltech101_classification.yml | 1 - ...ltech101_classification_has_validation.yml | 1 - tests/config/delta_mark_classification.yml | 1 - ...lta_mark_classification_has_validation.yml | 1 - tests/config/delta_mark_object_detection.yml | 1 - ...a_mark_object_detection_has_validation.yml | 1 - tests/config/make_yml_config.py | 4 +- .../config/openimagesv4_object_detection.yml | 1 - ...agesv4_object_detection_has_validation.yml | 1 - 13 files changed, 89 insertions(+), 96 deletions(-) diff --git a/blueoil/blueoil_init.py b/blueoil/blueoil_init.py index bfedef0b5..83895483d 100644 --- a/blueoil/blueoil_init.py +++ b/blueoil/blueoil_init.py @@ -307,7 +307,9 @@ def ask_questions(): 'default': learning_rate_schedule_map["constant"], } _tmp_learning_rate_schedule = prompt(learning_rate_schedule_question) - learning_rate_schedule = learning_rate_schedule_map[_tmp_learning_rate_schedule] + for key, value in learning_rate_schedule_map.items(): + if value == _tmp_learning_rate_schedule: + learning_rate_schedule = key if prompt(enable_data_augmentation): all_augmentor = {} diff --git a/blueoil/generate_lmnet_config.py b/blueoil/generate_lmnet_config.py index 9a45cd976..a9d9d4d89 100644 --- a/blueoil/generate_lmnet_config.py +++ b/blueoil/generate_lmnet_config.py @@ -16,10 +16,14 @@ import argparse import os import re +import importlib import yaml from jinja2 import Environment, FileSystemLoader +from lmnet.utils.module_loader import load_class + + # TODO(wakisaka): objecte detection, segmentation _TASK_TYPE_TEMPLATE_FILE = { "classification": "classification.tpl.py", @@ -132,10 +136,76 @@ def _blueoil_to_lmnet(blueoil_config): else: dataset_class_property = {"extend_dir": dataset_class_extend_dir} + # load dataset python module from string. + _loaded_dataset_module = importlib.import_module("lmnet.datasets.{}".format(dataset_module)) + # load dataset python module from string. + _loaded_dataset_class = load_class(_loaded_dataset_module, dataset_class) + _dataset_class = type('DATASET_CLASS', (_loaded_dataset_class,), dataset_class_property) + _dataset_obj = _dataset_class(subset="train", batch_size=1) + classes = _dataset_obj.classes + # trainer batch_size = blueoil_config["trainer"]["batch_size"] initial_learning_rate = blueoil_config["trainer"]["initial_learning_rate"] learning_rate_schedule = blueoil_config["trainer"]["learning_rate_schedule"] + max_epochs = blueoil_config["trainer"]["epochs"] + + step_per_epoch = float(_dataset_obj.num_per_epoch)/batch_size + + learning_rate_func = None + learning_rate_kwargs = None + if learning_rate_schedule == "constant": + optimizer_kwargs = {"momentum": 0.9, "learning_rate": initial_learning_rate} + else: + optimizer_kwargs = {"momentum": 0.9} + learning_rate_func = "tf.train.piecewise_constant" + + # import ipdb; ipdb.set_trace() + if learning_rate_schedule == "2-step-decay": + learning_rate_kwargs = { + "values": [ + initial_learning_rate, + initial_learning_rate / 10, + initial_learning_rate / 100 + ], + "boundaries": [ + int((step_per_epoch * (max_epochs - 1)) / 2), + int(step_per_epoch * (max_epochs - 1)) + ], + } + + elif learning_rate_schedule == "3-step-decay": + learning_rate_kwargs = { + "values": [ + initial_learning_rate, + initial_learning_rate / 10, + initial_learning_rate / 100, + initial_learning_rate / 1000 + ], + "boundaries": [ + int((step_per_epoch * (max_epochs - 1)) * 1 / 3), + int((step_per_epoch * (max_epochs - 1)) * 2 / 3), + int(step_per_epoch * (max_epochs - 1)) + ], + } + + elif learning_rate_schedule == "3-step-decay-with-warmup": + if max_epochs < 4: + raise ValueError("epoch number must be >= 4, when 3-step-decay-with-warmup is selected.") + learning_rate_kwargs = { + "values": [ + initial_learning_rate / 1000, + initial_learning_rate, + initial_learning_rate / 10, + initial_learning_rate / 100, + initial_learning_rate / 1000 + ], + "boundaries": [ + int((step_per_epoch * (max_epochs - 1)) * 1 / 3), + int((step_per_epoch * (max_epochs - 1)) * 2 / 3), + int(step_per_epoch * (max_epochs - 1)) + ], + } # common image_size = blueoil_config["common"]["image_size"] @@ -165,12 +235,13 @@ def _blueoil_to_lmnet(blueoil_config): "dataset_class_property": dataset_class_property, "batch_size": batch_size, - "max_epochs": "", - "max_steps": "", - "initial_learning_rate": initial_learning_rate, - "learning_rate_schedule": learning_rate_schedule, + "max_epochs": max_epochs, + "optimizer_kwargs": optimizer_kwargs, + "learning_rate_func": learning_rate_func, + "learning_rate_kwargs": learning_rate_kwargs, "image_size": image_size, + "classes": classes, "quantize_first_convolution": quantize_first_convolution, @@ -178,12 +249,6 @@ def _blueoil_to_lmnet(blueoil_config): "data_augmentation": data_augmentation } - # max_epochs or max_steps - if "steps" in blueoil_config["trainer"].keys(): - config["max_steps"] = blueoil_config["trainer"]["steps"] - elif "epochs" in blueoil_config["trainer"].keys(): - config["max_epochs"] = blueoil_config["trainer"]["epochs"] - # merge dict lmnet_config = default_lmnet_config.copy() lmnet_config.update(config) diff --git a/blueoil/templates/lmnet/classification.tpl.py b/blueoil/templates/lmnet/classification.tpl.py index d346c0974..6ac8dcfc5 100644 --- a/blueoil/templates/lmnet/classification.tpl.py +++ b/blueoil/templates/lmnet/classification.tpl.py @@ -44,16 +44,9 @@ BATCH_SIZE = {{batch_size}} DATA_FORMAT = "NHWC" TASK = Tasks.CLASSIFICATION -# In order to get instance property `classes`, instantiate DATASET_CLASS. -dataset_obj = DATASET_CLASS(subset="train", batch_size=1) -CLASSES = dataset_obj.classes -step_per_epoch = float(dataset_obj.num_per_epoch)/BATCH_SIZE +CLASSES = {{classes}} -{% if max_epochs -%} MAX_EPOCHS = {{max_epochs}} -{%- elif max_steps -%} -MAX_STEPS = {{max_steps}} -{%- endif %} SAVE_STEPS = {{save_steps}} TEST_STEPS = {{test_steps}} SUMMARISE_STEPS = {{summarise_steps}} @@ -75,32 +68,9 @@ NETWORK = EasyDict() NETWORK.OPTIMIZER_CLASS = tf.train.MomentumOptimizer - -if '{{learning_rate_setting}}' != 'constant': - NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9} - NETWORK.LEARNING_RATE_FUNC = tf.train.piecewise_constant - -if '{{learning_rate_setting}}' == '2-step-decay': - NETWORK.LEARNING_RATE_KWARGS = { - "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100], - "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) / 2), int(step_per_epoch * (MAX_EPOCHS - 1))], - } -elif '{{learning_rate_setting}}' == '3-step-decay': - NETWORK.LEARNING_RATE_KWARGS = { - "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], - "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], - } -elif '{{learning_rate_setting}}' == '3-step-decay-with-warmup': - if MAX_EPOCHS < 4: - raise ValueError("epoch number must be >= 4, when 3-step-decay-with-warmup is selected.") - NETWORK.LEARNING_RATE_KWARGS = { - "values": [{{initial_learning_rate}} / 1000, {{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], - "boundaries": [int(step_per_epoch * 1), int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], - } -elif '{{learning_rate_setting}}' == 'constant': - NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9, "learning_rate": {{initial_learning_rate}}} -else: - raise ValueError +NETWORK.OPTIMIZER_KWARGS = {{optimizer_kwargs}} +NETWORK.LEARNING_RATE_FUNC = {{learning_rate_func}} +NETWORK.LEARNING_RATE_KWARGS = {{learning_rate_kwargs}} NETWORK.IMAGE_SIZE = IMAGE_SIZE NETWORK.BATCH_SIZE = BATCH_SIZE diff --git a/blueoil/templates/lmnet/object_detection.tpl.py b/blueoil/templates/lmnet/object_detection.tpl.py index 6eba2ee2d..91332216e 100644 --- a/blueoil/templates/lmnet/object_detection.tpl.py +++ b/blueoil/templates/lmnet/object_detection.tpl.py @@ -49,16 +49,9 @@ BATCH_SIZE = {{batch_size}} DATA_FORMAT = "NHWC" TASK = Tasks.OBJECT_DETECTION -# In order to get instance property `classes`, instantiate DATASET_CLASS. -dataset_obj = DATASET_CLASS(subset="train", batch_size=1) -CLASSES = dataset_obj.classes -step_per_epoch = float(dataset_obj.num_per_epoch)/BATCH_SIZE +CLASSES = {{classes}} -{% if max_epochs -%} MAX_EPOCHS = {{max_epochs}} -{%- elif max_steps -%} -MAX_STEPS = {{max_steps}} -{%- endif %} SAVE_STEPS = {{save_steps}} TEST_STEPS = {{test_steps}} SUMMARISE_STEPS = {{summarise_steps}} @@ -94,39 +87,10 @@ ]) NETWORK = EasyDict() - -if '{{optimizer}}' == 'GradientDescentOptimizer': - NETWORK.OPTIMIZER_CLASS = tf.train.GradientDescentOptimizer -elif '{{optimizer}}' == 'MomentumOptimizer': - NETWORK.OPTIMIZER_CLASS = tf.train.MomentumOptimizer - NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9} -elif '{{optimizer}}' == 'AdamOptimizer': - NETWORK.OPTIMIZER_CLASS = tf.train.AdamOptimizer - -if '{{learning_rate_setting}}' != 'constant': - NETWORK.LEARNING_RATE_FUNC = tf.train.piecewise_constant - -if '{{learning_rate_setting}}' == '2-step-decay': - NETWORK.LEARNING_RATE_KWARGS = { - "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100], - "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) / 2), int(step_per_epoch * (MAX_EPOCHS - 1))], - } -elif '{{learning_rate_setting}}' == '3-step-decay': - NETWORK.LEARNING_RATE_KWARGS = { - "values": [{{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], - "boundaries": [int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], - } -elif '{{learning_rate_setting}}' == '3-step-decay-with-warmup': - if MAX_EPOCHS < 4: - raise ValueError("epoch number must be >= 4, when 3-step-decay-with-warmup is selected.") - NETWORK.LEARNING_RATE_KWARGS = { - "values": [{{initial_learning_rate}} / 1000, {{initial_learning_rate}}, {{initial_learning_rate}} / 10, {{initial_learning_rate}} / 100, {{initial_learning_rate}} / 1000], - "boundaries": [int(step_per_epoch * 1), int((step_per_epoch * (MAX_EPOCHS - 1)) * 1 / 3), int((step_per_epoch * (MAX_EPOCHS - 1)) * 2 / 3), int(step_per_epoch * (MAX_EPOCHS - 1))], - } -elif '{{learning_rate_setting}}' == 'constant': - NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9, "learning_rate": {{initial_learning_rate}}} -else: - raise ValueError +NETWORK.OPTIMIZER_CLASS = tf.train.MomentumOptimizer +NETWORK.OPTIMIZER_KWARGS = {{optimizer_kwargs}} +NETWORK.LEARNING_RATE_FUNC = {{learning_rate_func}} +NETWORK.LEARNING_RATE_KWARGS = {{learning_rate_kwargs}} NETWORK.IMAGE_SIZE = IMAGE_SIZE NETWORK.BATCH_SIZE = BATCH_SIZE diff --git a/tests/config/caltech101_classification.yml b/tests/config/caltech101_classification.yml index a0a7cae53..0972528a9 100644 --- a/tests/config/caltech101_classification.yml +++ b/tests/config/caltech101_classification.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. diff --git a/tests/config/caltech101_classification_has_validation.yml b/tests/config/caltech101_classification_has_validation.yml index a5c71bf92..bb4df2c45 100644 --- a/tests/config/caltech101_classification_has_validation.yml +++ b/tests/config/caltech101_classification_has_validation.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. diff --git a/tests/config/delta_mark_classification.yml b/tests/config/delta_mark_classification.yml index 8c6930119..23e187bcb 100644 --- a/tests/config/delta_mark_classification.yml +++ b/tests/config/delta_mark_classification.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. diff --git a/tests/config/delta_mark_classification_has_validation.yml b/tests/config/delta_mark_classification_has_validation.yml index 5f69b13dc..f15d4c770 100644 --- a/tests/config/delta_mark_classification_has_validation.yml +++ b/tests/config/delta_mark_classification_has_validation.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. diff --git a/tests/config/delta_mark_object_detection.yml b/tests/config/delta_mark_object_detection.yml index 5e093dd47..aaf7afae8 100644 --- a/tests/config/delta_mark_object_detection.yml +++ b/tests/config/delta_mark_object_detection.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. diff --git a/tests/config/delta_mark_object_detection_has_validation.yml b/tests/config/delta_mark_object_detection_has_validation.yml index 7d29a2f0f..8a25d165c 100644 --- a/tests/config/delta_mark_object_detection_has_validation.yml +++ b/tests/config/delta_mark_object_detection_has_validation.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. diff --git a/tests/config/make_yml_config.py b/tests/config/make_yml_config.py index 276fa0ac9..6ff209f87 100644 --- a/tests/config/make_yml_config.py +++ b/tests/config/make_yml_config.py @@ -113,7 +113,7 @@ ' optimizer: MomentumOptimizer\n', ] -trainer_lr_schedule_comment = """ +trainer_lr_schedule_comment = """\ # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. @@ -244,7 +244,7 @@ def learning_settings_to_yaml(index): #fp.write(str(trainer_optimizer_comment)) # trainer optimizer #fp.write(str(trainer_optimizers[index])) - # trainer lr setting comment + # trainer lr schedule comment fp.write(str(trainer_lr_schedule_comment)) # trainer lr schedule fp.write(str(trainer_lr_schedules[index])) diff --git a/tests/config/openimagesv4_object_detection.yml b/tests/config/openimagesv4_object_detection.yml index 11b433fab..9a5eff356 100644 --- a/tests/config/openimagesv4_object_detection.yml +++ b/tests/config/openimagesv4_object_detection.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. diff --git a/tests/config/openimagesv4_object_detection_has_validation.yml b/tests/config/openimagesv4_object_detection_has_validation.yml index db5f9eba4..b247d1283 100644 --- a/tests/config/openimagesv4_object_detection_has_validation.yml +++ b/tests/config/openimagesv4_object_detection_has_validation.yml @@ -11,7 +11,6 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. From 91f4e1d9ea1a5f2ca117c133a03d07f1e693c9e8 Mon Sep 17 00:00:00 2001 From: Takuya Wakisaka Date: Wed, 26 Dec 2018 17:37:05 +0900 Subject: [PATCH 4/7] fix 3-step-decay-warmup --- blueoil/blueoil_init.py | 6 ++++-- blueoil/generate_lmnet_config.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/blueoil/blueoil_init.py b/blueoil/blueoil_init.py index 83895483d..f42028c93 100644 --- a/blueoil/blueoil_init.py +++ b/blueoil/blueoil_init.py @@ -15,6 +15,7 @@ # ============================================================================= import inspect import re +from collections import OrderedDict import whaaaaat from jinja2 import Environment, FileSystemLoader @@ -101,12 +102,13 @@ # ] -learning_rate_schedule_map = { +learning_rate_schedule_map = OrderedDict({ "constant": "'constant' -> constant learning rate.", "2-step-decay": "'2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1.", "3-step-decay": "'3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1", "3-step-decay-with-warmup": "'3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'", -} +}) + def network_name_choices(task_type): if task_type == 'classification': diff --git a/blueoil/generate_lmnet_config.py b/blueoil/generate_lmnet_config.py index 17773a7c8..18d054a2e 100644 --- a/blueoil/generate_lmnet_config.py +++ b/blueoil/generate_lmnet_config.py @@ -160,7 +160,6 @@ def _blueoil_to_lmnet(blueoil_config): optimizer_kwargs = {"momentum": 0.9} learning_rate_func = "tf.train.piecewise_constant" - # import ipdb; ipdb.set_trace() if learning_rate_schedule == "2-step-decay": learning_rate_kwargs = { "values": [ @@ -201,6 +200,7 @@ def _blueoil_to_lmnet(blueoil_config): initial_learning_rate / 1000 ], "boundaries": [ + int(step_per_epoch * 1), int((step_per_epoch * (max_epochs - 1)) * 1 / 3), int((step_per_epoch * (max_epochs - 1)) * 2 / 3), int(step_per_epoch * (max_epochs - 1)) From e3a5675d50dd3beb1c10d7c18f91f7bcb47b42cb Mon Sep 17 00:00:00 2001 From: Takuya Wakisaka Date: Wed, 26 Dec 2018 17:38:29 +0900 Subject: [PATCH 5/7] update example configuration --- docs/usage/init.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/init.md b/docs/usage/init.md index f70e79e94..3ebf54eff 100644 --- a/docs/usage/init.md +++ b/docs/usage/init.md @@ -22,7 +22,7 @@ This is an example of configuration. image size (integer x integer): 32x32 how many epochs do you run training (integer): 100 initial learning rate: 0.001 - message': 'choose learning rate setting(tune1 / tune2 / tune3 / fixed): tune1 -> "2 times decay" + choose learning rate schedule: '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. enable data augmentation? No apply quantization at the first layer: yes ``` From ea8f63aa5b530401a43f9002c4ffa0fc7ee9a4d4 Mon Sep 17 00:00:00 2001 From: Takuya Wakisaka Date: Wed, 26 Dec 2018 22:34:14 +0900 Subject: [PATCH 6/7] Change learning rate schdule text more readable --- blueoil/blueoil_init.py | 15 ++++++++------- blueoil/templates/blueoil-config.tpl.yml | 8 ++++---- tests/config/caltech101_classification.yml | 8 ++++---- .../caltech101_classification_has_validation.yml | 8 ++++---- tests/config/delta_mark_classification.yml | 8 ++++---- .../delta_mark_classification_has_validation.yml | 8 ++++---- tests/config/delta_mark_object_detection.yml | 8 ++++---- ...delta_mark_object_detection_has_validation.yml | 8 ++++---- tests/config/make_yml_config.py | 9 +++++---- tests/config/openimagesv4_object_detection.yml | 8 ++++---- ...enimagesv4_object_detection_has_validation.yml | 8 ++++---- 11 files changed, 49 insertions(+), 47 deletions(-) diff --git a/blueoil/blueoil_init.py b/blueoil/blueoil_init.py index f42028c93..514c22fb8 100644 --- a/blueoil/blueoil_init.py +++ b/blueoil/blueoil_init.py @@ -102,12 +102,12 @@ # ] -learning_rate_schedule_map = OrderedDict({ - "constant": "'constant' -> constant learning rate.", - "2-step-decay": "'2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1.", - "3-step-decay": "'3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1", - "3-step-decay-with-warmup": "'3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'", -}) +learning_rate_schedule_map = OrderedDict([ + ("constant", "'constant' -> constant learning rate."), + ("2-step-decay", "'2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1."), + ("3-step-decay", "'3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1"), + ("3-step-decay-with-warmup", "'3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'"), +]) def network_name_choices(task_type): @@ -304,7 +304,8 @@ def ask_questions(): learning_rate_schedule_question = { 'type': 'rawlist', 'name': 'value', - 'message': 'choose learning rate schedule:', + 'message': 'choose learning rate schedule \ +({epochs} is the number of training epochs you entered before):', 'choices': list(learning_rate_schedule_map.values()), 'default': learning_rate_schedule_map["constant"], } diff --git a/blueoil/templates/blueoil-config.tpl.yml b/blueoil/templates/blueoil-config.tpl.yml index cdef99cec..d45a12b01 100644 --- a/blueoil/templates/blueoil-config.tpl.yml +++ b/blueoil/templates/blueoil-config.tpl.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: {{ batch_size }} epochs: {{ training_epochs }} - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: {{ learning_rate_schedule }} initial_learning_rate: {{ initial_learning_rate_value }} diff --git a/tests/config/caltech101_classification.yml b/tests/config/caltech101_classification.yml index 0972528a9..df4424cb2 100644 --- a/tests/config/caltech101_classification.yml +++ b/tests/config/caltech101_classification.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/caltech101_classification_has_validation.yml b/tests/config/caltech101_classification_has_validation.yml index bb4df2c45..aa66dc765 100644 --- a/tests/config/caltech101_classification_has_validation.yml +++ b/tests/config/caltech101_classification_has_validation.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_classification.yml b/tests/config/delta_mark_classification.yml index 23e187bcb..d2ec9bc97 100644 --- a/tests/config/delta_mark_classification.yml +++ b/tests/config/delta_mark_classification.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_classification_has_validation.yml b/tests/config/delta_mark_classification_has_validation.yml index f15d4c770..f6a89355b 100644 --- a/tests/config/delta_mark_classification_has_validation.yml +++ b/tests/config/delta_mark_classification_has_validation.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_object_detection.yml b/tests/config/delta_mark_object_detection.yml index aaf7afae8..495d4110c 100644 --- a/tests/config/delta_mark_object_detection.yml +++ b/tests/config/delta_mark_object_detection.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_object_detection_has_validation.yml b/tests/config/delta_mark_object_detection_has_validation.yml index 8a25d165c..a345135d4 100644 --- a/tests/config/delta_mark_object_detection_has_validation.yml +++ b/tests/config/delta_mark_object_detection_has_validation.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/make_yml_config.py b/tests/config/make_yml_config.py index 6ff209f87..1ad065c2e 100644 --- a/tests/config/make_yml_config.py +++ b/tests/config/make_yml_config.py @@ -114,11 +114,12 @@ ] trainer_lr_schedule_comment = """\ - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' \ +({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. """ trainer_lr_schedules = [ diff --git a/tests/config/openimagesv4_object_detection.yml b/tests/config/openimagesv4_object_detection.yml index 9a5eff356..cfa1a9bdd 100644 --- a/tests/config/openimagesv4_object_detection.yml +++ b/tests/config/openimagesv4_object_detection.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/openimagesv4_object_detection_has_validation.yml b/tests/config/openimagesv4_object_detection_has_validation.yml index b247d1283..cea40b0ea 100644 --- a/tests/config/openimagesv4_object_detection_has_validation.yml +++ b/tests/config/openimagesv4_object_detection_has_validation.yml @@ -11,11 +11,11 @@ dataset: trainer: batch_size: 1 epochs: 1 - # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup'. + # supported 'learning_rate_schedule' is 'constant', '2-step-decay', '3-step-decay', '3-step-decay-with-warmup' ({epochs} is the number of training epochs you entered before). # 'constant' -> constant learning rate. - # '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. - # '3-step-decay' -> learning rate reduce to 1/10 on epochs/3 and epochs*2/3 and epochs-1 - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay' + # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. + # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 From 1b82b4419a82cd5b47ecd35f840f8b1c1441d67b Mon Sep 17 00:00:00 2001 From: Takuya Wakisaka Date: Thu, 27 Dec 2018 14:17:43 +0900 Subject: [PATCH 7/7] Change learning rate schdule messages --- blueoil/blueoil_init.py | 2 +- blueoil/templates/blueoil-config.tpl.yml | 2 +- docs/usage/init.md | 2 +- tests/config/caltech101_classification.yml | 2 +- tests/config/caltech101_classification_has_validation.yml | 2 +- tests/config/delta_mark_classification.yml | 2 +- tests/config/delta_mark_classification_has_validation.yml | 2 +- tests/config/delta_mark_object_detection.yml | 2 +- tests/config/delta_mark_object_detection_has_validation.yml | 2 +- tests/config/make_yml_config.py | 3 ++- tests/config/openimagesv4_object_detection.yml | 2 +- tests/config/openimagesv4_object_detection_has_validation.yml | 2 +- 12 files changed, 13 insertions(+), 12 deletions(-) diff --git a/blueoil/blueoil_init.py b/blueoil/blueoil_init.py index 514c22fb8..018dca1d5 100644 --- a/blueoil/blueoil_init.py +++ b/blueoil/blueoil_init.py @@ -106,7 +106,7 @@ ("constant", "'constant' -> constant learning rate."), ("2-step-decay", "'2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1."), ("3-step-decay", "'3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1"), - ("3-step-decay-with-warmup", "'3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'"), + ("3-step-decay-with-warmup", "'3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'"), ]) diff --git a/blueoil/templates/blueoil-config.tpl.yml b/blueoil/templates/blueoil-config.tpl.yml index d45a12b01..fda0be17e 100644 --- a/blueoil/templates/blueoil-config.tpl.yml +++ b/blueoil/templates/blueoil-config.tpl.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: {{ learning_rate_schedule }} initial_learning_rate: {{ initial_learning_rate_value }} diff --git a/docs/usage/init.md b/docs/usage/init.md index 3ebf54eff..9c33361f8 100644 --- a/docs/usage/init.md +++ b/docs/usage/init.md @@ -22,7 +22,7 @@ This is an example of configuration. image size (integer x integer): 32x32 how many epochs do you run training (integer): 100 initial learning rate: 0.001 - choose learning rate schedule: '2-step-decay' -> learning rate reduce to 1/10 on epochs/2 and epochs-1. + choose learning rate schedule ({epochs} is the number of training epochs you entered before): '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. enable data augmentation? No apply quantization at the first layer: yes ``` diff --git a/tests/config/caltech101_classification.yml b/tests/config/caltech101_classification.yml index df4424cb2..e699921f2 100644 --- a/tests/config/caltech101_classification.yml +++ b/tests/config/caltech101_classification.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/caltech101_classification_has_validation.yml b/tests/config/caltech101_classification_has_validation.yml index aa66dc765..d51f14014 100644 --- a/tests/config/caltech101_classification_has_validation.yml +++ b/tests/config/caltech101_classification_has_validation.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_classification.yml b/tests/config/delta_mark_classification.yml index d2ec9bc97..388f4122b 100644 --- a/tests/config/delta_mark_classification.yml +++ b/tests/config/delta_mark_classification.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_classification_has_validation.yml b/tests/config/delta_mark_classification_has_validation.yml index f6a89355b..f03e9f59c 100644 --- a/tests/config/delta_mark_classification_has_validation.yml +++ b/tests/config/delta_mark_classification_has_validation.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_object_detection.yml b/tests/config/delta_mark_object_detection.yml index 495d4110c..5e189d7e8 100644 --- a/tests/config/delta_mark_object_detection.yml +++ b/tests/config/delta_mark_object_detection.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/delta_mark_object_detection_has_validation.yml b/tests/config/delta_mark_object_detection_has_validation.yml index a345135d4..af36531b8 100644 --- a/tests/config/delta_mark_object_detection_has_validation.yml +++ b/tests/config/delta_mark_object_detection_has_validation.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/make_yml_config.py b/tests/config/make_yml_config.py index 1ad065c2e..b7b11cada 100644 --- a/tests/config/make_yml_config.py +++ b/tests/config/make_yml_config.py @@ -119,7 +119,8 @@ # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, \ +then train the same way as '3-step-decay'. """ trainer_lr_schedules = [ diff --git a/tests/config/openimagesv4_object_detection.yml b/tests/config/openimagesv4_object_detection.yml index cfa1a9bdd..1c8493978 100644 --- a/tests/config/openimagesv4_object_detection.yml +++ b/tests/config/openimagesv4_object_detection.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001 diff --git a/tests/config/openimagesv4_object_detection_has_validation.yml b/tests/config/openimagesv4_object_detection_has_validation.yml index cea40b0ea..3f68329d2 100644 --- a/tests/config/openimagesv4_object_detection_has_validation.yml +++ b/tests/config/openimagesv4_object_detection_has_validation.yml @@ -15,7 +15,7 @@ trainer: # 'constant' -> constant learning rate. # '2-step-decay' -> learning rate decrease by 1/10 on {epochs}/2 and {epochs}-1. # '3-step-decay' -> learning rate decrease by 1/10 on {epochs}/3 and {epochs}*2/3 and {epochs}-1. - # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train same as '3-step-decay'. + # '3-step-decay-with-warmup' -> warmup learning rate 1/1000 in first epoch, then train the same way as '3-step-decay'. learning_rate_schedule: constant initial_learning_rate: 0.001