From 6b3c80ec48df315c0bac1b415ebd6bffde98e9bc Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:13:00 -0400 Subject: [PATCH 01/11] add val step arg to metrics --- pytorch_lightning/trainer/evaluation_loop.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index a6451d361033e..d871e5e7576e0 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -379,7 +379,7 @@ def _evaluate( dl_outputs.append(output) - self.__eval_add_step_metrics(output) + self.__eval_add_step_metrics(output, batch_idx) # track debug metrics self.dev_debugger.track_eval_loss_history(test_mode, batch_idx, dataloader_idx, output) @@ -505,14 +505,14 @@ def __gather_epoch_end_eval_results(self, outputs): eval_results = eval_results[0] return eval_results - def __eval_add_step_metrics(self, output): + def __eval_add_step_metrics(self, output, batch_idx): # track step level metrics if isinstance(output, EvalResult) and not self.running_sanity_check: step_log_metrics = output.batch_log_metrics step_pbar_metrics = output.batch_pbar_metrics if len(step_log_metrics) > 0: - self.log_metrics(step_log_metrics, {}) + self.log_metrics(step_log_metrics, {}, step=batch_idx) if len(step_pbar_metrics) > 0: self.add_progress_bar_metrics(step_pbar_metrics) From 6dbfa8c607038dc81bfe9ad0cdbf145cb33f01cc Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:20:08 -0400 Subject: [PATCH 02/11] add val step arg to metrics --- pytorch_lightning/trainer/logging.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/logging.py b/pytorch_lightning/trainer/logging.py index c90ba59abf735..f84b0711142ca 100644 --- a/pytorch_lightning/trainer/logging.py +++ b/pytorch_lightning/trainer/logging.py @@ -64,10 +64,12 @@ def log_metrics(self, metrics, grad_norm_dic, step=None): if "step" in scalar_metrics and step is None: step = scalar_metrics.pop("step") - else: + + elif step is None: # added metrics by Lightning for convenience scalar_metrics['epoch'] = self.current_epoch step = step if step is not None else self.global_step + # log actual metrics if self.is_global_zero and self.logger is not None: self.logger.agg_and_log_metrics(scalar_metrics, step=step) From 89a932dc21973838d0eedfb814555dad20ad7f7a Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:23:41 -0400 Subject: [PATCH 03/11] add val step arg to metrics --- pytorch_lightning/trainer/evaluation_loop.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index d871e5e7576e0..7c5309c3ccb98 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -512,6 +512,7 @@ def __eval_add_step_metrics(self, output, batch_idx): step_pbar_metrics = output.batch_pbar_metrics if len(step_log_metrics) > 0: + import pdb; pdb.set_trace() self.log_metrics(step_log_metrics, {}, step=batch_idx) if len(step_pbar_metrics) > 0: From c5db554ebc651e35ac778480b56f36b1645ca5c7 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:25:38 -0400 Subject: [PATCH 04/11] add val step arg to metrics --- pytorch_lightning/trainer/evaluation_loop.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index 7c5309c3ccb98..3620a5cf4d137 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -512,8 +512,12 @@ def __eval_add_step_metrics(self, output, batch_idx): step_pbar_metrics = output.batch_pbar_metrics if len(step_log_metrics) > 0: - import pdb; pdb.set_trace() - self.log_metrics(step_log_metrics, {}, step=batch_idx) + # make the metrics appear as a different line in the same graph + metrics_by_epoch = {} + for k, v in step_log_metrics: + metrics_by_epoch[f'{self.current_epoch}/{k}'] = v + + self.log_metrics(metrics_by_epoch, {}, step=batch_idx) if len(step_pbar_metrics) > 0: self.add_progress_bar_metrics(step_pbar_metrics) From daed89450f19527ca8edd0929739f889de702eda Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:26:19 -0400 Subject: [PATCH 05/11] add val step arg to metrics --- pytorch_lightning/trainer/evaluation_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index 3620a5cf4d137..dc8580e19efd7 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -514,7 +514,7 @@ def __eval_add_step_metrics(self, output, batch_idx): if len(step_log_metrics) > 0: # make the metrics appear as a different line in the same graph metrics_by_epoch = {} - for k, v in step_log_metrics: + for k, v in step_log_metrics.items(): metrics_by_epoch[f'{self.current_epoch}/{k}'] = v self.log_metrics(metrics_by_epoch, {}, step=batch_idx) From ee994922b81c51031b02d87ea434585a9f2a9082 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:26:51 -0400 Subject: [PATCH 06/11] add val step arg to metrics --- pytorch_lightning/trainer/evaluation_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index dc8580e19efd7..55cf5ea76f628 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -515,7 +515,7 @@ def __eval_add_step_metrics(self, output, batch_idx): # make the metrics appear as a different line in the same graph metrics_by_epoch = {} for k, v in step_log_metrics.items(): - metrics_by_epoch[f'{self.current_epoch}/{k}'] = v + metrics_by_epoch[f'epoch_{self.current_epoch}/{k}'] = v self.log_metrics(metrics_by_epoch, {}, step=batch_idx) From e874298d70797b37983c1aec38a60546a7d4c6cb Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:27:15 -0400 Subject: [PATCH 07/11] add val step arg to metrics --- pytorch_lightning/trainer/evaluation_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index 55cf5ea76f628..68e44456fe998 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -217,7 +217,7 @@ def add_progress_bar_metrics(self, *args): """Warning: this is just empty shell for code implemented in other class.""" @abstractmethod - def log_metrics(self, *args): + def log_metrics(self, *args, **kwargs): """Warning: this is just empty shell for code implemented in other class.""" @abstractmethod From 533570ef9c8314655731302178bda1e8b474cefe Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:31:24 -0400 Subject: [PATCH 08/11] add val step arg to metrics --- pytorch_lightning/trainer/evaluation_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index 68e44456fe998..3b07b81dae2c1 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -515,7 +515,7 @@ def __eval_add_step_metrics(self, output, batch_idx): # make the metrics appear as a different line in the same graph metrics_by_epoch = {} for k, v in step_log_metrics.items(): - metrics_by_epoch[f'epoch_{self.current_epoch}/{k}'] = v + metrics_by_epoch[f'{k}/epoch_{self.current_epoch}'] = v self.log_metrics(metrics_by_epoch, {}, step=batch_idx) From 93d67b43eb36b5d9a0434dce267faafbb560899b Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 07:56:44 -0400 Subject: [PATCH 09/11] add val step arg to metrics --- .../trainer/test_validation_steps_result_return.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/trainer/test_validation_steps_result_return.py b/tests/trainer/test_validation_steps_result_return.py index 8162f57287e71..02773f5fbd8e8 100644 --- a/tests/trainer/test_validation_steps_result_return.py +++ b/tests/trainer/test_validation_steps_result_return.py @@ -214,12 +214,15 @@ def test_val_step_only_step_metrics(tmpdir): # make sure we logged the correct epoch metrics total_empty_epoch_metrics = 0 + epoch = 0 for metric in trainer.dev_debugger.logged_metrics: + if 'epoch' in metric: + epoch += 1 if len(metric) > 2: assert 'no_val_no_pbar' not in metric assert 'val_step_pbar_acc' not in metric - assert metric['val_step_log_acc'] - assert metric['val_step_log_pbar_acc'] + assert metric[f'val_step_log_acc/epoch_{epoch}'] + assert metric[f'val_step_log_pbar_acc/epoch_{epoch}'] else: total_empty_epoch_metrics += 1 @@ -228,6 +231,8 @@ def test_val_step_only_step_metrics(tmpdir): # make sure we logged the correct epoch pbar metrics total_empty_epoch_metrics = 0 for metric in trainer.dev_debugger.pbar_added_metrics: + if 'epoch' in metric: + epoch += 1 if len(metric) > 2: assert 'no_val_no_pbar' not in metric assert 'val_step_log_acc' not in metric @@ -288,11 +293,12 @@ def test_val_step_epoch_step_metrics(tmpdir): for metric_idx in range(0, len(trainer.dev_debugger.logged_metrics), batches + 1): batch_metrics = trainer.dev_debugger.logged_metrics[metric_idx: metric_idx + batches] epoch_metric = trainer.dev_debugger.logged_metrics[metric_idx + batches] + epoch = epoch_metric['epoch'] # make sure the metric was split for batch_metric in batch_metrics: - assert 'step_val_step_log_acc' in batch_metric - assert 'step_val_step_log_pbar_acc' in batch_metric + assert f'step_val_step_log_acc/epoch_{epoch}' in batch_metric + assert f'step_val_step_log_pbar_acc/epoch_{epoch}' in batch_metric # make sure the epoch split was correct assert 'epoch_val_step_log_acc' in epoch_metric From 46d69f06e653bf08912d043b6c3bb7665f527ea7 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 08:10:19 -0400 Subject: [PATCH 10/11] add step metrics --- tests/trainer/test_validation_steps_result_return.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/trainer/test_validation_steps_result_return.py b/tests/trainer/test_validation_steps_result_return.py index 02773f5fbd8e8..c01a626cb9c8b 100644 --- a/tests/trainer/test_validation_steps_result_return.py +++ b/tests/trainer/test_validation_steps_result_return.py @@ -427,11 +427,11 @@ def test_val_step_full_loop_result_dp(tmpdir): assert 'train_step_metric' in seen_keys assert 'train_step_end_metric' in seen_keys assert 'epoch_train_epoch_end_metric' in seen_keys - assert 'step_validation_step_metric' in seen_keys + assert 'step_validation_step_metric/epoch_0' in seen_keys assert 'epoch_validation_step_metric' in seen_keys assert 'validation_step_end_metric' in seen_keys assert 'validation_epoch_end_metric' in seen_keys - assert 'step_test_step_metric' in seen_keys + assert 'step_test_step_metric/epoch_0' in seen_keys assert 'epoch_test_step_metric' in seen_keys assert 'test_step_end_metric' in seen_keys assert 'test_epoch_end_metric' in seen_keys From 0be1e67246d915cb9f31e2aa4a7ffe1af1adbc38 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 15 Aug 2020 08:23:02 -0400 Subject: [PATCH 11/11] add step metrics --- tests/trainer/test_validation_steps_result_return.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/trainer/test_validation_steps_result_return.py b/tests/trainer/test_validation_steps_result_return.py index c01a626cb9c8b..28f012535d1ab 100644 --- a/tests/trainer/test_validation_steps_result_return.py +++ b/tests/trainer/test_validation_steps_result_return.py @@ -431,7 +431,7 @@ def test_val_step_full_loop_result_dp(tmpdir): assert 'epoch_validation_step_metric' in seen_keys assert 'validation_step_end_metric' in seen_keys assert 'validation_epoch_end_metric' in seen_keys - assert 'step_test_step_metric/epoch_0' in seen_keys + assert 'step_test_step_metric/epoch_2' in seen_keys assert 'epoch_test_step_metric' in seen_keys assert 'test_step_end_metric' in seen_keys assert 'test_epoch_end_metric' in seen_keys