From df2c6f134c063c867d5d391a1869ed80de43835f Mon Sep 17 00:00:00 2001 From: xingzhaolee Date: Thu, 12 Mar 2020 09:07:02 +0800 Subject: [PATCH] Run on_validation_end only on main process in DDP --- pytorch_lightning/callbacks/model_checkpoint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py index 832a56654e25d..c3d096eb75402 100644 --- a/pytorch_lightning/callbacks/model_checkpoint.py +++ b/pytorch_lightning/callbacks/model_checkpoint.py @@ -175,6 +175,10 @@ def format_checkpoint_name(self, epoch, metrics, ver=None): return filepath def on_validation_end(self, trainer, pl_module): + # only run on main process + if trainer.proc_rank != 0: + return + metrics = trainer.callback_metrics epoch = trainer.current_epoch self.epochs_since_last_check += 1