From 99b98e2d986e9fc4e6abe54a78edc9a9f1e28665 Mon Sep 17 00:00:00 2001 From: Xing Zhao LEE <49442025+xingzhaolee@users.noreply.github.com> Date: Thu, 12 Mar 2020 22:50:00 +0800 Subject: [PATCH] Run on_validation_end only on main process in DDP (#1125) Co-authored-by: xingzhaolee --- pytorch_lightning/callbacks/model_checkpoint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py index 832a56654e25d7..c3d096eb75402f 100644 --- a/pytorch_lightning/callbacks/model_checkpoint.py +++ b/pytorch_lightning/callbacks/model_checkpoint.py @@ -175,6 +175,10 @@ def format_checkpoint_name(self, epoch, metrics, ver=None): return filepath def on_validation_end(self, trainer, pl_module): + # only run on main process + if trainer.proc_rank != 0: + return + metrics = trainer.callback_metrics epoch = trainer.current_epoch self.epochs_since_last_check += 1