Run on_validation_end only on main process in DDP (Lightning-AI#1125)

Co-authored-by: xingzhaolee <[email protected]>
tullie · Apr 3, 2020 · 99b98e2 · 99b98e2
1 parent 59c2387
commit 99b98e2
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py
@@ -175,6 +175,10 @@ def format_checkpoint_name(self, epoch, metrics, ver=None):
         return filepath
 
     def on_validation_end(self, trainer, pl_module):
+        # only run on main process
+        if trainer.proc_rank != 0:
+            return
+
         metrics = trainer.callback_metrics
         epoch = trainer.current_epoch
         self.epochs_since_last_check += 1