From af98ceb2304c8c0f54a42b48e7a0954277237d27 Mon Sep 17 00:00:00 2001
From: wkliao <wkliao@users.noreply.github.com>
Date: Fri, 19 Jul 2024 13:01:58 -0500
Subject: [PATCH] Skip MPI communication for error code checking when PnetCDF
 is used

Note PnetCDF always keeps metadata consistent among all MPI ranks.
---
 src/clib/pio_nc.cpp | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/clib/pio_nc.cpp b/src/clib/pio_nc.cpp
index b19ebc7e0..4b8bdbfa7 100644
--- a/src/clib/pio_nc.cpp
+++ b/src/clib/pio_nc.cpp
@@ -469,12 +469,17 @@ int PIOc_inq_unlimdims_impl(int ncid, int *nunlimdimsp, int *unlimdimidsp)
 #endif
 
     /* A failure to inquire is not fatal */
-    mpierr = MPI_Allreduce(MPI_IN_PLACE, &ierr, 1, MPI_INT, MPI_BOR, ios->my_comm);
+    if (file->iotype != PIO_IOTYPE_PNETCDF) {
+        /* Skip MPI communication for checking error code, because PnetCDF
+         * always keeps metadata consistent among all ranks.
+         */
+        mpierr = MPI_Allreduce(MPI_IN_PLACE, &ierr, 1, MPI_INT, MPI_BOR, ios->my_comm);
 
-    if(mpierr != MPI_SUCCESS){
-        spio_ltimer_stop(ios->io_fstats->tot_timer_name);
-        spio_ltimer_stop(file->io_fstats->tot_timer_name);
-        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+        if(mpierr != MPI_SUCCESS){
+            spio_ltimer_stop(ios->io_fstats->tot_timer_name);
+            spio_ltimer_stop(file->io_fstats->tot_timer_name);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+        }
     }
 
     if(ierr != PIO_NOERR){
@@ -484,6 +489,12 @@ int PIOc_inq_unlimdims_impl(int ncid, int *nunlimdimsp, int *unlimdimidsp)
         return ierr;
     }
 
+    /* Skip MPI communications below, because PnetCDF always keeps metadata
+     * consistent among all ranks.
+     */
+    if (file->iotype == PIO_IOTYPE_PNETCDF)
+        goto fn_exit;
+
     /* Broadcast results to all tasks. Ignore NULL parameters. */
     if ((mpierr = MPI_Bcast(&tmp_nunlimdims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
     {
@@ -508,6 +519,7 @@ int PIOc_inq_unlimdims_impl(int ncid, int *nunlimdimsp, int *unlimdimidsp)
             return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         }
 
+fn_exit:
     spio_ltimer_stop(ios->io_fstats->tot_timer_name);
     spio_ltimer_stop(file->io_fstats->tot_timer_name);
     return PIO_NOERR;