Update examples (#1371)

* Update examples

Mainly Series::close, Iteration::close, Series::writeIterations,
Series::readIterations, READ_LINEAR

* Fix: Python bindings use copy return value policies

* Fix: Use explicit ::close() call in WriteIterations

examples/10_streaming_read.cpp

@@ -39,6 +39,9 @@ int main()
             extents[i] = rc.getExtent();
         }
 
+        // The iteration can be closed in order to help free up resources.
+        // The iteration's content will be flushed automatically.
+        // An iteration once closed cannot (yet) be reopened.
         iteration.close();
 
         for (size_t i = 0; i < 3; ++i)
@@ -55,6 +58,14 @@ int main()
         }
     }
 
+    /* The files in 'series' are still open until the object is destroyed, on
+     * which it cleanly flushes and closes all open file handles.
+     * When running out of scope on return, the 'Series' destructor is called.
+     * Alternatively, one can call `series.close()` to the same effect as
+     * calling the destructor, including the release of file handles.
+     */
+    series.close();
+
     return 0;
 #else
     std::cout << "The streaming example requires that openPMD has been built "

examples/10_streaming_read.py

@@ -53,3 +53,10 @@
             print("dim: {}".format(dim))
             chunk = loadedChunks[i]
             print(chunk)
+
+    # The files in 'series' are still open until the object is destroyed, on
+    # which it cleanly flushes and closes all open file handles.
+    # When running out of scope on return, the 'Series' destructor is called.
+    # Alternatively, one can call `series.close()` to the same effect as
+    # calling the destructor, including the release of file handles.
+    series.close()

examples/10_streaming_write.cpp

@@ -45,6 +45,14 @@ int main()
         iteration.close();
     }
 
+    /* The files in 'series' are still open until the object is destroyed, on
+     * which it cleanly flushes and closes all open file handles.
+     * When running out of scope on return, the 'Series' destructor is called.
+     * Alternatively, one can call `series.close()` to the same effect as
+     * calling the destructor, including the release of file handles.
+     */
+    series.close();
+
     return 0;
 #else
     std::cout << "The streaming example requires that openPMD has been built "

examples/10_streaming_write.py

@@ -80,3 +80,10 @@
         # If not closing an iteration explicitly, it will be implicitly closed
         # upon creating the next iteration.
         iteration.close()
+
+    # The files in 'series' are still open until the object is destroyed, on
+    # which it cleanly flushes and closes all open file handles.
+    # When running out of scope on return, the 'Series' destructor is called.
+    # Alternatively, one can call `series.close()` to the same effect as
+    # calling the destructor, including the release of file handles.
+    series.close()

examples/11_particle_dataframe.py

@@ -96,3 +96,5 @@
             idx_max * E.grid_spacing + E.grid_global_offset)
         print("maximum intensity I={} at index={} z={}mu".format(
             Intensity_max, idx_max, pos_max[2]))
+
+    s.close()

examples/12_span_write.cpp

@@ -84,6 +84,14 @@ void span_write(std::string const &filename)
         }
         iteration.close();
     }
+
+    /* The files in 'series' are still open until the object is destroyed, on
+     * which it cleanly flushes and closes all open file handles.
+     * When running out of scope on return, the 'Series' destructor is called.
+    # Alternatively, one can call `series.close()` to the same effect as
+    # calling the destructor, including the release of file handles.
+     */
+    series.close();
 }
 
 int main()

examples/12_span_write.py

@@ -27,6 +27,13 @@ def span_write(filename):
             j += 1
         iteration.close()
 
+    # The files in 'series' are still open until the object is destroyed, on
+    # which it cleanly flushes and closes all open file handles.
+    # When running out of scope on return, the 'Series' destructor is called.
+    # Alternatively, one can call `series.close()` to the same effect as
+    # calling the destructor, including the release of file handles.
+    series.close()
+
 
 if __name__ == "__main__":
     for ext in io.file_extensions:

examples/13_write_dynamic_configuration.cpp

@@ -128,5 +128,13 @@ chunks = "auto"
         iteration.close();
     }
 
+    /* The files in 'series' are still open until the object is destroyed, on
+     * which it cleanly flushes and closes all open file handles.
+     * When running out of scope on return, the 'Series' destructor is called.
+     * Alternatively, one can call `series.close()` to the same effect as
+     * calling the destructor, including the release of file handles.
+     */
+    series.close();
+
     return 0;
 }

examples/13_write_dynamic_configuration.py

@@ -146,6 +146,13 @@ def main():
         # upon creating the next iteration.
         iteration.close()
 
+    # The files in 'series' are still open until the object is destroyed, on
+    # which it cleanly flushes and closes all open file handles.
+    # When running out of scope on return, the 'Series' destructor is called.
+    # Alternatively, one can call `series.close()` to the same effect as
+    # calling the destructor, including the release of file handles.
+    series.close()
+
 
 if __name__ == "__main__":
     main()

examples/1_structure.cpp

@@ -39,7 +39,8 @@ int main()
      * to the openPMD standard. Creation of new elements happens on access
      * inside the tree-like structure. Required attributes are initialized to
      * reasonable defaults for every object. */
-    ParticleSpecies electrons = series.iterations[1].particles["electrons"];
+    ParticleSpecies electrons =
+        series.writeIterations()[1].particles["electrons"];
 
     /* Data to be moved from memory to persistent storage is structured into
      * Records, each holding an unbounded number of RecordComponents. If a
@@ -59,9 +60,17 @@ int main()
     electrons["positionOffset"]["x"].resetDataset(dataset);
     electrons["positionOffset"]["x"].makeConstant(22.0);
 
+    // The iteration can be closed in order to help free up resources.
+    // The iteration's content will be flushed automatically.
+    // An iteration once closed cannot (yet) be reopened.
+    series.writeIterations()[1].close();
+
     /* The files in 'series' are still open until the object is destroyed, on
      * which it cleanly flushes and closes all open file handles.
      * When running out of scope on return, the 'Series' destructor is called.
+     * Alternatively, one can call `series.close()` to the same effect as
+     * calling the destructor, including the release of file handles.
      */
+    series.close();
     return 0;
 }

examples/2_read_serial.cpp

@@ -91,7 +91,11 @@ int main()
     }
 
     auto all_data = E_x.loadChunk<double>();
-    series.flush();
+
+    // The iteration can be closed in order to help free up resources.
+    // The iteration's content will be flushed automatically.
+    // An iteration once closed cannot (yet) be reopened.
+    i.close();
     cout << "Full E/x starts with:\n\t{";
     for (size_t col = 0; col < extent[1] && col < 5; ++col)
         cout << all_data.get()[col] << ", ";
@@ -103,5 +107,6 @@ int main()
      * Alternatively, one can call `series.close()` to the same effect as
      * calling the destructor, including the release of file handles.
      */
+    series.close();
     return 0;
 }

examples/2_read_serial.py

@@ -61,7 +61,11 @@
     #     print("")
 
     all_data = E_x.load_chunk()
-    series.flush()
+
+    # The iteration can be closed in order to help free up resources.
+    # The iteration's content will be flushed automatically.
+    # An iteration once closed cannot (yet) be reopened.
+    i.close()
     print("Full E/x is of shape {0} and starts with:".format(all_data.shape))
     print(all_data[0, 0, :5])
 

examples/2a_read_thetaMode_serial.cpp

@@ -69,11 +69,17 @@ int main()
     // toCartesianSliceYZ(E_z_modes).loadChunk<double>();  # (y, z)
     // series.flush();
 
+    // The iteration can be closed in order to help free up resources.
+    // The iteration's content will be flushed automatically.
+    // An iteration once closed cannot (yet) be reopened.
+    i.close();
+
     /* The files in 'series' are still open until the object is destroyed, on
      * which it cleanly flushes and closes all open file handles.
      * When running out of scope on return, the 'Series' destructor is called.
      * Alternatively, one can call `series.close()` to the same effect as
      * calling the destructor, including the release of file handles.
      */
+    series.close();
     return 0;
 }

examples/2a_read_thetaMode_serial.py

@@ -51,6 +51,13 @@
     # E_z_yz  = toCartesianSliceYZ(E_z_modes)[:, :]  # (y, z)
     # series.flush()
 
+    # The iteration can be closed in order to help free up resources.
+    # The iteration's content will be flushed automatically.
+    # An iteration once closed cannot (yet) be reopened.
+    # Alternatively, one can call `series.close()` to the same effect as
+    # calling the destructor, including the release of file handles.
+    i.close()
+
     # The files in 'series' are still open until the series is closed, at which
     # time it cleanly flushes and closes all open file handles.
     # One can close the object explicitly to trigger this.

examples/3_write_serial.cpp

@@ -45,7 +45,7 @@ int main(int argc, char *argv[])
     cout << "Created an empty " << series.iterationEncoding() << " Series\n";
 
     MeshRecordComponent rho =
-        series.iterations[1].meshes["rho"][MeshRecordComponent::SCALAR];
+        series.writeIterations()[1].meshes["rho"][MeshRecordComponent::SCALAR];
     cout << "Created a scalar mesh Record with all required openPMD "
             "attributes\n";
 
@@ -67,7 +67,11 @@ int main(int argc, char *argv[])
     cout << "Stored the whole Dataset contents as a single chunk, "
             "ready to write content\n";
 
-    series.flush();
+    // The iteration can be closed in order to help free up resources.
+    // The iteration's content will be flushed automatically.
+    // An iteration once closed cannot (yet) be reopened.
+    series.writeIterations()[1].close();
+
     cout << "Dataset content has been fully written\n";
 
     /* The files in 'series' are still open until the object is destroyed, on
@@ -76,5 +80,6 @@ int main(int argc, char *argv[])
      * Alternatively, one can call `series.close()` to the same effect as
      * calling the destructor, including the release of file handles.
      */
+    series.close();
     return 0;
 }

examples/3_write_serial.py

@@ -28,7 +28,7 @@
     print("Created an empty {0} Series".format(series.iteration_encoding))
 
     print(len(series.iterations))
-    rho = series.iterations[1]. \
+    rho = series.write_iterations()[1]. \
         meshes["rho"][io.Mesh_Record_Component.SCALAR]
 
     dataset = io.Dataset(data.dtype, data.shape)
@@ -47,7 +47,10 @@
     print("Stored the whole Dataset contents as a single chunk, " +
           "ready to write content")
 
-    series.flush()
+    # The iteration can be closed in order to help free up resources.
+    # The iteration's content will be flushed automatically.
+    # An iteration once closed cannot (yet) be reopened.
+    series.write_iterations()[1].close()
     print("Dataset content has been fully written")
 
     # The files in 'series' are still open until the series is closed, at which

examples/3a_write_thetaMode_serial.cpp

@@ -51,7 +51,7 @@ int main()
     geos << "m=" << num_modes << ";imag=+";
     std::string const geometryParameters = geos.str();
 
-    Mesh E = series.iterations[0].meshes["E"];
+    Mesh E = series.writeIterations()[0].meshes["E"];
     E.setGeometry(Mesh::Geometry::thetaMode);
     E.setGeometryParameters(geometryParameters);
     E.setDataOrder(Mesh::DataOrder::C);
@@ -84,13 +84,17 @@ int main()
     E_t.resetDataset(Dataset(Datatype::FLOAT, {num_fields, N_r, N_z}));
     E_t.storeChunk(E_t_data, Offset{0, 0, 0}, Extent{num_fields, N_r, N_z});
 
-    series.flush();
+    // The iteration can be closed in order to help free up resources.
+    // The iteration's content will be flushed automatically.
+    // An iteration once closed cannot (yet) be reopened.
+    series.writeIterations()[0].close();
 
     /* The files in 'series' are still open until the object is destroyed, on
      * which it cleanly flushes and closes all open file handles.
      * When running out of scope on return, the 'Series' destructor is called.
      * Alternatively, one can call `series.close()` to the same effect as
      * calling the destructor, including the release of file handles.
      */
+    series.close();
     return 0;
 }

examples/3a_write_thetaMode_serial.py

@@ -30,7 +30,7 @@
 
     geometry_parameters = "m={0};imag=+".format(num_modes)
 
-    E = series.iterations[0].meshes["E"]
+    E = series.write_iterations()[0].meshes["E"]
     E.geometry = io.Geometry.thetaMode
     E.geometry_parameters = geometry_parameters
     E.grid_spacing = [1.0, 1.0]
@@ -62,7 +62,10 @@
     E_t.reset_dataset(io.Dataset(E_t_data.dtype, E_t_data.shape))
     E_t.store_chunk(E_t_data)
 
-    series.flush()
+    # The iteration can be closed in order to help free up resources.
+    # The iteration's content will be flushed automatically.
+    # An iteration once closed cannot (yet) be reopened.
+    series.write_iterations()[0].close()
 
     # The files in 'series' are still open until the series is closed, at which
     # time it cleanly flushes and closes all open file handles.

examples/3b_write_resizable_particles.cpp

@@ -32,7 +32,8 @@ int main()
     Series series =
         Series("../samples/3b_write_resizable_particles.h5", Access::CREATE);
 
-    ParticleSpecies electrons = series.iterations[0].particles["electrons"];
+    ParticleSpecies electrons =
+        series.writeIterations()[0].particles["electrons"];
 
     // our initial data to write
     std::vector<double> x{0., 1., 2., 3., 4.};
@@ -78,8 +79,14 @@ int main()
     rc_xo.resetDataset(dataset);
     rc_yo.resetDataset(dataset);
 
-    // after this call, the provided data buffers can be used again or deleted
-    series.flush();
+    // Attributable::seriesFlush() can be used alternatively if the Series
+    // handle is not currently in scope
+    rc_yo.seriesFlush();
+
+    // The iteration can be closed in order to help free up resources.
+    // The iteration's content will be flushed automatically.
+    // An iteration once closed cannot (yet) be reopened.
+    series.writeIterations()[0].close();
 
     // rinse and repeat as needed :)
 
@@ -89,5 +96,6 @@ int main()
      * Alternatively, one can call `series.close()` to the same effect as
      * calling the destructor, including the release of file handles.
      */
+    series.close();
     return 0;
 }

examples/3b_write_resizable_particles.py

@@ -60,8 +60,11 @@
     rc_xo.reset_dataset(dataset)
     rc_yo.reset_dataset(dataset)
 
+    # The iteration can be closed in order to help free up resources.
+    # The iteration's content will be flushed automatically.
+    # An iteration once closed cannot (yet) be reopened.
     # after this call, the provided data buffers can be used again or deleted
-    series.flush()
+    series.write_iterations()[0].close()
 
     # rinse and repeat as needed :)
 

examples/4_read_parallel.cpp

@@ -55,7 +55,11 @@ int main(int argc, char *argv[])
         cout << "Queued the loading of a single chunk per MPI rank from "
                 "disk, "
                 "ready to execute\n";
-    series.flush();
+
+    // The iteration can be closed in order to help free up resources.
+    // The iteration's content will be flushed automatically.
+    // An iteration once closed cannot (yet) be reopened.
+    series.iterations[100].close();
 
     if (0 == mpi_rank)
         cout << "Chunks have been read from disk\n";
@@ -78,6 +82,12 @@ int main(int argc, char *argv[])
         // this barrier is not necessary but structures the example output
         MPI_Barrier(MPI_COMM_WORLD);
     }
+    // The files in 'series' are still open until the series is closed, at which
+    // time it cleanly flushes and closes all open file handles.
+    // One can close the object explicitly to trigger this.
+    // Alternatively, this will automatically happen once the garbage collector
+    // claims (every copy of) the series object.
+    // In any case, this must happen before MPI_Finalize() is called
     series.close();
 
     // openPMD::Series MUST be destructed or closed at this point