From 8ff56158c87d8fca899d2713dc2a5c2d6b020e66 Mon Sep 17 00:00:00 2001 From: "Todd A. Oliver" Date: Thu, 1 Feb 2024 14:27:36 -0600 Subject: [PATCH] Add doxygen comments for IO classes (#244) Plus a couple name changes to be a tiny bit more descriptive --- src/io.cpp | 19 ++-- src/io.hpp | 251 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 242 insertions(+), 28 deletions(-) diff --git a/src/io.cpp b/src/io.cpp index 8791d89bc..37b2cc085 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -29,6 +29,9 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -----------------------------------------------------------------------------------el- +/** @file + * @copydoc io.hpp + */ #include "io.hpp" #include @@ -371,7 +374,7 @@ hsize_t get_variable_size_hdf5(hid_t file, std::string name) { } // convenience function to read solution data for parallel restarts -void read_partitioned_soln_data(hid_t file, string varName, size_t index, double *data) { +void read_variable_data_hdf5(hid_t file, string varName, size_t index, double *data) { hid_t data_soln; herr_t status; @@ -407,7 +410,7 @@ void IOFamily::readDistributeSerializedVariable(hid_t file, const IOVar &var, in Vector data_serial; data_serial.SetSize(numDof); - read_partitioned_soln_data(file, varName, 0, data_serial.HostWrite()); + read_variable_data_hdf5(file, varName, 0, data_serial.HostWrite()); // assign solution owned by rank 0 Array lvdofs, gvdofs; @@ -462,7 +465,7 @@ void IOFamily::readDistributeSerializedVariable(hid_t file, const IOVar &var, in } // convenience function to write HDF5 data -void write_soln_data(hid_t group, string varName, hid_t dataspace, const double *data) { +void write_variable_data_hdf5(hid_t group, string varName, hid_t dataspace, const double *data) { hid_t data_soln; herr_t status; assert(group >= 0); @@ -607,7 +610,7 @@ void IOFamily::writePartitioned(hid_t file) { // save raw data for (auto var : vars_) { if (rank0_) grvy_printf(ginfo, " --> Saving (%s)\n", var.varName_.c_str()); - write_soln_data(group, var.varName_, dataspace, data + var.index_ * dims[0]); + write_variable_data_hdf5(group, var.varName_, dataspace, data + var.index_ * dims[0]); } if (group >= 0) H5Gclose(group); @@ -639,7 +642,7 @@ void IOFamily::writeSerial(hid_t file) { // save raw data for (auto var : vars_) { grvy_printf(ginfo, " --> Saving (%s)\n", var.varName_.c_str()); - write_soln_data(group, var.varName_, dataspace, data + var.index_ * dims[0]); + write_variable_data_hdf5(group, var.varName_, dataspace, data + var.index_ * dims[0]); } if (group >= 0) H5Gclose(group); @@ -661,7 +664,7 @@ void IOFamily::readPartitioned(hid_t file) { if (var.inRestartFile_) { std::string h5Path = group_ + "/" + var.varName_; if (rank0_) grvy_printf(ginfo, "--> Reading h5 path = %s\n", h5Path.c_str()); - read_partitioned_soln_data(file, h5Path.c_str(), var.index_ * numInSoln, data); + read_variable_data_hdf5(file, h5Path.c_str(), var.index_ * numInSoln, data); } } } @@ -710,7 +713,7 @@ void IOFamily::readChangeOrder(hid_t file, int read_order) { if (var.inRestartFile_) { std::string h5Path = group_ + "/" + var.varName_; if (rank0_) grvy_printf(ginfo, "--> Reading h5 path = %s\n", h5Path.c_str()); - read_partitioned_soln_data(file, h5Path.c_str(), var.index_ * aux_dof, data); + read_variable_data_hdf5(file, h5Path.c_str(), var.index_ * aux_dof, data); } } @@ -761,7 +764,7 @@ void IODataOrganizer::registerIOVar(std::string group, std::string varName, int } // return the index to IO family given a group name -int IODataOrganizer::getIOFamilyIndex(std::string group) { +int IODataOrganizer::getIOFamilyIndex(std::string group) const { for (size_t i = 0; i < families_.size(); i++) if (families_[i].group_ == group) return (i); diff --git a/src/io.hpp b/src/io.hpp index fc414f6ae..556f5cd30 100644 --- a/src/io.hpp +++ b/src/io.hpp @@ -31,6 +31,11 @@ // -----------------------------------------------------------------------------------el- #ifndef IO_HPP_ #define IO_HPP_ +/** @file + * @brief Contains utilities for abstracting field IO tasks + * + * The primary class is IODataOrganizer. For example usage, see M2ulPhyS. + */ #include @@ -43,86 +48,292 @@ class IODataOrganizer; +/** + * @brief Stores info about variables to read/write. + * + * This info is used in IOFamily and IODataOrganizer and is not + * intended to be needed separately. + */ class IOVar { public: - std::string varName_; // solution variable - int index_; // variable index in the pargrid function - bool inRestartFile_; // Check if we want to read this variable + /** Variable name, which combines with family name to set path in hdf5 file */ + std::string varName_; + + /** Variable index within variables stored in corresponding ParGridFunction */ + int index_; // variable index in the pargrid function + + /** Whether variable is to be read + * + * This allows variables in the state to be skipped, e.g., to enable + * restarts from models that do not have all the same state variables. + */ + bool inRestartFile_; }; +/** + * @brief Stores info about a "family" of variables to read/write. + * + * Here a "family" refers to data stored in a single + * mfem::ParGridFunction object. Thus, the family has a single mesh + * and parallel decomposition. For the moment, the underlying mesh + * and decomposition is assumed fixed after initialization. + * + * This class is intended to be used through IODataOrganizer, which + * can track multiple IOFamily objects. + */ class IOFamily { friend class IODataOrganizer; protected: + /** Indicates whether current mpi rank is rank 0*/ bool rank0_ = false; + /** Local number of elements in this mpi rank */ int local_ne_ = -1; + + /** Global number of elements in whole mesh */ int global_ne_ = -1; + /** Local number of degrees of freedom on this mpi rank, as returned by ParGridFunction::GetNDofs */ int local_ndofs_ = -1; + + /** Global number of degrees of freedom */ int global_ndofs_ = -1; - std::string description_; // family description - std::string group_; // HDF5 group name - mfem::ParGridFunction *pfunc_; // pargrid function owning the data for this IO family - std::vector vars_; // solution var info for this family + /** Description string */ + std::string description_; + + /** Group name, used to set corresponding group in hdf5 restart files */ + std::string group_; + + /** mfem::ParGridFunction owning the data to be written or read into */ + mfem::ParGridFunction *pfunc_ = nullptr; + /** Variables that make up the family */ + std::vector vars_; + + /** Whether this family allows initialization (read) from other order elements */ bool allowsAuxRestart; - // true if the family is to be found in restart files + /** true if the family is to be found in restart files */ bool inRestartFile; - // space and grid function used for serial read/write + /** mfem::FiniteElementSpace for serial mesh (used if serial read and/or write requested) */ mfem::FiniteElementSpace *serial_fes = nullptr; + + /** mfem::GridFunction on serial mesh (used if serial read and/or write requested) */ mfem::GridFunction *serial_sol = nullptr; - // additional data used for serial read/write (needed to properly - // serialize fields in write or properly distribute data in read) - int *local_to_global_elem_ = nullptr; // maps from local elem index to global - mfem::Array *partitioning_ = nullptr; // partitioning[i] mpi rank for element i (in global numbering) + /** Map from local to global element numbering (used for serial read/write) */ + int *local_to_global_elem_ = nullptr; + + /** Partition array; partitioning[i] = mpi rank that owns the ith element (global numbering) */ + mfem::Array *partitioning_ = nullptr; - // Variables used for "auxilliary" read (i.e., restart from different order soln) - mfem::FiniteElementCollection *fec_ = nullptr; // collection used to instantiate pfunc_ + /** mfem::FiniteElementCollection used by pfunc_ (used for variable order read) */ + mfem::FiniteElementCollection *fec_ = nullptr; + + /** mfem::FiniteElementCollection with read order (used for variable order read) */ mfem::FiniteElementCollection *aux_fec_ = nullptr; + + /** mfem::FiniteElementCollection with read order elements (used for variable order read) */ mfem::ParFiniteElementSpace *aux_pfes_ = nullptr; + + /** mfem::ParGridFunction with read order elements (used for variable order read) */ mfem::ParGridFunction *aux_pfunc_ = nullptr; + /** + * @brief Prepare for serial write by collecting data onto rank 0 + * + * Each rank sends its data in pfunc_ to rank 0 and stores the + * results in serial_sol + */ void serializeForWrite(); + + /** + * @brief Read and distribute a single variable in a serialized restart file + * + * Rank 0 reads the data and sends to appropriate rank and location in pfunc_ + * @param file Open HDF5 file handle (e.g., as returned by H5Fcreate or H5Fopen) + * @param var IOVar for the variable being read + */ void readDistributeSerializedVariable(hid_t file, const IOVar &var, int numDof, double *data); public: + /** + * @brief Constructor, must provide description, group name, and ParGridFunction + * + * @param desc Description string + * @param grp Group name + * @param pf Pointer to mfem::ParGridFunction that owns the data + */ IOFamily(std::string desc, std::string grp, mfem::ParGridFunction *pf); + /** @brief "Partitioned" write (each mpi rank writes its local part of the ParGridFunction to a separate file) */ void writePartitioned(hid_t file); + + /** @brief "Serial" write (all data communicated to rank 0, which writes everything to a single file) */ void writeSerial(hid_t file); + + /** @brief "Partitioned" read (inverse of writePartitioned) */ void readPartitioned(hid_t file); + + /** @brief "Serial" read (inverse of readPartitioned) */ void readSerial(hid_t file); + + /** + * @brief "Partitioned" read of a different order solution + * + * which is then interpolated onto the desired space, as defined by pfunc_ + * @param file Open HDF5 file handle (e.g., as returned by H5Fcreate or H5Fopen) + * @param read_order Polynomial order of function in HDF5 file + */ void readChangeOrder(hid_t file, int read_order); }; +/** + * @brief Provides IO interface for multiple "families" + * + * This class is used to read & write the solution data contained in + * the mfem::ParGridFunction object in each IOFamily that is + * "registered". Once families and variables are registered, the + * underlying data can be read or written with a single call. Since + * it is expected that the calling class will have some class-specific + * metadata to read or write in addition to the solution data, this + * class does not handle opening or creating the hdf5 file or reading + * or writing solution metadata. These tasks must be handled outside + * the IODataOrganizer class. For example usage, see + * M2ulPhyS::initSolutionAndVisualizationVectors and + * M2ulPhyS::write_restart_files_hdf5. + */ class IODataOrganizer { protected: + /** Whether serial read/write is supported. To enable serial support, must call initializeSerial */ bool supports_serial_ = false; - public: - std::vector families_; // registered IO families + /** Registered IO families */ + std::vector families_; + /** Get the index of the IOFamily with group name group */ + int getIOFamilyIndex(std::string group) const; + + public: + /** + * @brief Register an IOFamily + * + * Constructs the family using the input arguments. + * + * @param description Description string for the family + * @param group Group name for the family + * @param pfunc mfem::ParGridFunction for the family + * @param auxRestart Allow initialization from different order read (optional, defaults to true) + * @param inRestartFile If false, skip family on read (optional, defaults to true) + * @param fec FiniteElementCollection for grid function (optional, only used for different order read) + */ void registerIOFamily(std::string description, std::string group, mfem::ParGridFunction *pfunc, bool auxRestart = true, bool inRestartFile = true, mfem::FiniteElementCollection *fec = NULL); + /** Destructor */ ~IODataOrganizer(); + /** + * @brief Register an IOVar as part of the corresponding family + * + * @param group Group name for the family to add the variable to + * @param varName Name of the variable + * @param index Variable index within the ParGridFunction of the family + * @param inRestartFile If false, skip variable on read (optional, defaults to true) + */ void registerIOVar(std::string group, std::string varName, int index, bool inRestartFile = true); - int getIOFamilyIndex(std::string group); + /** + * @brief Initialize data supporting serial read/write. + * + * @param root True for mpi rank = 0 + * @param serial True if we want seral read and/or write + * @param serial_mesh Pointer to serial mesh object + * @param locToGlob Map from local element index to global element index + * @param part Map from global element index to mpi rank owning that element + */ void initializeSerial(bool root, bool serial, mfem::Mesh *serial_mesh, int *locToGlob, mfem::Array *part); + /** + * @brief Write data from all families + * + * All families written to the same HDF5 file, which must be already + * open. If serial write requested, a data is written to a single + * HDF5 file by rank 0. Otherwise, all ranks write to separate + * files. + * + * @param file HDF5 file handle (e.g., as returned by H5Fcreate or H5Fopen) + * @param serial True for serial write + */ void write(hid_t file, bool serial); + + /** + * @brief Read data for all families + * + * All families are read from the same HDF5 file, which must be + * already open. If serial read is requested, the data is read from + * a single HDF5 file by rank 0. Otherwise, all ranks read from + * separate files. For variable order (i.e., data in file had + * different order than desired), the order of the file being read + * must be supplied. + * + * @param file HDF5 file handle (e.g., as returned by H5Fopen) + * @param serial True for serial read + * @param read_order Polynomial order for data in file (optional, only required for variable order read) + */ void read(hid_t file, bool serial, int read_order = -1); }; +/** + * @brief Get number of points to be read + * + * This convenience function returns the number of points to be read + * for a given name. This is used frequently to check for consistency + * between the size of the read and the size of the container being + * read into. + * + * @param file Open HDF5 file + * @param name Name to query (corresponds to "family group/variable name") + * @returns Size of corresponding data in file + */ hsize_t get_variable_size_hdf5(hid_t file, std::string name); -void read_partitioned_soln_data(hid_t file, std::string varName, size_t index, double *data); -void write_soln_data(hid_t group, std::string varName, hid_t dataspace, const double *data); + +/** + * @brief Read data from hdf5 file + * + * Reads data from the hdf5 file into the data buffer, starting at + * location index (i.e., fill starting at data[index]) + * + * @param file Open HDF5 file + * @param varName Name of object to read (corresponds to "family group/variable name") + * @param index Starting location within data buffer + * @param data Data buffer + */ +void read_variable_data_hdf5(hid_t file, std::string varName, size_t index, double *data); + +/** + * @brief Write data to an hdf5 file + * + * @param group Group within the hdf5 file to write to (e.g., as returned by H5Gcreate) + * @param varName Name of variable to write + * @param dataspace HDF5 dataspace (e.g., as returned by H5Screate_simple) + * @param data Buffer with data to write + */ +void write_variable_data_hdf5(hid_t group, std::string varName, hid_t dataspace, const double *data); + +/** + * @brief Read/write partitioning information + * + * For normal restarts, this is used to write/read the partitioning + * information to ensure that it remains consistent across the + * restart. For serial restarts, it is also used to properly + * distribute the incoming fields. + * + * @todo Refactor this function to make it more generic and fit better + * into the IODataOrganizer paradigm. + */ void partitioning_file_hdf5(std::string mode, const RunConfiguration &config, MPI_Groups *groupsMPI, int nelemGlobal, mfem::Array &partitioning); #endif // IO_HPP_