From dc4ccc2e7dd246343dce2e44f743716d1ba06f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 Dec 2023 15:35:34 +0100 Subject: [PATCH 1/3] Turn attribute_writing_ranks into global option New concept: global options with backend-specific implementation --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 10 +++++++++- src/Series.cpp | 15 +++++++++++++++ test/ParallelIOTest.cpp | 3 ++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2d93837980..828cd2e65c 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -102,7 +102,7 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( MPI_Comm_rank(communicator, &rank); auto throw_error = []() { throw error::BackendConfigSchema( - {"adios2", "attribute_writing_ranks"}, + {"attribute_writing_ranks"}, "Type must be either an integer or an array of integers."); }; if (attribute_writing_ranks.is_array()) @@ -190,6 +190,8 @@ template void ADIOS2IOHandlerImpl::init( json::TracingJSON cfg, Callback &&callbackWriteAttributesFromRank) { + std::cout << "Initializing ADIOS2 with config:\n" + << cfg.json() << std::endl; // allow overriding through environment variable m_engineType = auxiliary::getEnvString("OPENPMD_ADIOS2_ENGINE", m_engineType); @@ -208,6 +210,12 @@ void ADIOS2IOHandlerImpl::init( groupTableViaEnv == 0 ? UseGroupTable::No : UseGroupTable::Yes; } + // Backend-independent options with backend-dependent implementations + if (cfg.json().contains("attribute_writing_ranks")) + { + callbackWriteAttributesFromRank(cfg["attribute_writing_ranks"].json()); + } + if (cfg.json().contains("adios2")) { m_config = cfg["adios2"]; diff --git a/src/Series.cpp b/src/Series.cpp index 9079f4d791..5201dfe52d 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2436,6 +2436,21 @@ namespace template void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) { + constexpr std::array + backend_independent_options_with_backend_specific_implementation = { + "attribute_writing_ranks"}; + + for (auto const &opt : + backend_independent_options_with_backend_specific_implementation) + { + // Suppress warnings for these options: The backends might or might not + // take those hints + if (options.json().contains(opt)) + { + options[opt]; + } + } + auto &series = get(); getJsonOption( options, "defer_iteration_parsing", series.m_parseLazily); diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9f3ae43ee2..ea7367e238 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1187,9 +1187,10 @@ doshuffle = "BLOSC_BITSHUFFLE" std::string writeConfigBP4 = R"END( +attribute_writing_ranks = 0 + [adios2] unused = "parameter" -attribute_writing_ranks = 0 )END" #if openPMD_HAS_ADIOS_2_9 "use_group_table = true" From 0b841f4a507523ec1f701164a8ec33d3587bc064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 Dec 2023 15:46:25 +0100 Subject: [PATCH 2/3] Read attribute_writing_ranks parameter also from HDF5 --- include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp | 1 + src/IO/HDF5/ParallelHDF5IOHandler.cpp | 40 ++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp index 9fee9978f0..0662f6afa7 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp @@ -116,6 +116,7 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl */ std::optional m_communicator; #endif + bool m_writeAttributesFromThisRank = true; json::TracingJSON m_config; std::optional m_buffered_dataset_config; diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index ecdbc8ab89..865245e316 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -70,7 +70,7 @@ std::future ParallelHDF5IOHandler::flush(internal::ParsedFlushParams &) ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl( AbstractIOHandler *handler, MPI_Comm comm, json::TracingJSON config) - : HDF5IOHandlerImpl{handler, std::move(config), /* do_warn_unused_params = */ false} + : HDF5IOHandlerImpl{handler, config, /* do_warn_unused_params = */ false} , m_mpiComm{comm} , m_mpiInfo{MPI_INFO_NULL} /* MPI 3.0+: MPI_INFO_ENV */ { @@ -81,6 +81,44 @@ ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl( m_fileAccessProperty = H5Pcreate(H5P_FILE_ACCESS); m_fileCreateProperty = H5Pcreate(H5P_FILE_CREATE); + if (config.json().contains("attribute_writing_ranks")) + { + auto const &attribute_writing_ranks = + config["attribute_writing_ranks"].json(); + int rank = 0; + MPI_Comm_rank(comm, &rank); + auto throw_error = []() { + throw error::BackendConfigSchema( + {"attribute_writing_ranks"}, + "Type must be either an integer or an array of integers."); + }; + if (attribute_writing_ranks.is_array()) + { + m_writeAttributesFromThisRank = false; + for (auto const &val : attribute_writing_ranks) + { + if (!val.is_number()) + { + throw_error(); + } + if (val.get() == rank) + { + m_writeAttributesFromThisRank = true; + break; + } + } + } + else if (attribute_writing_ranks.is_number()) + { + m_writeAttributesFromThisRank = + attribute_writing_ranks.get() == rank; + } + else + { + throw_error(); + } + } + #if H5_VERSION_GE(1, 10, 1) auto const hdf5_spaced_allocation = auxiliary::getEnvString("OPENPMD_HDF5_PAGED_ALLOCATION", "ON"); From d04c16db9286696eef6ae5a6b887f88621639504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 Dec 2023 16:58:16 +0100 Subject: [PATCH 3/3] WIP: Implement attribute_writing_ranks in HDF5 --- examples/5_write_parallel.cpp | 16 ++ src/IO/HDF5/HDF5IOHandler.cpp | 434 ++++++++++++++++++---------------- 2 files changed, 248 insertions(+), 202 deletions(-) diff --git a/examples/5_write_parallel.cpp b/examples/5_write_parallel.cpp index 3cf0f01883..2ca9f92f70 100644 --- a/examples/5_write_parallel.cpp +++ b/examples/5_write_parallel.cpp @@ -20,10 +20,13 @@ */ #include +#include + #include #include #include +#include #include // std::vector using std::cout; @@ -59,6 +62,19 @@ stripe_count = -1 chunks = "auto" )"; +#if 1 + subfiling_config = + []() { + std::stringstream res; + res << "attribute_writing_ranks = [" + << auxiliary::getEnvString("RANKS", "0") << "]"; + auto res_str = res.str(); + std::cout << "RETURNING: '" << res_str << "'" << std::endl; + return res_str; + }() + + subfiling_config; +#endif + // open file for writing Series series = Series( "../samples/5_parallel_write.h5", diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index d0f75e5734..5bdbc0a510 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #if openPMD_HAVE_HDF5 #include "openPMD/Datatype.hpp" @@ -1711,210 +1712,239 @@ void HDF5IOHandlerImpl::writeAttribute( "attribute write"); } - using DT = Datatype; - switch (dtype) + if (m_writeAttributesFromThisRank) { - case DT::CHAR: { - char c = att.get(); - status = H5Awrite(attribute_id, dataType, &c); - break; - } - case DT::UCHAR: { - auto u = att.get(); - status = H5Awrite(attribute_id, dataType, &u); - break; - } - case DT::SCHAR: { - auto u = att.get(); - status = H5Awrite(attribute_id, dataType, &u); - break; - } - case DT::SHORT: { - auto i = att.get(); - status = H5Awrite(attribute_id, dataType, &i); - break; - } - case DT::INT: { - int i = att.get(); - status = H5Awrite(attribute_id, dataType, &i); - break; - } - case DT::LONG: { - long i = att.get(); - status = H5Awrite(attribute_id, dataType, &i); - break; - } - case DT::LONGLONG: { - auto i = att.get(); - status = H5Awrite(attribute_id, dataType, &i); - break; - } - case DT::USHORT: { - auto u = att.get(); - status = H5Awrite(attribute_id, dataType, &u); - break; - } - case DT::UINT: { - auto u = att.get(); - status = H5Awrite(attribute_id, dataType, &u); - break; - } - case DT::ULONG: { - auto u = att.get(); - status = H5Awrite(attribute_id, dataType, &u); - break; - } - case DT::ULONGLONG: { - auto u = att.get(); - status = H5Awrite(attribute_id, dataType, &u); - break; - } - case DT::FLOAT: { - auto f = att.get(); - status = H5Awrite(attribute_id, dataType, &f); - break; - } - case DT::DOUBLE: { - auto d = att.get(); - status = H5Awrite(attribute_id, dataType, &d); - break; - } - case DT::LONG_DOUBLE: { - auto d = att.get(); - status = H5Awrite(attribute_id, dataType, &d); - break; - } - case DT::CFLOAT: { - std::complex f = att.get>(); - status = H5Awrite(attribute_id, dataType, &f); - break; - } - case DT::CDOUBLE: { - std::complex d = att.get>(); - status = H5Awrite(attribute_id, dataType, &d); - break; - } - case DT::CLONG_DOUBLE: { - std::complex d = att.get>(); - status = H5Awrite(attribute_id, dataType, &d); - break; - } - case DT::STRING: - status = - H5Awrite(attribute_id, dataType, att.get().c_str()); - break; - case DT::VEC_CHAR: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_SHORT: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_INT: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_LONG: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_LONGLONG: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_UCHAR: - status = H5Awrite( - attribute_id, - dataType, - att.get>().data()); - break; - case DT::VEC_SCHAR: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_USHORT: - status = H5Awrite( - attribute_id, - dataType, - att.get>().data()); - break; - case DT::VEC_UINT: - status = H5Awrite( - attribute_id, - dataType, - att.get>().data()); - break; - case DT::VEC_ULONG: - status = H5Awrite( - attribute_id, - dataType, - att.get>().data()); - break; - case DT::VEC_ULONGLONG: - status = H5Awrite( - attribute_id, - dataType, - att.get>().data()); - break; - case DT::VEC_FLOAT: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_DOUBLE: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_LONG_DOUBLE: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::VEC_CFLOAT: - status = H5Awrite( - attribute_id, - dataType, - att.get>>().data()); - break; - case DT::VEC_CDOUBLE: - status = H5Awrite( - attribute_id, - dataType, - att.get>>().data()); - break; - case DT::VEC_CLONG_DOUBLE: - status = H5Awrite( - attribute_id, - dataType, - att.get>>().data()); - break; - case DT::VEC_STRING: { - auto vs = att.get>(); - size_t max_len = 0; - for (std::string const &s : vs) - max_len = std::max(max_len, s.size() + 1); - std::unique_ptr c_str(new char[max_len * vs.size()]()); - for (size_t i = 0; i < vs.size(); ++i) - strncpy(c_str.get() + i * max_len, vs[i].c_str(), max_len); - status = H5Awrite(attribute_id, dataType, c_str.get()); - break; - } - case DT::ARR_DBL_7: - status = H5Awrite( - attribute_id, dataType, att.get>().data()); - break; - case DT::BOOL: { - bool b = att.get(); - status = H5Awrite(attribute_id, dataType, &b); - break; - } - case DT::UNDEFINED: - default: - throw std::runtime_error("[HDF5] Datatype not implemented in HDF5 IO"); + // std::cout << "Writing attribute " << parameters.name + // << ", filling it with value "; + // std::visit( + // [](auto const &val) { + // using T = + // std::remove_cv_t>; + // if constexpr ( + // !auxiliary::IsVector_v && !auxiliary::IsArray_v) + // { + // std::cout << val; + // } else + // { + // std::cout << "VECTOR/ARRAY"; + // } + // }, + // att.getResource()); + // std::cout << std::endl; + using DT = Datatype; + switch (dtype) + { + case DT::CHAR: { + char c = att.get(); + status = H5Awrite(attribute_id, dataType, &c); + break; + } + case DT::UCHAR: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } + case DT::SCHAR: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } + case DT::SHORT: { + auto i = att.get(); + status = H5Awrite(attribute_id, dataType, &i); + break; + } + case DT::INT: { + int i = att.get(); + status = H5Awrite(attribute_id, dataType, &i); + break; + } + case DT::LONG: { + long i = att.get(); + status = H5Awrite(attribute_id, dataType, &i); + break; + } + case DT::LONGLONG: { + auto i = att.get(); + status = H5Awrite(attribute_id, dataType, &i); + break; + } + case DT::USHORT: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } + case DT::UINT: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } + case DT::ULONG: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } + case DT::ULONGLONG: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } + case DT::FLOAT: { + auto f = att.get(); + status = H5Awrite(attribute_id, dataType, &f); + break; + } + case DT::DOUBLE: { + auto d = att.get(); + status = H5Awrite(attribute_id, dataType, &d); + break; + } + case DT::LONG_DOUBLE: { + auto d = att.get(); + status = H5Awrite(attribute_id, dataType, &d); + break; + } + case DT::CFLOAT: { + std::complex f = att.get>(); + status = H5Awrite(attribute_id, dataType, &f); + break; + } + case DT::CDOUBLE: { + std::complex d = att.get>(); + status = H5Awrite(attribute_id, dataType, &d); + break; + } + case DT::CLONG_DOUBLE: { + std::complex d = att.get>(); + status = H5Awrite(attribute_id, dataType, &d); + break; + } + case DT::STRING: + status = H5Awrite( + attribute_id, dataType, att.get().c_str()); + break; + case DT::VEC_CHAR: + status = H5Awrite( + attribute_id, dataType, att.get>().data()); + break; + case DT::VEC_SHORT: + status = H5Awrite( + attribute_id, dataType, att.get>().data()); + break; + case DT::VEC_INT: + status = H5Awrite( + attribute_id, dataType, att.get>().data()); + break; + case DT::VEC_LONG: + status = H5Awrite( + attribute_id, dataType, att.get>().data()); + break; + case DT::VEC_LONGLONG: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_UCHAR: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_SCHAR: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_USHORT: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_UINT: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_ULONG: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_ULONGLONG: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_FLOAT: + status = H5Awrite( + attribute_id, dataType, att.get>().data()); + break; + case DT::VEC_DOUBLE: + status = H5Awrite( + attribute_id, dataType, att.get>().data()); + break; + case DT::VEC_LONG_DOUBLE: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::VEC_CFLOAT: + status = H5Awrite( + attribute_id, + dataType, + att.get>>().data()); + break; + case DT::VEC_CDOUBLE: + status = H5Awrite( + attribute_id, + dataType, + att.get>>().data()); + break; + case DT::VEC_CLONG_DOUBLE: + status = H5Awrite( + attribute_id, + dataType, + att.get>>().data()); + break; + case DT::VEC_STRING: { + auto vs = att.get>(); + size_t max_len = 0; + for (std::string const &s : vs) + max_len = std::max(max_len, s.size() + 1); + std::unique_ptr c_str(new char[max_len * vs.size()]()); + for (size_t i = 0; i < vs.size(); ++i) + strncpy(c_str.get() + i * max_len, vs[i].c_str(), max_len); + status = H5Awrite(attribute_id, dataType, c_str.get()); + break; + } + case DT::ARR_DBL_7: + status = H5Awrite( + attribute_id, + dataType, + att.get>().data()); + break; + case DT::BOOL: { + bool b = att.get(); + status = H5Awrite(attribute_id, dataType, &b); + break; + } + case DT::UNDEFINED: + default: + throw std::runtime_error( + "[HDF5] Datatype not implemented in HDF5 IO"); + } + VERIFY( + status == 0, + "[HDF5] Internal error: Failed to write attribute " + name + + " at " + concrete_h5_file_position(writable)); } - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to write attribute " + name + " at " + - concrete_h5_file_position(writable)); status = H5Tclose(dataType); VERIFY(