From 5086434e546dc98933013c7382d00ea0b13453df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franz=20P=C3=B6schel?= <franz.poeschel@gmail.com>
Date: Wed, 20 Dec 2023 16:58:16 +0100
Subject: [PATCH] WIP: Implement attribute_writing_ranks in HDF5

---
 examples/5_write_parallel.cpp |  19 +-
 src/IO/HDF5/HDF5IOHandler.cpp | 446 ++++++++++++++++++----------------
 2 files changed, 254 insertions(+), 211 deletions(-)

diff --git a/examples/5_write_parallel.cpp b/examples/5_write_parallel.cpp
index 8587175fe7..737f5f734e 100644
--- a/examples/5_write_parallel.cpp
+++ b/examples/5_write_parallel.cpp
@@ -20,10 +20,13 @@
  */
 #include <openPMD/openPMD.hpp>
 
+#include <openPMD/auxiliary/Environment.hpp>
+
 #include <mpi.h>
 
 #include <iostream>
 #include <memory>
+#include <sstream>
 #include <vector> // std::vector
 
 using std::cout;
@@ -49,7 +52,21 @@ int main(int argc, char *argv[])
 
     // open file for writing
     Series series = Series(
-        "../samples/5_parallel_write.h5", Access::CREATE, MPI_COMM_WORLD);
+        "../samples/5_parallel_write.h5",
+        Access::CREATE,
+        MPI_COMM_WORLD
+#if 1
+        ,
+        []() {
+            std::stringstream res;
+            res << "attribute_writing_ranks = ["
+                << auxiliary::getEnvString("RANKS", "0") << "]";
+            auto res_str = res.str();
+            std::cout << "RETURNING: '" << res_str << "'" << std::endl;
+            return res_str;
+        }()
+#endif
+    );
     if (0 == mpi_rank)
         cout << "Created an empty series in parallel with " << mpi_size
              << " MPI ranks\n";
diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp
index e2fa63d9b3..e801a7f2d4 100644
--- a/src/IO/HDF5/HDF5IOHandler.cpp
+++ b/src/IO/HDF5/HDF5IOHandler.cpp
@@ -21,6 +21,7 @@
 #include "openPMD/IO/HDF5/HDF5IOHandler.hpp"
 #include "openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp"
 #include "openPMD/auxiliary/Environment.hpp"
+#include <type_traits>
 
 #if openPMD_HAVE_HDF5
 #include "openPMD/Datatype.hpp"
@@ -1586,214 +1587,239 @@ void HDF5IOHandlerImpl::writeAttribute(
             "attribute write");
     }
 
-    using DT = Datatype;
-    switch (dtype)
+    if (m_writeAttributesFromThisRank)
     {
-    case DT::CHAR: {
-        char c = att.get<char>();
-        status = H5Awrite(attribute_id, dataType, &c);
-        break;
-    }
-    case DT::UCHAR: {
-        auto u = att.get<unsigned char>();
-        status = H5Awrite(attribute_id, dataType, &u);
-        break;
-    }
-    case DT::SCHAR: {
-        auto u = att.get<signed char>();
-        status = H5Awrite(attribute_id, dataType, &u);
-        break;
-    }
-    case DT::SHORT: {
-        auto i = att.get<short>();
-        status = H5Awrite(attribute_id, dataType, &i);
-        break;
-    }
-    case DT::INT: {
-        int i = att.get<int>();
-        status = H5Awrite(attribute_id, dataType, &i);
-        break;
-    }
-    case DT::LONG: {
-        long i = att.get<long>();
-        status = H5Awrite(attribute_id, dataType, &i);
-        break;
-    }
-    case DT::LONGLONG: {
-        auto i = att.get<long long>();
-        status = H5Awrite(attribute_id, dataType, &i);
-        break;
-    }
-    case DT::USHORT: {
-        auto u = att.get<unsigned short>();
-        status = H5Awrite(attribute_id, dataType, &u);
-        break;
-    }
-    case DT::UINT: {
-        auto u = att.get<unsigned int>();
-        status = H5Awrite(attribute_id, dataType, &u);
-        break;
-    }
-    case DT::ULONG: {
-        auto u = att.get<unsigned long>();
-        status = H5Awrite(attribute_id, dataType, &u);
-        break;
-    }
-    case DT::ULONGLONG: {
-        auto u = att.get<unsigned long long>();
-        status = H5Awrite(attribute_id, dataType, &u);
-        break;
-    }
-    case DT::FLOAT: {
-        auto f = att.get<float>();
-        status = H5Awrite(attribute_id, dataType, &f);
-        break;
-    }
-    case DT::DOUBLE: {
-        auto d = att.get<double>();
-        status = H5Awrite(attribute_id, dataType, &d);
-        break;
-    }
-    case DT::LONG_DOUBLE: {
-        auto d = att.get<long double>();
-        status = H5Awrite(attribute_id, dataType, &d);
-        break;
-    }
-    case DT::CFLOAT: {
-        std::complex<float> f = att.get<std::complex<float> >();
-        status = H5Awrite(attribute_id, dataType, &f);
-        break;
-    }
-    case DT::CDOUBLE: {
-        std::complex<double> d = att.get<std::complex<double> >();
-        status = H5Awrite(attribute_id, dataType, &d);
-        break;
-    }
-    case DT::CLONG_DOUBLE: {
-        std::complex<long double> d = att.get<std::complex<long double> >();
-        status = H5Awrite(attribute_id, dataType, &d);
-        break;
-    }
-    case DT::STRING:
-        status =
-            H5Awrite(attribute_id, dataType, att.get<std::string>().c_str());
-        break;
-    case DT::VEC_CHAR:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::vector<char> >().data());
-        break;
-    case DT::VEC_SHORT:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::vector<short> >().data());
-        break;
-    case DT::VEC_INT:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::vector<int> >().data());
-        break;
-    case DT::VEC_LONG:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::vector<long> >().data());
-        break;
-    case DT::VEC_LONGLONG:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::vector<long long> >().data());
-        break;
-    case DT::VEC_UCHAR:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<unsigned char> >().data());
-        break;
-    case DT::VEC_SCHAR:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<signed char> >().data());
-        break;
-    case DT::VEC_USHORT:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<unsigned short> >().data());
-        break;
-    case DT::VEC_UINT:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<unsigned int> >().data());
-        break;
-    case DT::VEC_ULONG:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<unsigned long> >().data());
-        break;
-    case DT::VEC_ULONGLONG:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<unsigned long long> >().data());
-        break;
-    case DT::VEC_FLOAT:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::vector<float> >().data());
-        break;
-    case DT::VEC_DOUBLE:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::vector<double> >().data());
-        break;
-    case DT::VEC_LONG_DOUBLE:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<long double> >().data());
-        break;
-    case DT::VEC_CFLOAT:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<std::complex<float> > >().data());
-        break;
-    case DT::VEC_CDOUBLE:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<std::complex<double> > >().data());
-        break;
-    case DT::VEC_CLONG_DOUBLE:
-        status = H5Awrite(
-            attribute_id,
-            dataType,
-            att.get<std::vector<std::complex<long double> > >().data());
-        break;
-    case DT::VEC_STRING: {
-        auto vs = att.get<std::vector<std::string> >();
-        size_t max_len = 0;
-        for (std::string const &s : vs)
-            max_len = std::max(max_len, s.size() + 1);
-        std::unique_ptr<char[]> c_str(new char[max_len * vs.size()]());
-        for (size_t i = 0; i < vs.size(); ++i)
-            strncpy(c_str.get() + i * max_len, vs[i].c_str(), max_len);
-        status = H5Awrite(attribute_id, dataType, c_str.get());
-        break;
-    }
-    case DT::ARR_DBL_7:
-        status = H5Awrite(
-            attribute_id, dataType, att.get<std::array<double, 7> >().data());
-        break;
-    case DT::BOOL: {
-        bool b = att.get<bool>();
-        status = H5Awrite(attribute_id, dataType, &b);
-        break;
-    }
-    case DT::UNDEFINED:
-    default:
-        throw std::runtime_error("[HDF5] Datatype not implemented in HDF5 IO");
+        // std::cout << "Writing attribute " << parameters.name
+        //           << ", filling it with value ";
+        // std::visit(
+        //     [](auto const &val) {
+        //         using T =
+        //             std::remove_cv_t<std::remove_reference_t<decltype(val)>>;
+        //         if constexpr (
+        //             !auxiliary::IsVector_v<T> && !auxiliary::IsArray_v<T>)
+        //         {
+        //             std::cout << val;
+        //         } else
+        //         {
+        //             std::cout << "VECTOR/ARRAY";
+        //         }
+        //     },
+        //     att.getResource());
+        // std::cout << std::endl;
+        using DT = Datatype;
+        switch (dtype)
+        {
+        case DT::CHAR: {
+            char c = att.get<char>();
+            status = H5Awrite(attribute_id, dataType, &c);
+            break;
+        }
+        case DT::UCHAR: {
+            auto u = att.get<unsigned char>();
+            status = H5Awrite(attribute_id, dataType, &u);
+            break;
+        }
+        case DT::SCHAR: {
+            auto u = att.get<signed char>();
+            status = H5Awrite(attribute_id, dataType, &u);
+            break;
+        }
+        case DT::SHORT: {
+            auto i = att.get<short>();
+            status = H5Awrite(attribute_id, dataType, &i);
+            break;
+        }
+        case DT::INT: {
+            int i = att.get<int>();
+            status = H5Awrite(attribute_id, dataType, &i);
+            break;
+        }
+        case DT::LONG: {
+            long i = att.get<long>();
+            status = H5Awrite(attribute_id, dataType, &i);
+            break;
+        }
+        case DT::LONGLONG: {
+            auto i = att.get<long long>();
+            status = H5Awrite(attribute_id, dataType, &i);
+            break;
+        }
+        case DT::USHORT: {
+            auto u = att.get<unsigned short>();
+            status = H5Awrite(attribute_id, dataType, &u);
+            break;
+        }
+        case DT::UINT: {
+            auto u = att.get<unsigned int>();
+            status = H5Awrite(attribute_id, dataType, &u);
+            break;
+        }
+        case DT::ULONG: {
+            auto u = att.get<unsigned long>();
+            status = H5Awrite(attribute_id, dataType, &u);
+            break;
+        }
+        case DT::ULONGLONG: {
+            auto u = att.get<unsigned long long>();
+            status = H5Awrite(attribute_id, dataType, &u);
+            break;
+        }
+        case DT::FLOAT: {
+            auto f = att.get<float>();
+            status = H5Awrite(attribute_id, dataType, &f);
+            break;
+        }
+        case DT::DOUBLE: {
+            auto d = att.get<double>();
+            status = H5Awrite(attribute_id, dataType, &d);
+            break;
+        }
+        case DT::LONG_DOUBLE: {
+            auto d = att.get<long double>();
+            status = H5Awrite(attribute_id, dataType, &d);
+            break;
+        }
+        case DT::CFLOAT: {
+            std::complex<float> f = att.get<std::complex<float>>();
+            status = H5Awrite(attribute_id, dataType, &f);
+            break;
+        }
+        case DT::CDOUBLE: {
+            std::complex<double> d = att.get<std::complex<double>>();
+            status = H5Awrite(attribute_id, dataType, &d);
+            break;
+        }
+        case DT::CLONG_DOUBLE: {
+            std::complex<long double> d = att.get<std::complex<long double>>();
+            status = H5Awrite(attribute_id, dataType, &d);
+            break;
+        }
+        case DT::STRING:
+            status = H5Awrite(
+                attribute_id, dataType, att.get<std::string>().c_str());
+            break;
+        case DT::VEC_CHAR:
+            status = H5Awrite(
+                attribute_id, dataType, att.get<std::vector<char>>().data());
+            break;
+        case DT::VEC_SHORT:
+            status = H5Awrite(
+                attribute_id, dataType, att.get<std::vector<short>>().data());
+            break;
+        case DT::VEC_INT:
+            status = H5Awrite(
+                attribute_id, dataType, att.get<std::vector<int>>().data());
+            break;
+        case DT::VEC_LONG:
+            status = H5Awrite(
+                attribute_id, dataType, att.get<std::vector<long>>().data());
+            break;
+        case DT::VEC_LONGLONG:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<long long>>().data());
+            break;
+        case DT::VEC_UCHAR:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<unsigned char>>().data());
+            break;
+        case DT::VEC_SCHAR:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<signed char>>().data());
+            break;
+        case DT::VEC_USHORT:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<unsigned short>>().data());
+            break;
+        case DT::VEC_UINT:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<unsigned int>>().data());
+            break;
+        case DT::VEC_ULONG:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<unsigned long>>().data());
+            break;
+        case DT::VEC_ULONGLONG:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<unsigned long long>>().data());
+            break;
+        case DT::VEC_FLOAT:
+            status = H5Awrite(
+                attribute_id, dataType, att.get<std::vector<float>>().data());
+            break;
+        case DT::VEC_DOUBLE:
+            status = H5Awrite(
+                attribute_id, dataType, att.get<std::vector<double>>().data());
+            break;
+        case DT::VEC_LONG_DOUBLE:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<long double>>().data());
+            break;
+        case DT::VEC_CFLOAT:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<std::complex<float>>>().data());
+            break;
+        case DT::VEC_CDOUBLE:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<std::complex<double>>>().data());
+            break;
+        case DT::VEC_CLONG_DOUBLE:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::vector<std::complex<long double>>>().data());
+            break;
+        case DT::VEC_STRING: {
+            auto vs = att.get<std::vector<std::string>>();
+            size_t max_len = 0;
+            for (std::string const &s : vs)
+                max_len = std::max(max_len, s.size() + 1);
+            std::unique_ptr<char[]> c_str(new char[max_len * vs.size()]());
+            for (size_t i = 0; i < vs.size(); ++i)
+                strncpy(c_str.get() + i * max_len, vs[i].c_str(), max_len);
+            status = H5Awrite(attribute_id, dataType, c_str.get());
+            break;
+        }
+        case DT::ARR_DBL_7:
+            status = H5Awrite(
+                attribute_id,
+                dataType,
+                att.get<std::array<double, 7>>().data());
+            break;
+        case DT::BOOL: {
+            bool b = att.get<bool>();
+            status = H5Awrite(attribute_id, dataType, &b);
+            break;
+        }
+        case DT::UNDEFINED:
+        default:
+            throw std::runtime_error(
+                "[HDF5] Datatype not implemented in HDF5 IO");
+        }
+        VERIFY(
+            status == 0,
+            "[HDF5] Internal error: Failed to write attribute " + name +
+                " at " + concrete_h5_file_position(writable));
     }
-    VERIFY(
-        status == 0,
-        "[HDF5] Internal error: Failed to write attribute " + name + " at " +
-            concrete_h5_file_position(writable));
 
     status = H5Tclose(dataType);
     VERIFY(
@@ -2385,19 +2411,19 @@ void HDF5IOHandlerImpl::readAttribute(
         }
         else if (H5Tequal(attr_type, m_H5T_CFLOAT))
         {
-            std::vector<std::complex<float> > vcf(dims[0], 0);
+            std::vector<std::complex<float>> vcf(dims[0], 0);
             status = H5Aread(attr_id, attr_type, vcf.data());
             a = Attribute(vcf);
         }
         else if (H5Tequal(attr_type, m_H5T_CDOUBLE))
         {
-            std::vector<std::complex<double> > vcd(dims[0], 0);
+            std::vector<std::complex<double>> vcd(dims[0], 0);
             status = H5Aread(attr_id, attr_type, vcd.data());
             a = Attribute(vcd);
         }
         else if (H5Tequal(attr_type, m_H5T_CLONG_DOUBLE))
         {
-            std::vector<std::complex<long double> > vcld(dims[0], 0);
+            std::vector<std::complex<long double>> vcld(dims[0], 0);
             status = H5Aread(attr_id, attr_type, vcld.data());
             a = Attribute(vcld);
         }
@@ -2418,7 +2444,7 @@ void HDF5IOHandlerImpl::readAttribute(
                 tmpBuffer,
                 nullptr,
                 H5P_DEFAULT);
-            std::vector<std::complex<long double> > vcld{
+            std::vector<std::complex<long double>> vcld{
                 tmpBuffer, tmpBuffer + dims[0]};
             delete[] tmpBuffer;
             a = Attribute(std::move(vcld));