diff --git a/src/Series.cpp b/src/Series.cpp index c78d7e3c4d..ca3da5fb9b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2144,10 +2144,29 @@ creating new iterations. case IterationEncoding::variableBased: { if (!currentSteps.has_value() || currentSteps.value().empty()) { - currentSteps = std::vector{ - read_only_this_single_iteration.has_value() - ? *read_only_this_single_iteration - : 0}; + if (!read_only_this_single_iteration.has_value()) + { + Parameter ld; + Parameter lp; + IOHandler()->enqueue(IOTask(&iterations, ld)); + IOHandler()->enqueue(IOTask(&iterations, lp)); + IOHandler()->flush(internal::defaultFlushParams); + if (ld.datasets->empty() && lp.paths->empty()) + { + return {}; // no iterations, just global attributes + } + else + { + // there is data, defaulting to calling this Iteration idx 0 + // when no further info is available + currentSteps = std::vector{0}; + } + } + else + { + currentSteps = std::vector{ + *read_only_this_single_iteration}; + } } else if (read_only_this_single_iteration.has_value()) { @@ -3242,7 +3261,10 @@ Series::snapshots(std::optional const snapshot_workflow) access::writeOnly(access) && // 5. The backend is ADIOS2 in a recent enough version to support // modifiable attributes (v2.9). - IOHandler()->fullSupportForVariableBasedEncoding()) + IOHandler()->fullSupportForVariableBasedEncoding() && + // 6. The Series must not yet be written, otherwise we're too late + // for this + !this->written()) { setIterationEncoding_internal( IterationEncoding::variableBased, diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index a21e0c7adb..eb54c20333 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -57,7 +57,8 @@ TEST_CASE("parallel_multi_series_test", "[parallel]") .append(".") .append(file_ending), Access::CREATE, - MPI_COMM_WORLD); + MPI_COMM_WORLD, + R"(iteration_encoding = "variable_based")"); allSeries.back().iterations[sn].setAttribute("wululu", sn); allSeries.back().flush(); } @@ -110,7 +111,7 @@ void write_test_zero_extent( for (int step = 0; step <= max_step; step += 20) { - Iteration it = o.iterations[step]; + Iteration it = o.writeIterations()[step]; it.setAttribute("yolo", "yo"); if (rank != 0 || declareFromAll) @@ -467,7 +468,7 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) // array record component -> array record component // should work - auto E_x = write.iterations[0].meshes["E"]["x"]; + auto E_x = write.writeIterations()[0].meshes["E"]["x"]; E_x.resetDataset(ds1); E_x.storeChunk(data1, {mpi_rank, 0}, {1, 25}); write.flush(); @@ -518,7 +519,7 @@ TEST_CASE("adios_write_test", "[parallel][adios]") auto mpi_rank = static_cast(rank); o.setAuthor("Parallel ADIOS2"); - ParticleSpecies &e = o.iterations[1].particles["e"]; + ParticleSpecies &e = o.writeIterations()[1].particles["e"]; std::vector position_global(mpi_size); double pos{0.}; @@ -673,7 +674,7 @@ void write_4D_test(std::string const &file_ending) std::string name = "../samples/parallel_write_4d." + file_ending; Series o = Series(name, Access::CREATE, MPI_COMM_WORLD); - auto it = o.iterations[1]; + auto it = o.writeIterations()[1]; auto E_x = it.meshes["E"]["x"]; // every rank out of mpi_size MPI ranks contributes two writes: @@ -708,7 +709,7 @@ void write_makeconst_some(std::string const &file_ending) std::cout << name << std::endl; Series o = Series(name, Access::CREATE, MPI_COMM_WORLD); - auto it = o.iterations[1]; + auto it = o.writeIterations()[1]; // I would have expected we need this, since the first call that writes // data below (makeConstant) is not executed in MPI collective manner // it.open(); @@ -1138,7 +1139,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]") int size, rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - auto iteration = write.iterations[0]; + auto iteration = write.writeIterations()[0]; auto E_x = iteration.meshes["E"]["x"]; E_x.resetDataset({Datatype::DOUBLE, {10}}); write.flush(); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 650d425209..581b8aa298 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -131,7 +131,7 @@ namespace detail template void writeChar(Series &series, std::string const &component_name) { - auto component = series.iterations[0].meshes["E"][component_name]; + auto component = series.writeIterations()[0].meshes["E"][component_name]; std::vector data(10); component.resetDataset({determineDatatype(), {10}}); component.storeChunk(data, {0}, {10}); @@ -288,7 +288,7 @@ TEST_CASE("multi_series_test", "[serial]") .append(".") .append(file_ending), Access::CREATE); - allSeries.back().iterations[sn].setAttribute("wululu", sn); + allSeries.back().writeIterations()[sn].setAttribute("wululu", sn); allSeries.back().flush(); } } @@ -647,12 +647,13 @@ void close_and_copy_attributable_test(std::string const &file_ending) { if (iteration_ptr) { - *iteration_ptr = series.iterations[i]; + *iteration_ptr = series.writeIterations()[i]; } else { // use copy constructor - iteration_ptr = std::make_unique(series.iterations[i]); + iteration_ptr = + std::make_unique(series.writeIterations()[i]); } Record electronPositions = iteration_ptr->particles["e"]["position"]; // TODO set this automatically to zero if not provided @@ -784,17 +785,20 @@ inline void empty_dataset_test(std::string const &file_ending) "../samples/empty_datasets." + file_ending, Access::CREATE); auto makeEmpty_dim_7_int = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int"]; + series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_int"]; auto makeEmpty_dim_7_long = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool"]; + series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_bool"]; auto makeEmpty_dim_7_int_alt = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int_alt"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_dim_7_int_alt"]; auto makeEmpty_dim_7_long_alt = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool_alt"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_dim_7_bool_alt"]; auto makeEmpty_resetDataset_dim3 = - series.iterations[1].meshes["rho"]["makeEmpty_resetDataset_dim3"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_resetDataset_dim3"]; auto makeEmpty_resetDataset_dim3_notallzero = - series.iterations[1] + series.writeIterations()[1] .meshes["rho"]["makeEmpty_resetDataset_dim3_notallzero"]; makeEmpty_dim_7_int.makeEmpty(7); makeEmpty_dim_7_long.makeEmpty(7); @@ -899,17 +903,18 @@ inline void constant_scalar(std::string const &file_ending) Series s = Series("../samples/constant_scalar." + file_ending, Access::CREATE); s.setOpenPMD("2.0.0"); - auto rho = s.iterations[1].meshes["rho"][MeshRecordComponent::SCALAR]; - REQUIRE(s.iterations[1].meshes["rho"].scalar()); + auto rho = + s.writeIterations()[1].meshes["rho"][MeshRecordComponent::SCALAR]; + REQUIRE(s.writeIterations()[1].meshes["rho"].scalar()); rho.resetDataset(Dataset(Datatype::CHAR, {1, 2, 3})); rho.makeConstant(static_cast('a')); REQUIRE(rho.constant()); // mixed constant/non-constant - auto E_x = s.iterations[1].meshes["E"]["x"]; + auto E_x = s.writeIterations()[1].meshes["E"]["x"]; E_x.resetDataset(Dataset(Datatype::FLOAT, {1, 2, 3})); E_x.makeConstant(static_cast(13.37)); - auto E_y = s.iterations[1].meshes["E"]["y"]; + auto E_y = s.writeIterations()[1].meshes["E"]["y"]; E_y.resetDataset(Dataset(Datatype::UINT, {1, 2, 3})); UniquePtrWithLambda E( new unsigned int[6], [](unsigned int const *p) { delete[] p; }); @@ -918,7 +923,7 @@ inline void constant_scalar(std::string const &file_ending) E_y.storeChunk(std::move(E), {0, 0, 0}, {1, 2, 3}); // store a number of predefined attributes in E - Mesh &E_mesh = s.iterations[1].meshes["E"]; + Mesh &E_mesh = s.writeIterations()[1].meshes["E"]; // test that these can be defined successively E_mesh.setGridUnitDimension({{{UnitDimension::L, 1}}, {}, {}}); E_mesh.setGridUnitDimension( @@ -943,21 +948,21 @@ inline void constant_scalar(std::string const &file_ending) std::vector{gridUnitSI, gridUnitSI, gridUnitSI}); // constant scalar - auto pos = - s.iterations[1].particles["e"]["position"][RecordComponent::SCALAR]; + auto pos = s.writeIterations()[1] + .particles["e"]["position"][RecordComponent::SCALAR]; pos.resetDataset(Dataset(Datatype::DOUBLE, {3, 2, 1})); pos.makeConstant(static_cast(42.)); auto posOff = - s.iterations[1] + s.writeIterations()[1] .particles["e"]["positionOffset"][RecordComponent::SCALAR]; posOff.resetDataset(Dataset(Datatype::INT, {3, 2, 1})); posOff.makeConstant(static_cast(-42)); // mixed constant/non-constant - auto vel_x = s.iterations[1].particles["e"]["velocity"]["x"]; + auto vel_x = s.writeIterations()[1].particles["e"]["velocity"]["x"]; vel_x.resetDataset(Dataset(Datatype::SHORT, {3, 2, 1})); vel_x.makeConstant(static_cast(-1)); - auto vel_y = s.iterations[1].particles["e"]["velocity"]["y"]; + auto vel_y = s.writeIterations()[1].particles["e"]["velocity"]["y"]; vel_y.resetDataset(Dataset(Datatype::ULONGLONG, {3, 2, 1})); UniquePtrWithLambda vel( new unsigned long long[6], @@ -1146,7 +1151,7 @@ TEST_CASE("flush_without_position_positionOffset", "[serial]") Series s = Series( "../samples/flush_without_position_positionOffset." + file_ending, Access::CREATE); - ParticleSpecies e = s.iterations[0].particles["e"]; + ParticleSpecies e = s.writeIterations()[0].particles["e"]; RecordComponent weighting = e["weighting"][RecordComponent::SCALAR]; weighting.resetDataset(Dataset(Datatype::FLOAT, Extent{2, 2})); weighting.storeChunk( @@ -1407,7 +1412,7 @@ inline void dtype_test( // should be possible to parse without error upon opening // the series for reading { - auto E = s.iterations[0].meshes["E"]; + auto E = s.writeIterations()[0].meshes["E"]; E.setGridSpacing(std::vector{1.0, 1.0}); auto E_x = E["x"]; E_x.makeEmpty(1); @@ -1622,7 +1627,7 @@ inline void write_test( Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); - ParticleSpecies &e_1 = o.iterations[1].particles["e"]; + ParticleSpecies &e_1 = o.writeIterations()[1].particles["e"]; std::vector position_global(4); double pos{0.}; @@ -1657,7 +1662,7 @@ inline void write_test( e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); } - ParticleSpecies &e_2 = o.iterations[2].particles["e"]; + ParticleSpecies &e_2 = o.writeIterations()[2].particles["e"]; std::generate(position_global.begin(), position_global.end(), [&pos] { return pos++; @@ -1688,7 +1693,7 @@ inline void write_test( o.flush(); - ParticleSpecies &e_3 = o.iterations[3].particles["e"]; + ParticleSpecies &e_3 = o.writeIterations()[3].particles["e"]; std::generate(position_global.begin(), position_global.end(), [&pos] { return pos++; @@ -1797,7 +1802,8 @@ void test_complex(const std::string &backend) o.setAttribute( "longDoublesYouSay", std::complex(5.5, -4.55)); - auto Cflt = o.iterations[0].meshes["Cflt"][RecordComponent::SCALAR]; + auto Cflt = + o.writeIterations()[0].meshes["Cflt"][RecordComponent::SCALAR]; std::vector> cfloats(3); cfloats.at(0) = {1., 2.}; cfloats.at(1) = {-3., 4.}; @@ -1805,7 +1811,8 @@ void test_complex(const std::string &backend) Cflt.resetDataset(Dataset(Datatype::CFLOAT, {cfloats.size()})); Cflt.storeChunk(cfloats, {0}); - auto Cdbl = o.iterations[0].meshes["Cdbl"][RecordComponent::SCALAR]; + auto Cdbl = + o.writeIterations()[0].meshes["Cdbl"][RecordComponent::SCALAR]; std::vector> cdoubles(3); cdoubles.at(0) = {2., 1.}; cdoubles.at(1) = {-4., 3.}; @@ -1817,7 +1824,7 @@ void test_complex(const std::string &backend) if (o.backend() != "ADIOS2") { auto Cldbl = - o.iterations[0].meshes["Cldbl"][RecordComponent::SCALAR]; + o.writeIterations()[0].meshes["Cldbl"][RecordComponent::SCALAR]; cldoubles.at(0) = {3., 2.}; cldoubles.at(1) = {-5., 4.}; cldoubles.at(2) = {7., -6.}; @@ -2427,7 +2434,10 @@ TEST_CASE("sample_write_thetaMode", "[serial][thetaMode]") inline void bool_test(const std::string &backend) { { - Series o = Series("../samples/serial_bool." + backend, Access::CREATE); + Series o = Series( + "../samples/serial_bool." + backend, + Access::CREATE, + R"({"iteration_encoding": "variable_based"})"); o.setAttribute("Bool attribute true", true); o.setAttribute("Bool attribute false", false); @@ -2463,7 +2473,7 @@ inline void patch_test(const std::string &backend) { Series o = Series("../samples/serial_patch." + backend, Access::CREATE); - auto e = o.iterations[1].particles["e"]; + auto e = o.writeIterations()[1].particles["e"]; uint64_t const num_particles = 1u; auto dset_d = Dataset(Datatype::DOUBLE, {num_particles}); @@ -4647,7 +4657,12 @@ TEST_CASE("adios2_engines_and_file_endings") Series write( name, Access::CREATE, - json::merge("backend = \"adios2\"", jsonCfg)); + json::merge( + R"( + backend = "adios2" + iteration_encoding = "variable_based" + )", + jsonCfg)); } if (directory) { @@ -4712,7 +4727,11 @@ TEST_CASE("adios2_engines_and_file_endings") auto filesystemname = filesystemExt.empty() ? name : basename + filesystemExt; { - Series write(name, Access::CREATE, jsonCfg); + Series write( + name, + Access::CREATE, + json::merge( + R"(iteration_encoding = "variable_based")", jsonCfg)); write.close(); } if (directory) @@ -6118,8 +6137,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") // automatically (de)activate span-based storeChunking { Series write("../samples/span_based.bp", Access::CREATE); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6169,8 +6188,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") } })END"; Series write("../samples/span_based.bp", Access::CREATE, enable); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6233,8 +6252,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") } })END"; Series write("../samples/span_based.bp", Access::CREATE, disable); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6689,7 +6708,7 @@ void deferred_parsing(std::string const &extension) Series series(basename + "%06T." + extension, Access::CREATE); std::vector buffer(20); std::iota(buffer.begin(), buffer.end(), 0.f); - auto dataset = series.iterations[0].meshes["E"]["x"]; + auto dataset = series.writeIterations()[0].meshes["E"]["x"]; dataset.resetDataset({Datatype::FLOAT, {20}}); dataset.storeChunk(buffer, {0}, {20}); series.flush();