Skip to content

Commit

Permalink
wip: use variable-based encoding in tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@franzpoeschel
franzpoeschel committed Jan 15, 2025
1 parent 7372467 commit 32221d3
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 51 deletions.
32 changes: 27 additions & 5 deletions src/Series.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2144,10 +2144,29 @@ creating new iterations.
case IterationEncoding::variableBased: {
if (!currentSteps.has_value() || currentSteps.value().empty())
{
currentSteps = std::vector<IterationIndex_t>{
read_only_this_single_iteration.has_value()
? *read_only_this_single_iteration
: 0};
if (!read_only_this_single_iteration.has_value())
{
Parameter<Operation::LIST_DATASETS> ld;
Parameter<Operation::LIST_PATHS> lp;
IOHandler()->enqueue(IOTask(&iterations, ld));
IOHandler()->enqueue(IOTask(&iterations, lp));
IOHandler()->flush(internal::defaultFlushParams);
if (ld.datasets->empty() && lp.paths->empty())
{
return {}; // no iterations, just global attributes
}
else
{
// there is data, defaulting to calling this Iteration idx 0
// when no further info is available
currentSteps = std::vector<IterationIndex_t>{0};
}
}
else
{
currentSteps = std::vector<IterationIndex_t>{
*read_only_this_single_iteration};
}
}
else if (read_only_this_single_iteration.has_value())
{
Expand Down Expand Up @@ -3242,7 +3261,10 @@ Series::snapshots(std::optional<SnapshotWorkflow> const snapshot_workflow)
access::writeOnly(access) &&
// 5. The backend is ADIOS2 in a recent enough version to support
// modifiable attributes (v2.9).
IOHandler()->fullSupportForVariableBasedEncoding())
IOHandler()->fullSupportForVariableBasedEncoding() &&
// 6. The Series must not yet be written, otherwise we're too late
// for this
!this->written())
{
setIterationEncoding_internal(
IterationEncoding::variableBased,
Expand Down
15 changes: 8 additions & 7 deletions test/ParallelIOTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ TEST_CASE("parallel_multi_series_test", "[parallel]")
.append(".")
.append(file_ending),
Access::CREATE,
MPI_COMM_WORLD);
MPI_COMM_WORLD,
R"(iteration_encoding = "variable_based")");
allSeries.back().iterations[sn].setAttribute("wululu", sn);
allSeries.back().flush();
}
Expand Down Expand Up @@ -110,7 +111,7 @@ void write_test_zero_extent(

for (int step = 0; step <= max_step; step += 20)
{
Iteration it = o.iterations[step];
Iteration it = o.writeIterations()[step];
it.setAttribute("yolo", "yo");

if (rank != 0 || declareFromAll)
Expand Down Expand Up @@ -467,7 +468,7 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig)

// array record component -> array record component
// should work
auto E_x = write.iterations[0].meshes["E"]["x"];
auto E_x = write.writeIterations()[0].meshes["E"]["x"];
E_x.resetDataset(ds1);
E_x.storeChunk(data1, {mpi_rank, 0}, {1, 25});
write.flush();
Expand Down Expand Up @@ -518,7 +519,7 @@ TEST_CASE("adios_write_test", "[parallel][adios]")
auto mpi_rank = static_cast<uint64_t>(rank);

o.setAuthor("Parallel ADIOS2");
ParticleSpecies &e = o.iterations[1].particles["e"];
ParticleSpecies &e = o.writeIterations()[1].particles["e"];

std::vector<double> position_global(mpi_size);
double pos{0.};
Expand Down Expand Up @@ -673,7 +674,7 @@ void write_4D_test(std::string const &file_ending)
std::string name = "../samples/parallel_write_4d." + file_ending;
Series o = Series(name, Access::CREATE, MPI_COMM_WORLD);

auto it = o.iterations[1];
auto it = o.writeIterations()[1];
auto E_x = it.meshes["E"]["x"];

// every rank out of mpi_size MPI ranks contributes two writes:
Expand Down Expand Up @@ -708,7 +709,7 @@ void write_makeconst_some(std::string const &file_ending)
std::cout << name << std::endl;
Series o = Series(name, Access::CREATE, MPI_COMM_WORLD);

auto it = o.iterations[1];
auto it = o.writeIterations()[1];
// I would have expected we need this, since the first call that writes
// data below (makeConstant) is not executed in MPI collective manner
// it.open();
Expand Down Expand Up @@ -1138,7 +1139,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]")
int size, rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
auto iteration = write.iterations[0];
auto iteration = write.writeIterations()[0];
auto E_x = iteration.meshes["E"]["x"];
E_x.resetDataset({Datatype::DOUBLE, {10}});
write.flush();
Expand Down
97 changes: 58 additions & 39 deletions test/SerialIOTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ namespace detail
template <typename Char>
void writeChar(Series &series, std::string const &component_name)
{
auto component = series.iterations[0].meshes["E"][component_name];
auto component = series.writeIterations()[0].meshes["E"][component_name];
std::vector<Char> data(10);
component.resetDataset({determineDatatype<Char>(), {10}});
component.storeChunk(data, {0}, {10});
Expand Down Expand Up @@ -288,7 +288,7 @@ TEST_CASE("multi_series_test", "[serial]")
.append(".")
.append(file_ending),
Access::CREATE);
allSeries.back().iterations[sn].setAttribute("wululu", sn);
allSeries.back().writeIterations()[sn].setAttribute("wululu", sn);
allSeries.back().flush();
}
}
Expand Down Expand Up @@ -647,12 +647,13 @@ void close_and_copy_attributable_test(std::string const &file_ending)
{
if (iteration_ptr)
{
*iteration_ptr = series.iterations[i];
*iteration_ptr = series.writeIterations()[i];
}
else
{
// use copy constructor
iteration_ptr = std::make_unique<Iteration>(series.iterations[i]);
iteration_ptr =
std::make_unique<Iteration>(series.writeIterations()[i]);
}
Record electronPositions = iteration_ptr->particles["e"]["position"];
// TODO set this automatically to zero if not provided
Expand Down Expand Up @@ -784,17 +785,20 @@ inline void empty_dataset_test(std::string const &file_ending)
"../samples/empty_datasets." + file_ending, Access::CREATE);

auto makeEmpty_dim_7_int =
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int"];
series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_int"];
auto makeEmpty_dim_7_long =
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool"];
series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_bool"];
auto makeEmpty_dim_7_int_alt =
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int_alt"];
series.writeIterations()[1]
.meshes["rho"]["makeEmpty_dim_7_int_alt"];
auto makeEmpty_dim_7_long_alt =
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool_alt"];
series.writeIterations()[1]
.meshes["rho"]["makeEmpty_dim_7_bool_alt"];
auto makeEmpty_resetDataset_dim3 =
series.iterations[1].meshes["rho"]["makeEmpty_resetDataset_dim3"];
series.writeIterations()[1]
.meshes["rho"]["makeEmpty_resetDataset_dim3"];
auto makeEmpty_resetDataset_dim3_notallzero =
series.iterations[1]
series.writeIterations()[1]
.meshes["rho"]["makeEmpty_resetDataset_dim3_notallzero"];
makeEmpty_dim_7_int.makeEmpty<int>(7);
makeEmpty_dim_7_long.makeEmpty<long>(7);
Expand Down Expand Up @@ -899,17 +903,18 @@ inline void constant_scalar(std::string const &file_ending)
Series s =
Series("../samples/constant_scalar." + file_ending, Access::CREATE);
s.setOpenPMD("2.0.0");
auto rho = s.iterations[1].meshes["rho"][MeshRecordComponent::SCALAR];
REQUIRE(s.iterations[1].meshes["rho"].scalar());
auto rho =
s.writeIterations()[1].meshes["rho"][MeshRecordComponent::SCALAR];
REQUIRE(s.writeIterations()[1].meshes["rho"].scalar());
rho.resetDataset(Dataset(Datatype::CHAR, {1, 2, 3}));
rho.makeConstant(static_cast<char>('a'));
REQUIRE(rho.constant());

// mixed constant/non-constant
auto E_x = s.iterations[1].meshes["E"]["x"];
auto E_x = s.writeIterations()[1].meshes["E"]["x"];
E_x.resetDataset(Dataset(Datatype::FLOAT, {1, 2, 3}));
E_x.makeConstant(static_cast<float>(13.37));
auto E_y = s.iterations[1].meshes["E"]["y"];
auto E_y = s.writeIterations()[1].meshes["E"]["y"];
E_y.resetDataset(Dataset(Datatype::UINT, {1, 2, 3}));
UniquePtrWithLambda<unsigned int> E(
new unsigned int[6], [](unsigned int const *p) { delete[] p; });
Expand All @@ -918,7 +923,7 @@ inline void constant_scalar(std::string const &file_ending)
E_y.storeChunk(std::move(E), {0, 0, 0}, {1, 2, 3});

// store a number of predefined attributes in E
Mesh &E_mesh = s.iterations[1].meshes["E"];
Mesh &E_mesh = s.writeIterations()[1].meshes["E"];
// test that these can be defined successively
E_mesh.setGridUnitDimension({{{UnitDimension::L, 1}}, {}, {}});
E_mesh.setGridUnitDimension(
Expand All @@ -943,21 +948,21 @@ inline void constant_scalar(std::string const &file_ending)
std::vector<double>{gridUnitSI, gridUnitSI, gridUnitSI});

// constant scalar
auto pos =
s.iterations[1].particles["e"]["position"][RecordComponent::SCALAR];
auto pos = s.writeIterations()[1]
.particles["e"]["position"][RecordComponent::SCALAR];
pos.resetDataset(Dataset(Datatype::DOUBLE, {3, 2, 1}));
pos.makeConstant(static_cast<double>(42.));
auto posOff =
s.iterations[1]
s.writeIterations()[1]
.particles["e"]["positionOffset"][RecordComponent::SCALAR];
posOff.resetDataset(Dataset(Datatype::INT, {3, 2, 1}));
posOff.makeConstant(static_cast<int>(-42));

// mixed constant/non-constant
auto vel_x = s.iterations[1].particles["e"]["velocity"]["x"];
auto vel_x = s.writeIterations()[1].particles["e"]["velocity"]["x"];
vel_x.resetDataset(Dataset(Datatype::SHORT, {3, 2, 1}));
vel_x.makeConstant(static_cast<short>(-1));
auto vel_y = s.iterations[1].particles["e"]["velocity"]["y"];
auto vel_y = s.writeIterations()[1].particles["e"]["velocity"]["y"];
vel_y.resetDataset(Dataset(Datatype::ULONGLONG, {3, 2, 1}));
UniquePtrWithLambda<unsigned long long> vel(
new unsigned long long[6],
Expand Down Expand Up @@ -1146,7 +1151,7 @@ TEST_CASE("flush_without_position_positionOffset", "[serial]")
Series s = Series(
"../samples/flush_without_position_positionOffset." + file_ending,
Access::CREATE);
ParticleSpecies e = s.iterations[0].particles["e"];
ParticleSpecies e = s.writeIterations()[0].particles["e"];
RecordComponent weighting = e["weighting"][RecordComponent::SCALAR];
weighting.resetDataset(Dataset(Datatype::FLOAT, Extent{2, 2}));
weighting.storeChunk(
Expand Down Expand Up @@ -1407,7 +1412,7 @@ inline void dtype_test(
// should be possible to parse without error upon opening
// the series for reading
{
auto E = s.iterations[0].meshes["E"];
auto E = s.writeIterations()[0].meshes["E"];
E.setGridSpacing(std::vector<long double>{1.0, 1.0});
auto E_x = E["x"];
E_x.makeEmpty<double>(1);
Expand Down Expand Up @@ -1622,7 +1627,7 @@ inline void write_test(
Series o =
Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg);

ParticleSpecies &e_1 = o.iterations[1].particles["e"];
ParticleSpecies &e_1 = o.writeIterations()[1].particles["e"];

std::vector<double> position_global(4);
double pos{0.};
Expand Down Expand Up @@ -1657,7 +1662,7 @@ inline void write_test(
e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1});
}

ParticleSpecies &e_2 = o.iterations[2].particles["e"];
ParticleSpecies &e_2 = o.writeIterations()[2].particles["e"];

std::generate(position_global.begin(), position_global.end(), [&pos] {
return pos++;
Expand Down Expand Up @@ -1688,7 +1693,7 @@ inline void write_test(

o.flush();

ParticleSpecies &e_3 = o.iterations[3].particles["e"];
ParticleSpecies &e_3 = o.writeIterations()[3].particles["e"];

std::generate(position_global.begin(), position_global.end(), [&pos] {
return pos++;
Expand Down Expand Up @@ -1797,15 +1802,17 @@ void test_complex(const std::string &backend)
o.setAttribute(
"longDoublesYouSay", std::complex<long double>(5.5, -4.55));

auto Cflt = o.iterations[0].meshes["Cflt"][RecordComponent::SCALAR];
auto Cflt =
o.writeIterations()[0].meshes["Cflt"][RecordComponent::SCALAR];
std::vector<std::complex<float>> cfloats(3);
cfloats.at(0) = {1., 2.};
cfloats.at(1) = {-3., 4.};
cfloats.at(2) = {5., -6.};
Cflt.resetDataset(Dataset(Datatype::CFLOAT, {cfloats.size()}));
Cflt.storeChunk(cfloats, {0});

auto Cdbl = o.iterations[0].meshes["Cdbl"][RecordComponent::SCALAR];
auto Cdbl =
o.writeIterations()[0].meshes["Cdbl"][RecordComponent::SCALAR];
std::vector<std::complex<double>> cdoubles(3);
cdoubles.at(0) = {2., 1.};
cdoubles.at(1) = {-4., 3.};
Expand All @@ -1817,7 +1824,7 @@ void test_complex(const std::string &backend)
if (o.backend() != "ADIOS2")
{
auto Cldbl =
o.iterations[0].meshes["Cldbl"][RecordComponent::SCALAR];
o.writeIterations()[0].meshes["Cldbl"][RecordComponent::SCALAR];
cldoubles.at(0) = {3., 2.};
cldoubles.at(1) = {-5., 4.};
cldoubles.at(2) = {7., -6.};
Expand Down Expand Up @@ -2427,7 +2434,10 @@ TEST_CASE("sample_write_thetaMode", "[serial][thetaMode]")
inline void bool_test(const std::string &backend)
{
{
Series o = Series("../samples/serial_bool." + backend, Access::CREATE);
Series o = Series(
"../samples/serial_bool." + backend,
Access::CREATE,
R"({"iteration_encoding": "variable_based"})");

o.setAttribute("Bool attribute true", true);
o.setAttribute("Bool attribute false", false);
Expand Down Expand Up @@ -2463,7 +2473,7 @@ inline void patch_test(const std::string &backend)
{
Series o = Series("../samples/serial_patch." + backend, Access::CREATE);

auto e = o.iterations[1].particles["e"];
auto e = o.writeIterations()[1].particles["e"];

uint64_t const num_particles = 1u;
auto dset_d = Dataset(Datatype::DOUBLE, {num_particles});
Expand Down Expand Up @@ -4647,7 +4657,12 @@ TEST_CASE("adios2_engines_and_file_endings")
Series write(
name,
Access::CREATE,
json::merge("backend = \"adios2\"", jsonCfg));
json::merge(
R"(
backend = "adios2"
iteration_encoding = "variable_based"
)",
jsonCfg));
}
if (directory)
{
Expand Down Expand Up @@ -4712,7 +4727,11 @@ TEST_CASE("adios2_engines_and_file_endings")
auto filesystemname =
filesystemExt.empty() ? name : basename + filesystemExt;
{
Series write(name, Access::CREATE, jsonCfg);
Series write(
name,
Access::CREATE,
json::merge(
R"(iteration_encoding = "variable_based")", jsonCfg));
write.close();
}
if (directory)
Expand Down Expand Up @@ -6118,8 +6137,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]")
// automatically (de)activate span-based storeChunking
{
Series write("../samples/span_based.bp", Access::CREATE);
auto E_uncompressed = write.iterations[0].meshes["E"]["x"];
auto E_compressed = write.iterations[0].meshes["E"]["y"];
auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"];
auto E_compressed = write.writeIterations()[0].meshes["E"]["y"];

Dataset ds{Datatype::INT, {10}};

Expand Down Expand Up @@ -6169,8 +6188,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]")
}
})END";
Series write("../samples/span_based.bp", Access::CREATE, enable);
auto E_uncompressed = write.iterations[0].meshes["E"]["x"];
auto E_compressed = write.iterations[0].meshes["E"]["y"];
auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"];
auto E_compressed = write.writeIterations()[0].meshes["E"]["y"];

Dataset ds{Datatype::INT, {10}};

Expand Down Expand Up @@ -6233,8 +6252,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]")
}
})END";
Series write("../samples/span_based.bp", Access::CREATE, disable);
auto E_uncompressed = write.iterations[0].meshes["E"]["x"];
auto E_compressed = write.iterations[0].meshes["E"]["y"];
auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"];
auto E_compressed = write.writeIterations()[0].meshes["E"]["y"];

Dataset ds{Datatype::INT, {10}};

Expand Down Expand Up @@ -6689,7 +6708,7 @@ void deferred_parsing(std::string const &extension)
Series series(basename + "%06T." + extension, Access::CREATE);
std::vector<float> buffer(20);
std::iota(buffer.begin(), buffer.end(), 0.f);
auto dataset = series.iterations[0].meshes["E"]["x"];
auto dataset = series.writeIterations()[0].meshes["E"]["x"];
dataset.resetDataset({Datatype::FLOAT, {20}});
dataset.storeChunk(buffer, {0}, {20});
series.flush();
Expand Down

0 comments on commit 32221d3

Please sign in to comment.