From 6cb26fa436a1ca6f50e12812ff785f8fd10fd6a0 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 23 Sep 2024 17:43:49 +0200 Subject: [PATCH 01/43] feature-scda: Start to write fixed-len array header --- src/sc_scda.c | 157 ++++++++++++++++++++++++++++++++++++++++++++++- test/test_scda.c | 28 +++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index ab66ad93..c4418118 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1736,7 +1736,7 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, /* TODO: respect encode parameter */ - /* file header section is always written and read on rank SC_SCDA_HEADER_ROOT */ + /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { sc_scda_fwrite_block_header_internal (fc, user_string, len, block_size, &count_err, errcode); @@ -1764,6 +1764,161 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, return fc; } +/** Internal function to write the array section header. + * + * This function is dedicated to be called in \ref sc_scda_fwrite_array. + * + * \param [in] fc The file context as in \ref sc_scda_fwrite_array + * before running the first serial code part. + * \param [in] user_string As in the documentation of \ref + * sc_scda_fwrite_array. + * \param [in] len As in the documentation of \ref + * sc_scda_fwrite_array. + * TODO + * \param [in] elem_size As in the documentation of \ref + * sc_scda_fwrite_array. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fwrite_array_header_internal (sc_scda_fcontext_t *fc, + const char *user_string, size_t *len, + size_t elem_count, size_t elem_size, + int *count_err, sc_scda_ferror_t *errcode) +{ + int mpiret; + int count; + int current_len; + int invalid_user_string, invalid_count; + int header_len; + char header_data[SC_SCDA_COMMON_FIELD + 2 * SC_SCDA_COUNT_FIELD]; + + *count_err = 0; + + header_len = SC_SCDA_COMMON_FIELD + 2 * SC_SCDA_COUNT_FIELD; + + /* get array file section header */ + + /* section-identifying character */ + current_len = 0; + + invalid_user_string = + sc_scda_get_common_section_header ('A', user_string, len, header_data); + /* We always translate the error code to have full coverage for the fuzzy + * error testing. + */ + sc_scda_scdaret_to_errcode (invalid_user_string ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid user string"); + + current_len += SC_SCDA_COMMON_FIELD; + + /* get count variable entry */ + /* element count */ + invalid_count = sc_scda_get_section_header_entry ('N', elem_count, + &header_data[current_len]); + sc_scda_scdaret_to_errcode (invalid_count ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid count"); + + current_len += SC_SCDA_COUNT_FIELD; + + /* element size */ + invalid_count = sc_scda_get_section_header_entry ('E', elem_size, + &header_data[current_len]); + sc_scda_scdaret_to_errcode (invalid_count ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid count"); + + current_len += SC_SCDA_COUNT_FIELD; + + SC_ASSERT (current_len == header_len); + + /* write array section header */ + mpiret = sc_io_write_at (fc->file, fc->accessed_bytes, header_data, + header_len, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing array section header"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (header_len, count, count_err); +} + +sc_scda_fcontext_t * +sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, + size_t *len, sc_array_t *array_data, + sc_array_t *elem_counts, size_t elem_size, int indirect, + int encode, sc_scda_ferror_t *errcode) +{ + int invalid_elem_counts; + int ret, count_err; + size_t elem_count, si; +#if 0 + const void *local_array_data; +#endif + + SC_ASSERT (fc != NULL); + SC_ASSERT (user_string != NULL); +#if 0 + SC_ASSERT (array_data != NULL); +#endif + SC_ASSERT (elem_counts != NULL); + SC_ASSERT (errcode != NULL); + + /* TODO: Also check elem_counts and/or indirect? */ + /* check if elem_size is collective */ + ret = sc_scda_check_coll_params (fc, (const char*) &elem_size, + sizeof (size_t), NULL, 0, NULL, 0); + sc_scda_scdaret_to_errcode (ret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_array: elem_size is not " + "collective"); + + /* TODO: respect encode parameter */ + + /* compute the global element count */ + /* TODO: only required on rank SC_SCDA_HEADER_ROOT */ + elem_count = 0; + for (si = 0; si < elem_counts->elem_count; ++si) { + elem_count += *((size_t *) sc_array_index (elem_counts, si)); + } + + /* check elem_counts array */ + invalid_elem_counts = !(elem_counts->elem_size == sizeof (sc_scda_ulong) && + elem_counts->elem_count == fc->mpisize); + sc_scda_scdaret_to_errcode (invalid_elem_counts ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Invalid elem_counts array"); + + /* TODO: check array_data; depends on indirect parameter */ + /* Call Alreduce to synchronize on check of array_data */ + + /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { + sc_scda_fwrite_array_header_internal (fc, user_string, len, elem_count, + elem_size, &count_err, errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); + +#if 0 + /* get pointer to local array data */ + if (indirect) { + /* indirect adressing */ + /* TODO: copy the data or use muliple write calls; maybe in batches? */ + } + else { + /* direct adressig */ + local_array_data = (const void *) array_data->array; + } + + /* write array data in parallel */ +#endif + + + return fc; +} + /** Check a read file header section and extract the user string. * * \param [in] file_header_data The read file header data as a byte buffer. diff --git a/test/test_scda.c b/test/test_scda.c index 0811b4fc..28789164 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -27,6 +27,31 @@ #define SC_SCDA_FILE_EXT "scd" #define SC_SCDA_TEST_FILE "sc_test_scda." SC_SCDA_FILE_EXT +static void +test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpisize) +{ + int indirect = 0; + int i; + size_t elem_size = 3; + sc_array_t elem_counts; + sc_scda_ferror_t errcode; + + sc_array_init_count (&elem_counts, sizeof (sc_scda_ulong), mpisize); + + /* set elem_counts */ + for (i = 0; i < mpisize; ++i) { + /* partition dependent */ + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 5; + } + + fc = sc_scda_fwrite_array (fc, "A fixed-len array section", NULL, NULL, + &elem_counts, elem_size, indirect, 0, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fwrite_array failed"); + + sc_array_reset (&elem_counts); +} + int main (int argc, char **argv) { @@ -188,6 +213,9 @@ main (int argc, char **argv) SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "scda_fwrite_block failed"); + /* write a fixed-size array section */ + test_scda_write_fixed_size_array (fc, mpisize); + /* intentionally try to write with non-collective block size */ if (mpisize > 1) { SC_GLOBAL_ESSENTIAL From dc618ff085b445f761d242690ae5dcebbb3d1c06 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 26 Sep 2024 14:19:56 +0200 Subject: [PATCH 02/43] feature-scda: Make test partition indepedent --- test/test_scda.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index 28789164..82680bcb 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -33,18 +33,26 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpisize) int indirect = 0; int i; size_t elem_size = 3; + const sc_scda_ulong global_elem_count = 12; + sc_scda_ulong per_proc_count, remainder_count; sc_array_t elem_counts; sc_scda_ferror_t errcode; sc_array_init_count (&elem_counts, sizeof (sc_scda_ulong), mpisize); + /* get the counts per process */ + per_proc_count = global_elem_count / (sc_scda_ulong) mpisize; + remainder_count = global_elem_count % (sc_scda_ulong) mpisize; + /* set elem_counts */ - for (i = 0; i < mpisize; ++i) { + for (i = 0; i < mpisize - 1; ++i) { /* partition dependent */ - *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 5; + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = per_proc_count; } + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpisize - 1)) = + (remainder_count == 0) ? per_proc_count : remainder_count; - fc = sc_scda_fwrite_array (fc, "A fixed-len array section", NULL, NULL, + fc = sc_scda_fwrite_array (fc, "A fixed-length array section", NULL, NULL, &elem_counts, elem_size, indirect, 0, &errcode); SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "sc_scda_fwrite_array failed"); From fc5a48c94e24a3d827f03b0e0c0bb9331773c1a7 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 26 Sep 2024 14:51:17 +0200 Subject: [PATCH 03/43] feature-scda: Work on fwrite_array param. checks --- src/sc_scda.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index c4418118..eebc40a3 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1850,7 +1850,8 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, sc_array_t *elem_counts, size_t elem_size, int indirect, int encode, sc_scda_ferror_t *errcode) { - int invalid_elem_counts; + int mpiret; + int invalid_elem_counts, global_invalid_elem_counts; int ret, count_err; size_t elem_count, si; #if 0 @@ -1875,22 +1876,34 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* TODO: respect encode parameter */ - /* compute the global element count */ - /* TODO: only required on rank SC_SCDA_HEADER_ROOT */ - elem_count = 0; - for (si = 0; si < elem_counts->elem_count; ++si) { - elem_count += *((size_t *) sc_array_index (elem_counts, si)); - } - /* check elem_counts array */ invalid_elem_counts = !(elem_counts->elem_size == sizeof (sc_scda_ulong) && elem_counts->elem_count == fc->mpisize); + /* synchronize */ + mpiret = sc_MPI_Allreduce (&invalid_elem_counts, &global_invalid_elem_counts, + 1, sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); + SC_CHECK_MPI (mpiret); sc_scda_scdaret_to_errcode (invalid_elem_counts ? SC_SCDA_FERR_ARG : SC_SCDA_FERR_SUCCESS, errcode, fc); SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Invalid elem_counts array"); + /* compute the global element count */ + /* the global count is only used on rank SC_SCDA_HEADER_ROOT */ + /* TODO: Compute this locally and use a checksum for the colletivness test? */ + elem_count = 0; + for (si = 0; si < elem_counts->elem_count; ++si) { + elem_count += *((size_t *) sc_array_index (elem_counts, si)); + } + + ret = sc_scda_check_coll_params (fc, (const char*) &elem_count, + sizeof (size_t), (const char*) &elem_size, + sizeof (size_t), NULL, 0); + sc_scda_scdaret_to_errcode (ret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_array: elem_counts or elem_size" + " is not collective"); + /* TODO: check array_data; depends on indirect parameter */ - /* Call Alreduce to synchronize on check of array_data */ + /* Call Allreduce to synchronize on check of array_data or collective test?*/ /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { @@ -1904,11 +1917,11 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, #if 0 /* get pointer to local array data */ if (indirect) { - /* indirect adressing */ + /* indirect addressing */ /* TODO: copy the data or use muliple write calls; maybe in batches? */ } else { - /* direct adressig */ + /* direct addressing */ local_array_data = (const void *) array_data->array; } From 56a12c79e80580fafe7acc7f0e481a9d1dbefa37 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 26 Sep 2024 15:53:27 +0200 Subject: [PATCH 04/43] feature-scda: Implement direct parallel write --- src/sc_scda.c | 58 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index eebc40a3..5f9b537d 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1851,24 +1851,26 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, int encode, sc_scda_ferror_t *errcode) { int mpiret; + int i; int invalid_elem_counts, global_invalid_elem_counts; int ret, count_err; + int num_local_elements, bytes_to_write; + int count; size_t elem_count, si; + sc_MPI_Offset offset; #if 0 const void *local_array_data; #endif SC_ASSERT (fc != NULL); SC_ASSERT (user_string != NULL); -#if 0 SC_ASSERT (array_data != NULL); -#endif SC_ASSERT (elem_counts != NULL); SC_ASSERT (errcode != NULL); /* TODO: Also check elem_counts and/or indirect? */ /* check if elem_size is collective */ - ret = sc_scda_check_coll_params (fc, (const char*) &elem_size, + ret = sc_scda_check_coll_params (fc, (const char *) &elem_size, sizeof (size_t), NULL, 0, NULL, 0); sc_scda_scdaret_to_errcode (ret, errcode, fc); SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_array: elem_size is not " @@ -1878,10 +1880,11 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* check elem_counts array */ invalid_elem_counts = !(elem_counts->elem_size == sizeof (sc_scda_ulong) && - elem_counts->elem_count == fc->mpisize); + elem_counts->elem_count == fc->mpisize); /* synchronize */ - mpiret = sc_MPI_Allreduce (&invalid_elem_counts, &global_invalid_elem_counts, - 1, sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); + mpiret = + sc_MPI_Allreduce (&invalid_elem_counts, &global_invalid_elem_counts, 1, + sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); SC_CHECK_MPI (mpiret); sc_scda_scdaret_to_errcode (invalid_elem_counts ? SC_SCDA_FERR_ARG : SC_SCDA_FERR_SUCCESS, errcode, fc); @@ -1895,15 +1898,16 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, elem_count += *((size_t *) sc_array_index (elem_counts, si)); } - ret = sc_scda_check_coll_params (fc, (const char*) &elem_count, - sizeof (size_t), (const char*) &elem_size, + ret = sc_scda_check_coll_params (fc, (const char *) &elem_count, + sizeof (size_t), (const char *) &elem_size, sizeof (size_t), NULL, 0); sc_scda_scdaret_to_errcode (ret, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_array: elem_counts or elem_size" + SC_SCDA_CHECK_COLL_ERR (errcode, fc, + "fwrite_array: elem_counts or elem_size" " is not collective"); /* TODO: check array_data; depends on indirect parameter */ - /* Call Allreduce to synchronize on check of array_data or collective test?*/ + /* Call Allreduce to synchronize on check of array_data or collective test? */ /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { @@ -1914,6 +1918,9 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, fc); + /* add number of written bytes */ + fc->accessed_bytes += SC_SCDA_COMMON_FIELD + 2 * SC_SCDA_COUNT_FIELD; + #if 0 /* get pointer to local array data */ if (indirect) { @@ -1928,6 +1935,37 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* write array data in parallel */ #endif + /* temporary */ + SC_ASSERT (!indirect); + + /* compute rank-dependent offset */ + offset = 0; + /* sum all element counts on previous processes */ + for (i = 0; i < fc->mpirank; ++i) { + offset += + (sc_MPI_Offset) * + ((sc_scda_ulong *) sc_array_index_int (elem_counts, i)); + } + offset *= (sc_MPI_Offset) elem_size; + + /* computer number of array data bytes that are locally written */ + num_local_elements = + (int) *((sc_scda_ulong *) sc_array_index_int (elem_counts, fc->mpirank)); + bytes_to_write = (int) elem_size *num_local_elements; + + mpiret = sc_io_write_at_all (fc->file, fc->accessed_bytes + offset, + array_data->array, bytes_to_write, sc_MPI_BYTE, + &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Writing block data padding"); + /* TODO: Implement collective count check macro */ + //SC_SCDA_CHECK_COLL_COUNT_ERR (bytes_to_write, count, count_err); + + /* TODO: update global number of written bytes */ + + /* TODO: get and write padding bytes in serial */ + + /* TODO: update global number of written bytes */ return fc; } From 8e42d21188141c59dddd6efd2dcc49fd27aa6563 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 26 Sep 2024 15:56:38 +0200 Subject: [PATCH 05/43] feature-scda: Test direct parallel data write --- test/test_scda.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index 82680bcb..a4c383fc 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -28,14 +28,18 @@ #define SC_SCDA_TEST_FILE "sc_test_scda." SC_SCDA_FILE_EXT static void -test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpisize) +test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, + int mpisize) { int indirect = 0; int i; + char *data_ptr; + size_t si; size_t elem_size = 3; + size_t local_elem_count; const sc_scda_ulong global_elem_count = 12; sc_scda_ulong per_proc_count, remainder_count; - sc_array_t elem_counts; + sc_array_t elem_counts, data; sc_scda_ferror_t errcode; sc_array_init_count (&elem_counts, sizeof (sc_scda_ulong), mpisize); @@ -46,18 +50,29 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpisize) /* set elem_counts */ for (i = 0; i < mpisize - 1; ++i) { - /* partition dependent */ *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = per_proc_count; } *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpisize - 1)) = (remainder_count == 0) ? per_proc_count : remainder_count; - fc = sc_scda_fwrite_array (fc, "A fixed-length array section", NULL, NULL, + /* create local data */ + local_elem_count = + (size_t) *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpirank)); + sc_array_init_size (&data, elem_size, local_elem_count); + for (si = 0; si < local_elem_count; ++si) { + data_ptr = (char *) sc_array_index (&data, si); + data_ptr[0] = 'a'; + data_ptr[1] = 'b'; + data_ptr[2] = 'c'; + } + + fc = sc_scda_fwrite_array (fc, "A fixed-length array section", NULL, &data, &elem_counts, elem_size, indirect, 0, &errcode); SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "sc_scda_fwrite_array failed"); sc_array_reset (&elem_counts); + sc_array_reset (&data); } int @@ -222,7 +237,7 @@ main (int argc, char **argv) "scda_fwrite_block failed"); /* write a fixed-size array section */ - test_scda_write_fixed_size_array (fc, mpisize); + test_scda_write_fixed_size_array (fc, mpirank, mpisize); /* intentionally try to write with non-collective block size */ if (mpisize > 1) { From d00c3c42d1370eb654971b611a29f53472a9d751 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 10:38:52 +0200 Subject: [PATCH 06/43] feature-scda: Implement and use collective count check macro --- src/sc_scda.c | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 5f9b537d..f3c792d4 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -137,7 +137,45 @@ SC_FREE (fc); \ return NULL;}} while (0) -/** Check for a count error of a serial (rank 0) I/O operation. +/** Check for a count error of a collective I/O operation. + * This macro is only valid to use after checking that there was not I/O error. + * The macro must be called collectively with \b fc and \b errcode being + * collective parameters. \b icount and \b ocount may depend on the MPI rank. + * The calling function must return NULL in case of an error. + */ +#define SC_SCDA_CHECK_COLL_COUNT_ERR(icount, ocount, fc, errcode) do { \ + int sc_scda_global_cerr; \ + int sc_scda_local_cerr; \ + int sc_scda_mpiret; \ + SC_ASSERT ( \ + sc_scda_ferror_is_success (*errcode)); \ + sc_scda_local_cerr = \ + ((int) icount != ocount); \ + sc_scda_mpiret = sc_MPI_Allreduce ( \ + &sc_scda_local_cerr, \ + &sc_scda_global_cerr,\ + 1, sc_MPI_INT, \ + sc_MPI_LOR, \ + fc->mpicomm); \ + SC_CHECK_MPI (sc_scda_mpiret); \ + sc_scda_scdaret_to_errcode ( \ + sc_scda_global_cerr ? SC_SCDA_FERR_COUNT :\ + SC_SCDA_FERR_SUCCESS,\ + errcode, fc); \ + SC_SCDA_CHECK_VERBOSE_COLL (*errcode, \ + "Read/write count check"); \ + if (sc_scda_global_cerr) { \ + SC_ASSERT ( \ + !sc_scda_ferror_is_success (*errcode));\ + SC_GLOBAL_LERRORF ("Count error for " \ + "collective I/O at %s:%d.\n",\ + __FILE__, __LINE__); \ + sc_scda_file_error_cleanup (&fc->file); \ + SC_FREE (fc); \ + return NULL; \ + }} while (0) \ + +/** Check for a count error of a serial I/O operation. * This macro is only valid to use after checking that there was not I/O error. * For a correct error handling it is required to skip the rest * of the non-collective code and then broadcast the count error flag. @@ -1958,8 +1996,8 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Writing block data padding"); - /* TODO: Implement collective count check macro */ - //SC_SCDA_CHECK_COLL_COUNT_ERR (bytes_to_write, count, count_err); + /* check for count error of the collective I/O operation */ + SC_SCDA_CHECK_COLL_COUNT_ERR (bytes_to_write, count, fc, errcode); /* TODO: update global number of written bytes */ From c59cfd8da0ac402ce922c4ae8a7531fb2d0df02f Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 11:44:28 +0200 Subject: [PATCH 07/43] feature-scda: Update doc. --- src/sc_scda.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index f3c792d4..94a9eeef 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1812,7 +1812,8 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, * sc_scda_fwrite_array. * \param [in] len As in the documentation of \ref * sc_scda_fwrite_array. - * TODO + * \param [in] elem_count As in the documentation of \ref + * sc_scda_fwrite_array. * \param [in] elem_size As in the documentation of \ref * sc_scda_fwrite_array. * \param [out] count_err A Boolean indicating if a count error occurred. From 924aea0a93a0ad192f48bd61730bb69ce36e8c74 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 14:02:08 +0200 Subject: [PATCH 08/43] feature-scda: Last byte as explicit padding param. --- src/sc_scda.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 94a9eeef..4e763263 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -491,30 +491,32 @@ sc_scda_pad_to_mod_len (size_t input_len) /** Pad data to a length that is congurent to 0 modulo \ref SC_SCDA_PADDING_MOD. * - * \param [in] input_data The input data. At least \b input_len bytes. - * The input data is required since the padding byte - * content dependt on the trailing input data byte. - * \param [in] input_len The length of \b input_data in number of bytes. + * \param [in] last_byte Pointer to the last data byte. + * This byte is required since the padding byte + * content depends on the trailing data byte. + * Must be NULL if \b data_len equals 0. + * \param [in] data_len The length of the data in number of bytes. * \param [out] padding On output the padding bytes. Must be at least - * \ref sc_scda_pad_to_mod_len (\b input_len) bytes. + * \ref sc_scda_pad_to_mod_len (\b data_len) bytes. */ static void -sc_scda_pad_to_mod (const char *input_data, size_t input_len, +sc_scda_pad_to_mod (const char *last_byte, size_t data_len, char *padding) { size_t num_pad_bytes; - SC_ASSERT (input_len == 0 || input_data != NULL); + SC_ASSERT (data_len == 0 || last_byte != NULL); + SC_ASSERT (data_len != 0 || last_byte == NULL); SC_ASSERT (padding != NULL); /* compute the number of padding bytes */ - num_pad_bytes = sc_scda_pad_to_mod_len (input_len); + num_pad_bytes = sc_scda_pad_to_mod_len (data_len); SC_ASSERT (num_pad_bytes >= 7); SC_ASSERT (num_pad_bytes <= SC_SCDA_PADDING_MOD_MAX); /* check for last byte to decide on padding format */ - if (input_len > 0 && input_data[input_len - 1] == '\n') { + if (data_len > 0 && *last_byte == '\n') { /* input data ends with a line break */ padding[0] = '='; } @@ -545,11 +547,14 @@ sc_scda_pad_to_mod_inplace (const char *input_data, size_t input_len, SC_ASSERT (input_len == 0 || input_data != NULL); SC_ASSERT (output_data != NULL); + const char *last_byte; + /* copy the input data */ sc_scda_copy_bytes (output_data, input_data, input_len); /* append the padding */ - sc_scda_pad_to_mod (input_data, input_len, &output_data[input_len]); + last_byte = (input_len > 0) ? &input_data[input_len - 1] : NULL; + sc_scda_pad_to_mod (last_byte, input_len, &output_data[input_len]); } /** Check if the padding bytes are correct w.r.t. \ref SC_SCDA_PADDING_MOD. @@ -1739,7 +1744,7 @@ sc_scda_fwrite_block_data_internal (sc_scda_fcontext_t *fc, /* get the padding bytes */ num_pad_bytes = sc_scda_pad_to_mod_len (block_size); - sc_scda_pad_to_mod (block_data->array, block_size, padding); + sc_scda_pad_to_mod (&block_data->array[block_size - 1], block_size, padding); /* write the padding bytes */ mpiret = sc_io_write_at (fc->file, fc->accessed_bytes + block_size, From 29a62fc6d062622a931694b682e1d90055a6a4c2 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 14:31:22 +0200 Subject: [PATCH 09/43] feature-scda: Write fix-len. array padding bytes --- src/sc_scda.c | 83 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 4e763263..040d7db2 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1888,6 +1888,47 @@ sc_scda_fwrite_array_header_internal (sc_scda_fcontext_t *fc, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (header_len, count, count_err); } +/** Internal function to write the fixed-length array padding. + * + * This function is dedicated to be called in \ref sc_scda_fwrite_array. + * This function is only valid to be called on the maximal not empty rank. + * + * \param [in] fc The file context as in \ref sc_scda_fwrite_array + * after running the collective write part. + * \param [in] last_byte A pointer to the last global data byte. + * \param [in] byte_count Number of collectively written bytes in the function + * \ref sc_scda_fwrite_array. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fwrite_padding_internal (sc_scda_fcontext_t *fc, const char *last_byte, + size_t byte_count, int *count_err, + sc_scda_ferror_t *errcode) +{ + int mpiret; + int count; + size_t num_pad_bytes; + /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ + char padding[SC_SCDA_PADDING_MOD + 6]; + + *count_err = 0; + + /* get the padding bytes */ + num_pad_bytes = sc_scda_pad_to_mod_len (byte_count); + sc_scda_pad_to_mod (last_byte, byte_count, padding); + + /* write the padding bytes */ + mpiret = sc_io_write_at (fc->file, fc->accessed_bytes, padding, + (int) num_pad_bytes, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing fixed-len. array data padding"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); +} + + sc_scda_fcontext_t * sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *array_data, @@ -1900,7 +1941,10 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, int ret, count_err; int num_local_elements, bytes_to_write; int count; + int last_byte_owner; size_t elem_count, si; + size_t collective_byte_count; + size_t num_pad_bytes; sc_MPI_Offset offset; #if 0 const void *local_array_data; @@ -1935,8 +1979,7 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Invalid elem_counts array"); /* compute the global element count */ - /* the global count is only used on rank SC_SCDA_HEADER_ROOT */ - /* TODO: Compute this locally and use a checksum for the colletivness test? */ + /* TODO: Use a checksum for the colletivness test? */ elem_count = 0; for (si = 0; si < elem_counts->elem_count; ++si) { elem_count += *((size_t *) sc_array_index (elem_counts, si)); @@ -1995,7 +2038,7 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* computer number of array data bytes that are locally written */ num_local_elements = (int) *((sc_scda_ulong *) sc_array_index_int (elem_counts, fc->mpirank)); - bytes_to_write = (int) elem_size *num_local_elements; + bytes_to_write = (int) elem_size * num_local_elements; mpiret = sc_io_write_at_all (fc->file, fc->accessed_bytes + offset, array_data->array, bytes_to_write, sc_MPI_BYTE, @@ -2005,11 +2048,39 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* check for count error of the collective I/O operation */ SC_SCDA_CHECK_COLL_COUNT_ERR (bytes_to_write, count, fc, errcode); - /* TODO: update global number of written bytes */ + /* update global number of written bytes */ + collective_byte_count = elem_count * elem_size; + fc->accessed_bytes += (sc_MPI_Offset) collective_byte_count; + + /* determine the rank that holds the last byte */ + /* TODO: Move this to a separate function? */ + last_byte_owner = 0; + for (i = fc->mpisize - 1; i >= 0; --i) { + if (*((sc_scda_ulong *)sc_array_index_int (elem_counts, i)) != 0) { + /* found maximal rank that is not empty */ + last_byte_owner = i; + break; + } + } + + /* get and write padding bytes in serial */ + if (fc->mpirank == last_byte_owner) { + const char *last_byte; - /* TODO: get and write padding bytes in serial */ + /* get last local/global byte */ + SC_ASSERT (elem_count == 0 || bytes_to_write > 0); + last_byte = (elem_count > 0) ? + &array_data->array[bytes_to_write - 1] : NULL; + /* the padding depends on the last data byte */ + sc_scda_fwrite_padding_internal (fc, last_byte, collective_byte_count, + &count_err, errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, last_byte_owner, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, last_byte_owner, fc); - /* TODO: update global number of written bytes */ + /* update global number of written bytes */ + num_pad_bytes = sc_scda_pad_to_mod_len (collective_byte_count); + fc->accessed_bytes += (sc_MPI_Offset) num_pad_bytes; return fc; } From eb7acc0f9b7ed4e5c48229ed64f501b981f844ce Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 14:33:11 +0200 Subject: [PATCH 10/43] feature-scda: Test an empty array --- test/test_scda.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index a4c383fc..f0627a45 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -31,11 +31,11 @@ static void test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, int mpisize) { - int indirect = 0; + const int indirect = 0; int i; char *data_ptr; size_t si; - size_t elem_size = 3; + const size_t elem_size = 3; size_t local_elem_count; const sc_scda_ulong global_elem_count = 12; sc_scda_ulong per_proc_count, remainder_count; @@ -71,6 +71,20 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "sc_scda_fwrite_array failed"); + /* write an empty array */ + + /* set elem_counts */ + for (i = 0; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 0; + } + + sc_array_resize (&data, 0); + + fc = sc_scda_fwrite_array (fc, "An empty array", NULL, &data, + &elem_counts, elem_size, indirect, 0, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fwrite_array empty array failed"); + sc_array_reset (&elem_counts); sc_array_reset (&data); } @@ -146,7 +160,7 @@ main (int argc, char **argv) if (mpisize > 1) { SC_GLOBAL_ESSENTIAL ("We expect two invalid scda function parameter errors." - " This is just for testing purposes and do not imply" + " This is just for testing purposes and does not imply" " erroneous code behavior.\n"); } /* fopen_write with non-collective fuzzy error parameters */ From 5a7d4fbf500b1f814995bfd00ee07c65c470adda Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 14:43:47 +0200 Subject: [PATCH 11/43] feature-scda: Separate func. for last byte owner --- src/sc_scda.c | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 040d7db2..ba8b80bb 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1928,6 +1928,36 @@ sc_scda_fwrite_padding_internal (sc_scda_fcontext_t *fc, const char *last_byte, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); } +/** Determine the maximal rank that is not empty. + * + * \param [in] fc A file context with filled MPI data. + * \param [in] elem_counts As the parameter \b elem_counts in \ref + * sc_scda_fwrite_array or \ref sc_scda_fwrite_varray. + * This array stores the number of elements per rank. + * \return The maximal rank that is not empty. If all ranks + * are empty, 0 is returned. + */ +static int +sc_scda_get_last_byte_owner (sc_scda_fcontext_t *fc, sc_array_t *elem_counts) +{ + int i; + int last_byte_owner; + + SC_ASSERT (fc != NULL); + SC_ASSERT (elem_counts != NULL); + + /* determine the rank that holds the last byte */ + last_byte_owner = 0; + for (i = fc->mpisize - 1; i >= 0; --i) { + if (*((sc_scda_ulong *)sc_array_index_int (elem_counts, i)) != 0) { + /* found maximal rank that is not empty */ + last_byte_owner = i; + break; + } + } + + return last_byte_owner; +} sc_scda_fcontext_t * sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, @@ -2053,15 +2083,7 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, fc->accessed_bytes += (sc_MPI_Offset) collective_byte_count; /* determine the rank that holds the last byte */ - /* TODO: Move this to a separate function? */ - last_byte_owner = 0; - for (i = fc->mpisize - 1; i >= 0; --i) { - if (*((sc_scda_ulong *)sc_array_index_int (elem_counts, i)) != 0) { - /* found maximal rank that is not empty */ - last_byte_owner = i; - break; - } - } + last_byte_owner = sc_scda_get_last_byte_owner (fc, elem_counts); /* get and write padding bytes in serial */ if (fc->mpirank == last_byte_owner) { From 44ab0f508624849fa2635121f14422a46bfdca0a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 18:24:30 +0200 Subject: [PATCH 12/43] feature-scda: Implement fixed-len. array section header read This commit contains a general rework of the generic count entry reading function to be prepared for the variable-sized array section type. --- src/sc_scda.c | 246 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 185 insertions(+), 61 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index ba8b80bb..8d2295b5 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2052,7 +2052,7 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* write array data in parallel */ #endif - /* temporary */ + /* TODO: temporary */ SC_ASSERT (!indirect); /* compute rank-dependent offset */ @@ -2074,7 +2074,7 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, array_data->array, bytes_to_write, sc_MPI_BYTE, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Writing block data padding"); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Writing fixed-length array padding"); /* check for count error of the collective I/O operation */ SC_SCDA_CHECK_COLL_COUNT_ERR (bytes_to_write, count, fc, errcode); @@ -2350,6 +2350,9 @@ sc_scda_fread_section_header_common_internal (sc_scda_fcontext_t *fc, case 'B': *type = 'B'; break; + case 'A': + *type = 'A'; + break; default: /* an invalid/unsupported format */ wrong_format = 1; @@ -2382,18 +2385,9 @@ sc_scda_fread_section_header_common_internal (sc_scda_fcontext_t *fc, "Invalid user string in section header"); } -/** Internal function to read a count entry in a section header. - * - * This code is only valid to be run in serial. It is crucial that - * fc->accessed_bytes is not updated inside of this function. Therefore, - * fc->accessed_bytes must be updated accordingly (and collectivly) afterwards. +/** Internal function to check a count entry in a section header. * - * This function is already prepared to be adjusted in the future to read a - * specified number of count entries. This will change the interface of this - * function. The reason that we will read multiple count entries in one function - * call instead of calling one function multiple times is that the error - * management for serial code parts relies on having one function that executes - * the serial code. + * This code is only valid to be run in serial. * * \param [in] fc The file context as in \ref * sc_scda_fread_section_header before running the @@ -2403,77 +2397,67 @@ sc_scda_fread_section_header_common_internal (sc_scda_fcontext_t *fc, * convention, the function call is not completed and * results in \ref SC_SCDA_FERR_FORMAT as error, * cf. \b errcode. - * \param [in] expc_ident The expected count entry identifier. If the \b ident - * is not as expected the error \ref - * SC_SCDA_FERR_FORMAT is set and the function flow is - * not completed. + * \param [in] expc_ident The expected count entry identifier. If the read + * identifier is not as expected the function returns + * false. * \param [out] count_var The count variable read from the count entry. * \param [out] count_err A Boolean indicating if a count error occurred. * For this parameter the term count refers to the * expected byte count for reading count (different * count) entry. - * \param [out] errcode An errcode that can be interpreted by \ref - * sc_scda_ferror_string or mapped to an error class - * by \ref sc_scda_ferror_class. + * \return True if the count entry is valid and has the + * expected identifier. */ -static void -sc_scda_fread_count_entry_internal (sc_scda_fcontext_t *fc, char *ident, - char expc_ident, size_t *count_var, - int *count_err, sc_scda_ferror_t *errcode) +static int +sc_scda_check_count_entry_internal (const char *count_entry, char expc_ident, + size_t *count_var) { - char count_entry[SC_SCDA_COUNT_FIELD]; char var_str[SC_SCDA_COUNT_MAX_DIGITS + 1]; - int mpiret; - int count; + char ident; int wrong_format, wrong_ident; long long unsigned read_count; size_t len = 0; - SC_ASSERT (fc != NULL); - SC_ASSERT (ident != NULL); SC_ASSERT (count_var != NULL); - *ident = ' '; *count_var = 0; - *count_err = 0; - - /* read a count entry */ - mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, count_entry, - SC_SCDA_COUNT_FIELD, sc_MPI_BYTE, &count); - sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read a count entry in a section header"); - SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); wrong_format = 0; /* check and get the count variable identifier */ switch (count_entry[0]) { case 'E': - *ident = 'E'; + ident = 'E'; + break; + case 'N': + ident = 'N'; break; default: /* invalid/unsupported count identifier */ wrong_format = 1; break; } - sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid count identifier"); + if (wrong_format) { + /* invalid count identifier */ + return -1; + } /* compare read count identifier to the expected count identifier */ - wrong_ident = (*ident != expc_ident); - sc_scda_scdaret_to_errcode (wrong_ident ? SC_SCDA_FERR_FORMAT : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Wrong count identifier in count entry"); + wrong_ident = (ident != expc_ident); + if (wrong_ident) { + /* Wrong count identifier in count entry */ + return -1; + } /* check count entry format */ if (count_entry[1] != ' ') { /* wrong format */ wrong_format = 1; } - sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Missing space in count entry"); + if (wrong_format) { + /* Missing space in count entry */ + return -1; + } /* check padding and extract count variable string */ sc_scda_init_nul (var_str, SC_SCDA_COUNT_MAX_DIGITS + 1); @@ -2481,9 +2465,10 @@ sc_scda_fread_count_entry_internal (sc_scda_fcontext_t *fc, char *ident, &len)) { wrong_format = 1; } - sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid count variable padding"); + if (wrong_format) { + /* Invalid count variable padding */ + return -1; + } /* If the padding to the length \ref SC_SCDA_COUNT_ENTRY was valid this * assertion must hold. @@ -2496,11 +2481,133 @@ sc_scda_fread_count_entry_internal (sc_scda_fcontext_t *fc, char *ident, /* conversion failed or is not possible */ wrong_format = 1; } - sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Extraction of count value failed"); + if (wrong_format) { + /* Extraction of count value failed */ + return -1; + } *count_var = (size_t) read_count; + + return 0; +} + +/** An internal function to read and check the block section header. + * + * This function is only valid to be called in serial. + * This function updates \b fc->accessed_bytes. Hence, it is crucial that + * \b fc->accessed_bytes is set collectivley afterwards. + * + * \param [in] fc The file context as in \ref + * sc_scda_fread_section_header before running the + * serial code part. + * \param [in] elem_size The read element size. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fread_block_header_internal (sc_scda_fcontext_t *fc, size_t *elem_size, + int *count_err, sc_scda_ferror_t *errcode) +{ + char count_entry[SC_SCDA_COUNT_FIELD]; + int mpiret; + int count; + int invalid_count_entry; + + SC_ASSERT (fc != NULL); + SC_ASSERT (count_err != NULL); + SC_ASSERT (errcode != NULL); + + *count_err = 0; + + /* read the count entry */ + mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, count_entry, + SC_SCDA_COUNT_FIELD, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read block section header count entry"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); + + /* check read count entry */ + invalid_count_entry = sc_scda_check_count_entry_internal (count_entry, 'E', + elem_size); + sc_scda_scdaret_to_errcode (invalid_count_entry ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, + errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid block count entry"); + + /* update the internal file pointer; only in serial */ + fc->accessed_bytes += SC_SCDA_COUNT_FIELD; +} + +/** An internal function to read and check the fixed-len array section header. + * + * This function is only valid to be called in serial. + * This function updates \b fc->accessed_bytes. Hence, it is crucial that + * \b fc->accessed_bytes is set collectivley afterwards. + * + * \param [in] fc The file context as in \ref + * sc_scda_fread_section_header before running the + * serial code part. + * \param [in] elem_size The read element size. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fread_array_header_internal (sc_scda_fcontext_t *fc, size_t *elem_count, + size_t *elem_size, int *count_err, + sc_scda_ferror_t *errcode) +{ + char count_entry[SC_SCDA_COUNT_FIELD]; + int mpiret; + int count; + int invalid_count_entry; + + SC_ASSERT (fc != NULL); + SC_ASSERT (count_err != NULL); + SC_ASSERT (errcode != NULL); + + *count_err = 0; + + /* read the first count entry */ + mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, count_entry, + SC_SCDA_COUNT_FIELD, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read block section header count entry"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); + + /* check read count entry */ + invalid_count_entry = sc_scda_check_count_entry_internal (count_entry,'N', + elem_count); + sc_scda_scdaret_to_errcode (invalid_count_entry ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, + errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid first fixed-length array count " + "entry"); + + /* update the internal file pointer; only in serial */ + fc->accessed_bytes += SC_SCDA_COUNT_FIELD; + + /* read the second count entry */ + mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, count_entry, + SC_SCDA_COUNT_FIELD, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read block section header count entry"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); + + /* check read count entry */ + invalid_count_entry = sc_scda_check_count_entry_internal (count_entry, 'E', + elem_size); + sc_scda_scdaret_to_errcode (invalid_count_entry ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, + errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid second fixed-length array count " + "entry"); + + /* update the internal file pointer; only in serial */ + fc->accessed_bytes += SC_SCDA_COUNT_FIELD; } sc_scda_fcontext_t * @@ -2511,7 +2618,6 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, { int count_err; int mpiret; - char var_ident; SC_ASSERT (fc != NULL); SC_ASSERT (user_string != NULL); @@ -2543,9 +2649,11 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, /* no count entries to read */ break; case 'B': - /* one count entry to read */ - sc_scda_fread_count_entry_internal (fc, &var_ident, 'E', elem_size, - &count_err, errcode); + sc_scda_fread_block_header_internal (fc, elem_size, &count_err, errcode); + break; + case 'A': + sc_scda_fread_array_header_internal (fc, elem_count, elem_size, + &count_err, errcode); break; default: /* rank SC_SCDA_HEADER_ROOT already checked if type is valid/supported */ @@ -2580,8 +2688,24 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, mpiret = sc_MPI_Bcast (elem_size, sizeof (size_t), sc_MPI_BYTE, SC_SCDA_HEADER_ROOT, fc->mpicomm); SC_CHECK_MPI (mpiret); - /* one count entry was read */ - fc->accessed_bytes += SC_SCDA_COUNT_FIELD; + if (fc->mpirank != SC_SCDA_HEADER_ROOT) { + /* update internal file pointer */ + fc->accessed_bytes += SC_SCDA_COUNT_FIELD; + } + break; + case 'A': + /* fixed-length array */ + /* elem_count and elem_size were read on rank SC_SCDA_HEADER_ROOT */ + mpiret = sc_MPI_Bcast (elem_count, sizeof (size_t), sc_MPI_BYTE, + SC_SCDA_HEADER_ROOT, fc->mpicomm); + SC_CHECK_MPI (mpiret); + mpiret = sc_MPI_Bcast (elem_size, sizeof (size_t), sc_MPI_BYTE, + SC_SCDA_HEADER_ROOT, fc->mpicomm); + SC_CHECK_MPI (mpiret); + if (fc->mpirank != SC_SCDA_HEADER_ROOT) { + /* update internal file pointer */ + fc->accessed_bytes += 2 * SC_SCDA_COUNT_FIELD; + } break; default: /* rank SC_SCDA_HEADER_ROOT already checked if type is valid/supported */ From 3027c6218381dec2bb26888c9bfac860df6bc533 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 30 Sep 2024 18:27:15 +0200 Subject: [PATCH 13/43] feature-scda: First test of array reading --- test/test_scda.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index f0627a45..7e036339 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -27,6 +27,9 @@ #define SC_SCDA_FILE_EXT "scd" #define SC_SCDA_TEST_FILE "sc_test_scda." SC_SCDA_FILE_EXT +#define SC_SCDA_GLOBAL_ARRAY_COUNT 12 +#define SC_SCDA_ARRAY_SIZE 3 + static void test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, int mpisize) @@ -35,9 +38,9 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, int i; char *data_ptr; size_t si; - const size_t elem_size = 3; + const size_t elem_size = SC_SCDA_ARRAY_SIZE; size_t local_elem_count; - const sc_scda_ulong global_elem_count = 12; + const sc_scda_ulong global_elem_count = SC_SCDA_GLOBAL_ARRAY_COUNT; sc_scda_ulong per_proc_count, remainder_count; sc_array_t elem_counts, data; sc_scda_ferror_t errcode; @@ -89,6 +92,29 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, sc_array_reset (&data); } +static void +test_scda_read_fixed_size_array (sc_scda_fcontext_t *fc) +{ + const int indirect = 0; + int decode; + char read_user_string[SC_SCDA_USER_STRING_BYTES + 1]; + char section_type; + size_t len; + size_t elem_count, elem_size; + sc_scda_ferror_t errcode; + + fc = sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'A' && + elem_count == SC_SCDA_GLOBAL_ARRAY_COUNT && + elem_size == SC_SCDA_ARRAY_SIZE, "Identifying section type"); + + /* TODO: read the empty array */ +} + int main (int argc, char **argv) { @@ -361,6 +387,8 @@ main (int argc, char **argv) || !strncmp (read_data, inline_data, SC_SCDA_INLINE_FIELD), "block data mismatch"); + test_scda_read_fixed_size_array (fc); + sc_scda_fclose (fc, &errcode); /* TODO: check errcode and return value */ SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), From 5db2c42312acbc7410dea43a83372455419a869a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 1 Oct 2024 09:48:24 +0200 Subject: [PATCH 14/43] feature-scda: Polish internal func. doc. --- src/sc_scda.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 8d2295b5..1354aaee 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2386,8 +2386,6 @@ sc_scda_fread_section_header_common_internal (sc_scda_fcontext_t *fc, } /** Internal function to check a count entry in a section header. - * - * This code is only valid to be run in serial. * * \param [in] fc The file context as in \ref * sc_scda_fread_section_header before running the @@ -2495,7 +2493,8 @@ sc_scda_check_count_entry_internal (const char *count_entry, char expc_ident, * * This function is only valid to be called in serial. * This function updates \b fc->accessed_bytes. Hence, it is crucial that - * \b fc->accessed_bytes is set collectivley afterwards. + * \b fc->accessed_bytes is set on all other ranks afterwards to ensure + * a collective internal file pointer. * * \param [in] fc The file context as in \ref * sc_scda_fread_section_header before running the @@ -2544,7 +2543,8 @@ sc_scda_fread_block_header_internal (sc_scda_fcontext_t *fc, size_t *elem_size, * * This function is only valid to be called in serial. * This function updates \b fc->accessed_bytes. Hence, it is crucial that - * \b fc->accessed_bytes is set collectivley afterwards. + * \b fc->accessed_bytes is set on all other ranks afterwards to ensure + * a collective internal file pointer. * * \param [in] fc The file context as in \ref * sc_scda_fread_section_header before running the From 7ffa3b84fe1d8fc18dc0e1dfe348fe36156d6d3a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 1 Oct 2024 10:06:51 +0200 Subject: [PATCH 15/43] feature-scda: Rework naming scheme for serial functions All serial I/O operations take place in separate functions and all these functions have the suffix _serial. --- src/sc_scda.c | 135 ++++++++++++++++++++++++-------------------------- 1 file changed, 66 insertions(+), 69 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 1354aaee..1cede3f0 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1281,9 +1281,9 @@ sc_scda_get_common_section_header (char section_char, const char* user_string, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fopen_write_header_internal (sc_scda_fcontext_t *fc, - const char *user_string, size_t *len, - int *count_err, sc_scda_ferror_t *errcode) +sc_scda_fopen_write_header_serial (sc_scda_fcontext_t *fc, + const char *user_string, size_t *len, + int *count_err, sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -1403,13 +1403,13 @@ sc_scda_fopen_write (sc_MPI_Comm mpicomm, SC_SCDA_CHECK_COLL_ERR (errcode, fc, "File open write"); if (fc->mpirank == SC_SCDA_HEADER_ROOT) { - sc_scda_fopen_write_header_internal (fc, user_string, len, &count_err, - errcode); + sc_scda_fopen_write_header_serial (fc, user_string, len, &count_err, + errcode); } /* This macro must be the first expression after the non-collective code part * since it must be the point where \ref SC_SCDA_CHECK_NONCOLL_ERR and \ref * SC_SCDA_CHECK_NONCOLL_COUNT_ERR can jump to, which are called in \ref - * sc_scda_fopen_write_header_internal. The macro handles the non-collective + * sc_scda_fopen_write_header_serial. The macro handles the non-collective * error, i.e. it broadcasts the errcode, which may encode success, from * rank 0 to all other ranks and in case of an error it closes the file, * frees the file context and returns NULL. Hence, it is valid that errcode @@ -1428,7 +1428,7 @@ sc_scda_fopen_write (sc_MPI_Comm mpicomm, * that it is valid that errcode is only initialized on rank 0 before calling * this macro. The macro argument count_err must point to the count error * Boolean that was set on rank SC_SCDA_HEADER_ROOT by \ref - * sc_scda_fopen_write_header_internal. + * sc_scda_fopen_write_header_serial. */ SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, fc); @@ -1459,10 +1459,9 @@ sc_scda_fopen_write (sc_MPI_Comm mpicomm, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fwrite_inline_header_internal (sc_scda_fcontext_t *fc, - const char *user_string, size_t *len, - int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fwrite_inline_header_serial (sc_scda_fcontext_t *fc, + const char *user_string, size_t *len, + int *count_err, sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -1512,9 +1511,9 @@ sc_scda_fwrite_inline_header_internal (sc_scda_fcontext_t *fc, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fwrite_inline_data_internal (sc_scda_fcontext_t *fc, - sc_array_t * inline_data, int *count_err, - sc_scda_ferror_t * errcode) +sc_scda_fwrite_inline_data_serial (sc_scda_fcontext_t *fc, + sc_array_t * inline_data, int *count_err, + sc_scda_ferror_t * errcode) { int mpiret; int count; @@ -1553,8 +1552,8 @@ sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, /* The file header section is always written and read on rank 0. */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { - sc_scda_fwrite_inline_header_internal (fc, user_string, len, &count_err, - errcode); + sc_scda_fwrite_inline_header_serial (fc, user_string, len, &count_err, + errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, @@ -1565,8 +1564,8 @@ sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, /* The inline data is written on the the user-defined rank root. */ if (fc->mpirank == root) { - sc_scda_fwrite_inline_data_internal (fc, inline_data, &count_err, - errcode); + sc_scda_fwrite_inline_data_serial (fc, inline_data, &count_err, + errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); @@ -1650,10 +1649,10 @@ sc_scda_get_section_header_entry (char ident, size_t var, char *output) * by \ref sc_scda_ferror_class. */ static void -sc_scda_fwrite_block_header_internal (sc_scda_fcontext_t *fc, - const char *user_string, size_t *len, - size_t block_size, int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fwrite_block_header_serial (sc_scda_fcontext_t *fc, + const char *user_string, size_t *len, + size_t block_size, int *count_err, + sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -1715,9 +1714,9 @@ sc_scda_fwrite_block_header_internal (sc_scda_fcontext_t *fc, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fwrite_block_data_internal (sc_scda_fcontext_t *fc, - sc_array_t * block_data, size_t block_size, - int *count_err, sc_scda_ferror_t * errcode) +sc_scda_fwrite_block_data_serial (sc_scda_fcontext_t *fc, + sc_array_t * block_data, size_t block_size, + int *count_err, sc_scda_ferror_t * errcode) { int mpiret; int count; @@ -1781,8 +1780,8 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { - sc_scda_fwrite_block_header_internal (fc, user_string, len, block_size, - &count_err, errcode); + sc_scda_fwrite_block_header_serial (fc, user_string, len, block_size, + &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, @@ -1793,8 +1792,8 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, /* The block data is written on the the user-defined rank root. */ if (fc->mpirank == root) { - sc_scda_fwrite_block_data_internal (fc, block_data, block_size, - &count_err, errcode); + sc_scda_fwrite_block_data_serial (fc, block_data, block_size, + &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); @@ -1827,10 +1826,10 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fwrite_array_header_internal (sc_scda_fcontext_t *fc, - const char *user_string, size_t *len, - size_t elem_count, size_t elem_size, - int *count_err, sc_scda_ferror_t *errcode) +sc_scda_fwrite_array_header_serial (sc_scda_fcontext_t *fc, + const char *user_string, size_t *len, + size_t elem_count, size_t elem_size, + int *count_err, sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -1904,9 +1903,9 @@ sc_scda_fwrite_array_header_internal (sc_scda_fcontext_t *fc, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fwrite_padding_internal (sc_scda_fcontext_t *fc, const char *last_byte, - size_t byte_count, int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fwrite_padding_serial (sc_scda_fcontext_t *fc, const char *last_byte, + size_t byte_count, int *count_err, + sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -2028,8 +2027,8 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { - sc_scda_fwrite_array_header_internal (fc, user_string, len, elem_count, - elem_size, &count_err, errcode); + sc_scda_fwrite_array_header_serial (fc, user_string, len, elem_count, + elem_size, &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, @@ -2094,8 +2093,8 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, last_byte = (elem_count > 0) ? &array_data->array[bytes_to_write - 1] : NULL; /* the padding depends on the last data byte */ - sc_scda_fwrite_padding_internal (fc, last_byte, collective_byte_count, - &count_err, errcode); + sc_scda_fwrite_padding_serial (fc, last_byte, collective_byte_count, + &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, last_byte_owner, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, last_byte_owner, fc); @@ -2196,9 +2195,9 @@ sc_scda_check_file_header (const char *file_header_data, char *user_string, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fopen_read_header_internal (sc_scda_fcontext_t * fc, - char *user_string, size_t *len, - int *count_err, sc_scda_ferror_t *errcode) +sc_scda_fopen_read_header_serial (sc_scda_fcontext_t * fc, + char *user_string, size_t *len, + int *count_err, sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -2274,8 +2273,8 @@ sc_scda_fopen_read (sc_MPI_Comm mpicomm, /* read file header section on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { - sc_scda_fopen_read_header_internal (fc, user_string, len, &count_err, - errcode); + sc_scda_fopen_read_header_serial (fc, user_string, len, &count_err, + errcode); } /* The macro to handle a non-collective error that is associated to a * preceding call of \ref SC_SCDA_CHECK_NONCOLL_ERR. @@ -2322,10 +2321,10 @@ sc_scda_fopen_read (sc_MPI_Comm mpicomm, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fread_section_header_common_internal (sc_scda_fcontext_t *fc, - char *type, char *user_string, - size_t *len, int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fread_section_header_common_serial (sc_scda_fcontext_t *fc, + char *type, char *user_string, + size_t *len, int *count_err, + sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -2506,8 +2505,8 @@ sc_scda_check_count_entry_internal (const char *count_entry, char expc_ident, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fread_block_header_internal (sc_scda_fcontext_t *fc, size_t *elem_size, - int *count_err, sc_scda_ferror_t *errcode) +sc_scda_fread_block_header_serial (sc_scda_fcontext_t *fc, size_t *elem_size, + int *count_err, sc_scda_ferror_t *errcode) { char count_entry[SC_SCDA_COUNT_FIELD]; int mpiret; @@ -2556,9 +2555,9 @@ sc_scda_fread_block_header_internal (sc_scda_fcontext_t *fc, size_t *elem_size, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fread_array_header_internal (sc_scda_fcontext_t *fc, size_t *elem_count, - size_t *elem_size, int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fread_array_header_serial (sc_scda_fcontext_t *fc, size_t *elem_count, + size_t *elem_size, int *count_err, + sc_scda_ferror_t *errcode) { char count_entry[SC_SCDA_COUNT_FIELD]; int mpiret; @@ -2632,8 +2631,8 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, /* read the common section header part first */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { - sc_scda_fread_section_header_common_internal (fc, type, user_string, len, - &count_err, errcode); + sc_scda_fread_section_header_common_serial (fc, type, user_string, len, + &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, @@ -2649,11 +2648,11 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, /* no count entries to read */ break; case 'B': - sc_scda_fread_block_header_internal (fc, elem_size, &count_err, errcode); + sc_scda_fread_block_header_serial (fc, elem_size, &count_err, errcode); break; case 'A': - sc_scda_fread_array_header_internal (fc, elem_count, elem_size, - &count_err, errcode); + sc_scda_fread_array_header_serial (fc, elem_count, elem_size, + &count_err, errcode); break; default: /* rank SC_SCDA_HEADER_ROOT already checked if type is valid/supported */ @@ -2732,9 +2731,8 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fread_inline_data_serial_internal (sc_scda_fcontext_t *fc, - sc_array_t *data, int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fread_inline_data_serial (sc_scda_fcontext_t *fc, sc_array_t *data, + int *count_err, sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -2778,7 +2776,7 @@ sc_scda_fread_inline_data (sc_scda_fcontext_t *fc, sc_array_t *data, int root, if (data != NULL) { /* the data is not skipped */ if (fc->mpirank == root) { - sc_scda_fread_inline_data_serial_internal (fc, data, &count_err, errcode); + sc_scda_fread_inline_data_serial (fc, data, &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); @@ -2808,10 +2806,9 @@ sc_scda_fread_inline_data (sc_scda_fcontext_t *fc, sc_array_t *data, int root, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fread_block_data_serial_internal (sc_scda_fcontext_t *fc, - sc_array_t *data, size_t block_size, - int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fread_block_data_serial (sc_scda_fcontext_t *fc, sc_array_t *data, + size_t block_size, int *count_err, + sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -2884,8 +2881,8 @@ sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, if (block_data != NULL) { /* the data is not skipped */ if (fc->mpirank == root) { - sc_scda_fread_block_data_serial_internal (fc, block_data, block_size, - &count_err, errcode); + sc_scda_fread_block_data_serial(fc, block_data, block_size, &count_err, + errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); From d4268fd38ab5eed2986df444280c85076a794d17 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 1 Oct 2024 10:08:50 +0200 Subject: [PATCH 16/43] feature-scda: Deactivate currently unused variable --- test/test_scda.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_scda.c b/test/test_scda.c index 7e036339..e7efa07a 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -95,7 +95,9 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, static void test_scda_read_fixed_size_array (sc_scda_fcontext_t *fc) { +#if 0 const int indirect = 0; +#endif int decode; char read_user_string[SC_SCDA_USER_STRING_BYTES + 1]; char section_type; From d25fdfd6a7c3c1f9a1421a7bb7fd87102a87a704 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 1 Oct 2024 10:29:15 +0200 Subject: [PATCH 17/43] feature-scda: Fix empty block write --- src/sc_scda.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 1cede3f0..7bbbc47a 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1724,6 +1724,7 @@ sc_scda_fwrite_block_data_serial (sc_scda_fcontext_t *fc, size_t num_pad_bytes; /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ char padding[SC_SCDA_PADDING_MOD + 6]; + const char *last_byte; *count_err = 0; @@ -1743,7 +1744,8 @@ sc_scda_fwrite_block_data_serial (sc_scda_fcontext_t *fc, /* get the padding bytes */ num_pad_bytes = sc_scda_pad_to_mod_len (block_size); - sc_scda_pad_to_mod (&block_data->array[block_size - 1], block_size, padding); + last_byte = (block_size > 0) ? &block_data->array[block_size - 1] : NULL; + sc_scda_pad_to_mod (last_byte, block_size, padding); /* write the padding bytes */ mpiret = sc_io_write_at (fc->file, fc->accessed_bytes + block_size, @@ -2073,7 +2075,7 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, array_data->array, bytes_to_write, sc_MPI_BYTE, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Writing fixed-length array padding"); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Writing fixed-length array data"); /* check for count error of the collective I/O operation */ SC_SCDA_CHECK_COLL_COUNT_ERR (bytes_to_write, count, fc, errcode); From 8a0c5ac64b338f0c4c496ddbe069353b3ceda973 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 13:59:41 +0200 Subject: [PATCH 18/43] feature-scda: More general array padding func. --- src/sc_scda.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 7bbbc47a..9c3beb8e 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1794,6 +1794,7 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, /* The block data is written on the the user-defined rank root. */ if (fc->mpirank == root) { + /* This function includes writing the padding. */ sc_scda_fwrite_block_data_serial (fc, block_data, block_size, &count_err, errcode); } @@ -1889,9 +1890,10 @@ sc_scda_fwrite_array_header_serial (sc_scda_fcontext_t *fc, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (header_len, count, count_err); } -/** Internal function to write the fixed-length array padding. +/** Internal function to write array padding. * - * This function is dedicated to be called in \ref sc_scda_fwrite_array. + * This function is dedicated to be called in \ref sc_scda_fwrite_array + * or in \ref sc_scda_fwrite_varray. * This function is only valid to be called on the maximal not empty rank. * * \param [in] fc The file context as in \ref sc_scda_fwrite_array @@ -1905,9 +1907,9 @@ sc_scda_fwrite_array_header_serial (sc_scda_fcontext_t *fc, * by \ref sc_scda_ferror_class. */ static void -sc_scda_fwrite_padding_serial (sc_scda_fcontext_t *fc, const char *last_byte, - size_t byte_count, int *count_err, - sc_scda_ferror_t *errcode) +sc_scda_fwrite_array_padding_serial (sc_scda_fcontext_t *fc, + const char *last_byte, size_t byte_count, + int *count_err, sc_scda_ferror_t *errcode) { int mpiret; int count; @@ -1925,7 +1927,7 @@ sc_scda_fwrite_padding_serial (sc_scda_fcontext_t *fc, const char *last_byte, mpiret = sc_io_write_at (fc->file, fc->accessed_bytes, padding, (int) num_pad_bytes, sc_MPI_BYTE, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing fixed-len. array data padding"); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing array data padding"); SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); } @@ -2095,8 +2097,8 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, last_byte = (elem_count > 0) ? &array_data->array[bytes_to_write - 1] : NULL; /* the padding depends on the last data byte */ - sc_scda_fwrite_padding_serial (fc, last_byte, collective_byte_count, - &count_err, errcode); + sc_scda_fwrite_array_padding_serial (fc, last_byte, collective_byte_count, + &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, last_byte_owner, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, last_byte_owner, fc); From 53a0641d7663c15d8154c91ee925ea3806a32c16 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 14:50:20 +0200 Subject: [PATCH 19/43] feature-scda: Fix fwrite_array test --- test/test_scda.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index e7efa07a..57c0490b 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -45,18 +45,20 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, sc_array_t elem_counts, data; sc_scda_ferror_t errcode; - sc_array_init_count (&elem_counts, sizeof (sc_scda_ulong), mpisize); + sc_array_init_count (&elem_counts, sizeof (sc_scda_ulong), + (size_t) mpisize); /* get the counts per process */ per_proc_count = global_elem_count / (sc_scda_ulong) mpisize; remainder_count = global_elem_count % (sc_scda_ulong) mpisize; /* set elem_counts */ - for (i = 0; i < mpisize - 1; ++i) { - *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = per_proc_count; + for (i = 0; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = + per_proc_count; } - *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpisize - 1)) = - (remainder_count == 0) ? per_proc_count : remainder_count; + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpisize - 1)) += + remainder_count; /* create local data */ local_elem_count = From ab302834f06a0cca893b162e42401d581b583636 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 15:02:42 +0200 Subject: [PATCH 20/43] feature-scda: Introduce local_partition_index function --- src/sc_scda.c | 65 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 9c3beb8e..6b9e90ba 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1962,6 +1962,52 @@ sc_scda_get_last_byte_owner (sc_scda_fcontext_t *fc, sc_array_t *elem_counts) return last_byte_owner; } +/** Get local partition byte offset and byte length. + * + * \param [in] fc A valid file context. + * \param [in] elem_counts As in the documentation of \ref + * sc_array_fwrite_array_data or \ref + * sc_array_fwrite_varray_data. + * Encodes the partition and must be conforming to + * \b fc. Cf. the documentation of the referenced + * functions for more information. + * \param [in] elem_size As in the documentation of \ref + * sc_array_fwrite_array_data or \ref + * sc_array_fwrite_varray_data. + * \param [out] offset On output filled with the local byte offset + * according to the given partition. + * \param [out] num_bytes On output filled with the number of local byte + * offset according to the given partition. + */ +static void +sc_scda_get_local_partition_index (sc_scda_fcontext_t *fc, + sc_array_t *elem_counts, size_t elem_size, + sc_MPI_Offset *offset, int *num_bytes) +{ + int i; + int num_local_elements; + + SC_ASSERT (fc != NULL); + SC_ASSERT (elem_counts != NULL); + SC_ASSERT (elem_counts->elem_size == sizeof (sc_scda_ulong)); + SC_ASSERT ((int) elem_counts->elem_count == fc->mpisize); + + /* compute rank-dependent offset */ + *offset = 0; + /* sum all element counts on previous processes */ + for (i = 0; i < fc->mpirank; ++i) { + *offset += + (sc_MPI_Offset) * + ((sc_scda_ulong *) sc_array_index_int (elem_counts, i)); + } + *offset *= (sc_MPI_Offset) elem_size; + + /* computer number of local array data bytes */ + num_local_elements = + (int) *((sc_scda_ulong *) sc_array_index_int (elem_counts, fc->mpirank)); + *num_bytes = (int) elem_size * num_local_elements; +} + sc_scda_fcontext_t * sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *array_data, @@ -1969,10 +2015,9 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, int encode, sc_scda_ferror_t *errcode) { int mpiret; - int i; int invalid_elem_counts, global_invalid_elem_counts; int ret, count_err; - int num_local_elements, bytes_to_write; + int bytes_to_write; int count; int last_byte_owner; size_t elem_count, si; @@ -2058,20 +2103,8 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* TODO: temporary */ SC_ASSERT (!indirect); - /* compute rank-dependent offset */ - offset = 0; - /* sum all element counts on previous processes */ - for (i = 0; i < fc->mpirank; ++i) { - offset += - (sc_MPI_Offset) * - ((sc_scda_ulong *) sc_array_index_int (elem_counts, i)); - } - offset *= (sc_MPI_Offset) elem_size; - - /* computer number of array data bytes that are locally written */ - num_local_elements = - (int) *((sc_scda_ulong *) sc_array_index_int (elem_counts, fc->mpirank)); - bytes_to_write = (int) elem_size * num_local_elements; + sc_scda_get_local_partition_index (fc, elem_counts, elem_size, &offset, + &bytes_to_write); mpiret = sc_io_write_at_all (fc->file, fc->accessed_bytes + offset, array_data->array, bytes_to_write, sc_MPI_BYTE, From ba6d6dcf4df58c87b9bd5599595a9b30202e9cdf Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 17:57:27 +0200 Subject: [PATCH 21/43] feature-scda: Adjust check_pad_to_mod --- src/sc_scda.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 6b9e90ba..32aee356 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -560,10 +560,12 @@ sc_scda_pad_to_mod_inplace (const char *input_data, size_t input_len, /** Check if the padding bytes are correct w.r.t. \ref SC_SCDA_PADDING_MOD. * * Since the mod padding depends on the trailing data byte this function also - * requires the raw data. + * requires this byte. * - * \param [in] data The raw data with byte count \b data_len. - * \param [in] data_len The length of \b data in number of bytes. + * \param [in] last_byte Pointer to the last data byte. + * This byte is required since the padding byte + * content depends on the trailing data byte. + * \param [in] data_len The raw data length in number of bytes. * \param [in] pad The padding bytes with byte count \b pad_len. * \param [in] pad_len The length of \b pad in number of bytes. * Must be at least 7. @@ -572,8 +574,8 @@ sc_scda_pad_to_mod_inplace (const char *input_data, size_t input_len, * condition. False, otherwise. */ static int -sc_scda_check_pad_to_mod (const char* data, size_t data_len, const char *pad, - size_t pad_len) +sc_scda_check_pad_to_mod (const char* last_byte, size_t data_len, + const char *pad, size_t pad_len) { size_t si; size_t num_pad_bytes; @@ -605,7 +607,7 @@ sc_scda_check_pad_to_mod (const char* data, size_t data_len, const char *pad, /* padding depends on the trailing data byte */ if ((!((pad[si] == '=' && data_len != 0 && - data[data_len - 1] == '\n') || pad[si] == '\n'))) { + *last_byte == '\n') || pad[si] == '\n'))) { /* wrong padding start */ return -1; } @@ -644,8 +646,8 @@ sc_scda_get_pad_to_mod (const char *padded_data, size_t padded_len, return -1; } - if (sc_scda_check_pad_to_mod (padded_data, raw_len, &padded_data[raw_len], - padded_len - raw_len)) { + if (sc_scda_check_pad_to_mod (&padded_data[raw_len - 1], raw_len, + &padded_data[raw_len], padded_len - raw_len)) { /* invalid padding bytes */ return -1; } From 596ddd4bd4b851d92de68aa69918c5c4b59430b4 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 17:59:10 +0200 Subject: [PATCH 22/43] feature-scda: Implement a general read data padding function --- src/sc_scda.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/sc_scda.c b/src/sc_scda.c index 32aee356..30c2d563 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2830,6 +2830,81 @@ sc_scda_fread_inline_data (sc_scda_fcontext_t *fc, sc_array_t *data, int root, return fc; } +/** TODO: Documentation + * @brief + * + * @param fc + * @param last_byte If \b last_byte is NULL and byte_count > 0 the last data + * byte is read from file. + * @param byte_count + * @param count_err + * @param errcode + */ +static void +sc_scda_fread_mod_padding_serial (sc_scda_fcontext_t *fc, + const char *last_byte, size_t byte_count, + int *count_err, sc_scda_ferror_t *errcode) +{ + int mpiret; + int count; + int read_last_byte; + sc_MPI_Offset offset; + int num_bytes; + int invalid_padding; + size_t num_pad_bytes; + /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ + char padding[SC_SCDA_PADDING_MOD + 6]; + const char *last_byte_internal; + char *padding_internal; + + *count_err = 0; + + num_pad_bytes = sc_scda_pad_to_mod_len (byte_count); + + /* Do we read the last data byte if there is one? */ + read_last_byte = last_byte == NULL && byte_count > 0; + + if (read_last_byte) { + offset = fc->accessed_bytes - 1; + num_bytes = (int) num_pad_bytes + 1; + } + else { + offset = fc->accessed_bytes; + num_bytes = (int) num_pad_bytes; + } + + mpiret = sc_io_read_at (fc->file, offset, padding, num_bytes, sc_MPI_BYTE, + &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read modulo data padding"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_bytes, count, count_err); + + /* check the padding */ + /* data padding depends the last data byte */ + if (byte_count == 0) { + /* there is no last byte */ + SC_ASSERT (last_byte == NULL); + last_byte_internal = NULL; + padding_internal = padding; + } + else if (read_last_byte) { + /* there is a last byte and it was read from file */ + last_byte_internal = &padding[0]; + padding_internal = &padding[1]; + } + else { + /* there is a last byte and it was passed to this function */ + last_byte_internal = last_byte; + padding_internal = padding; + } + + invalid_padding = sc_scda_check_pad_to_mod (last_byte_internal, byte_count, + padding_internal, num_pad_bytes); + sc_scda_scdaret_to_errcode (invalid_padding ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid modulo data padding"); +} + /** Internal function to read the block data. * * \param [in] fc The file context as in \ref From 403096eef43d8eccecedb1ae4fc950794c9ae44b Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 18:05:00 +0200 Subject: [PATCH 23/43] feature-scda: Use general data pad. read func. --- src/sc_scda.c | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 30c2d563..6e2610ab 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2926,9 +2926,7 @@ sc_scda_fread_block_data_serial (sc_scda_fcontext_t *fc, sc_array_t *data, { int mpiret; int count; - int invalid_array, invalid_padding; - size_t num_pad_bytes; - char paddding[SC_SCDA_PADDING_MOD_MAX]; + int invalid_array; *count_err = 0; @@ -2942,26 +2940,8 @@ sc_scda_fread_block_data_serial (sc_scda_fcontext_t *fc, sc_array_t *data, mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, data->array, (int) block_size, sc_MPI_BYTE, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read inline data"); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read block data"); SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); - - num_pad_bytes = sc_scda_pad_to_mod_len (block_size); - - /* read the padding the bytes */ - /* the padding depends on the trailing byte of the data */ - mpiret = sc_io_read_at (fc->file, fc->accessed_bytes + - (sc_MPI_Offset) block_size, paddding, - (int) num_pad_bytes, sc_MPI_BYTE, &count); - sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read inline data padding"); - SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); - - /* check the padding */ - invalid_padding = sc_scda_check_pad_to_mod (data->array, block_size, paddding, - num_pad_bytes); - sc_scda_scdaret_to_errcode (invalid_padding ? SC_SCDA_FERR_FORMAT : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid block data padding"); } sc_scda_fcontext_t * @@ -3003,8 +2983,20 @@ sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, } /* if no error occurred, we move the internal file pointer */ - fc->accessed_bytes += (sc_MPI_Offset) (block_size + - sc_scda_pad_to_mod_len (block_size)); + fc->accessed_bytes += (sc_MPI_Offset) block_size; + + /* read and check the data padding */ + if (fc->mpirank == root) { + const char *last_byte = (block_size > 0) ? &block_data->array[block_size - 1] : NULL; + + sc_scda_fread_mod_padding_serial (fc, last_byte, block_size, &count_err, + errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); + + /* update internal file pointer */ + fc->accessed_bytes += (sc_MPI_Offset) sc_scda_pad_to_mod_len (block_size); /* last function call can not be \ref sc_scda_fread_section_header anymore */ fc->header_before = 0; From 77ee080b66877430f31d3ae6e0171156aa10c0c7 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 18:06:42 +0200 Subject: [PATCH 24/43] feature-scda: First version of fread_array_data --- src/sc_scda.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/sc_scda.c b/src/sc_scda.c index 6e2610ab..c2d86796 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -3004,6 +3004,94 @@ sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, return fc; } +sc_scda_fcontext_t * +sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, + sc_array_t *elem_counts, size_t elem_size, + int indirect, sc_scda_ferror_t *errcode) +{ + int wrong_usage; + size_t elem_count, si; + size_t collective_byte_count; + sc_MPI_Offset offset; + int bytes_to_read; + int mpiret; + int count; + int count_err; + void *data; + + SC_ASSERT (fc != NULL); + SC_ASSERT (elem_counts != NULL); + SC_ASSERT (errcode != NULL); + + /* TODO: Temporary */ + SC_ASSERT (!indirect); + + /* check array_data; depends on indirect parameter */ + + /* check elem_counts */ + elem_count = 0; + for (si = 0; si < elem_counts->elem_count; ++si) { + elem_count += *((size_t *) sc_array_index (elem_counts, si)); + } + + /* check elem_size */ + + /* It is necessary that sc_scda_fread_section_header was called as last + * function call on fc and that it returned the array (A) section type. + */ + wrong_usage = !(fc->header_before && fc->last_type == 'A'); + sc_scda_scdaret_to_errcode (wrong_usage ? SC_SCDA_FERR_USAGE : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Wrong usage of scda functions"); + + if (array_data != NULL) { + /* on this rank the array data is not skipped */ + sc_scda_get_local_partition_index (fc, elem_counts, elem_size, &offset, + &bytes_to_read); + data = (void *) array_data->array; + } + else { + /* on this rank the array data is skipped */ + offset = 0; + bytes_to_read = 0; + /* By MPI standard 2.0 section 9.4.1 (cf. Data Access Conventions) the read + * buffer behaves as for the MPI 1.1 communication. + */ + data = NULL; + } + + /* collective read */ + mpiret = sc_io_read_at_all (fc->file, fc->accessed_bytes + offset, + data, bytes_to_read, sc_MPI_BYTE, + &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Reading fixed-length array data"); + /* check for count error of the collective I/O operation */ + SC_SCDA_CHECK_COLL_COUNT_ERR (bytes_to_read, count, fc, errcode); + + /* update internal file pointer */ + collective_byte_count = elem_count * elem_size; + fc->accessed_bytes += (sc_MPI_Offset) collective_byte_count; + + /* padding is always read and checked */ + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { + /* last data byte is read from the file if collective_byte_count > 0 */ + sc_scda_fread_mod_padding_serial (fc, NULL, collective_byte_count, + &count_err, errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); + + /* update internal file pointer */ + fc->accessed_bytes += (sc_MPI_Offset) sc_scda_pad_to_mod_len (collective_byte_count); + + /* last function call can not be \ref sc_scda_fread_section_header anymore */ + fc->header_before = 0; + + return fc; +} + int sc_scda_fclose (sc_scda_fcontext_t * fc, sc_scda_ferror_t * errcode) { From df801159f53aa1a314854fc4ed3d82a8e0f0fb93 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 7 Oct 2024 18:08:17 +0200 Subject: [PATCH 25/43] feature-scda: Test fread_array_data --- test/test_scda.c | 88 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 12 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index 57c0490b..32ccbc4d 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -95,28 +95,92 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, } static void -test_scda_read_fixed_size_array (sc_scda_fcontext_t *fc) +test_scda_read_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, + int mpisize) { -#if 0 const int indirect = 0; -#endif + int i; int decode; char read_user_string[SC_SCDA_USER_STRING_BYTES + 1]; char section_type; + char *data_ptr; size_t len; - size_t elem_count, elem_size; + size_t elem_count, elem_size, si; + size_t num_local_elements; sc_scda_ferror_t errcode; + sc_array_t array_data, elem_counts; + sc_scda_ulong per_proc_count, remainder_count; - fc = sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, - &elem_count, &elem_size, &decode, - &errcode); + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'A' + && elem_count == SC_SCDA_GLOBAL_ARRAY_COUNT + && elem_size == SC_SCDA_ARRAY_SIZE, + "Identifying section type"); + + /* read array data */ + sc_array_init (&array_data, elem_size); + sc_array_init_size (&elem_counts, sizeof (sc_scda_ulong), (size_t) mpisize); + + /* get the counts per process */ + per_proc_count = elem_count / (sc_scda_ulong) mpisize; + remainder_count = elem_count % (sc_scda_ulong) mpisize; + + /* set elem_counts, i.e. the reading partition */ + for (i = 0; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = + per_proc_count; + } + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpisize - 1)) += + remainder_count; + + /* allocate space for data that will be read */ + num_local_elements = + (size_t) *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpirank)); + sc_array_resize (&array_data, num_local_elements); + + /* read the array data */ + fc = sc_scda_fread_array_data (fc, &array_data, &elem_counts, elem_size, + indirect, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_array_data failed"); + + /* check read data */ + for (si = 0; si < num_local_elements; ++si) { + data_ptr = (char *) sc_array_index (&array_data, si); + SC_CHECK_ABORT (data_ptr[0] == 'a' && data_ptr[1] == 'b' && + data_ptr[2] == 'c', "sc_scda_fread_array_data data " + "mismatch"); + } + + sc_array_reset (&array_data); + sc_array_reset (&elem_counts); + + /* read the empty array */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "sc_scda_fread_section_header failed"); - SC_CHECK_ABORT (section_type == 'A' && - elem_count == SC_SCDA_GLOBAL_ARRAY_COUNT && - elem_size == SC_SCDA_ARRAY_SIZE, "Identifying section type"); + SC_CHECK_ABORT (section_type == 'A' && elem_count == 0 + && elem_size == SC_SCDA_ARRAY_SIZE, + "Identifying section type"); + + /* define trivial partition; a partition is always required */ + sc_array_init_count (&elem_counts, sizeof (sc_scda_ulong), (size_t) mpisize); + for (i = 0; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 0; + } - /* TODO: read the empty array */ + fc = sc_scda_fread_array_data (fc, NULL, &elem_counts, elem_size, indirect, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_array_data skip empty array failed"); + + sc_array_reset (&elem_counts); } int @@ -391,7 +455,7 @@ main (int argc, char **argv) || !strncmp (read_data, inline_data, SC_SCDA_INLINE_FIELD), "block data mismatch"); - test_scda_read_fixed_size_array (fc); + test_scda_read_fixed_size_array (fc, mpirank, mpisize); sc_scda_fclose (fc, &errcode); /* TODO: check errcode and return value */ From 8ec1e613c6206c0a4f51342ebcf9f27cb7cabe95 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 10:31:19 +0200 Subject: [PATCH 26/43] feature-scda: Adjust get_pad_to_mod to last byte semantic --- src/sc_scda.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index c2d86796..ea4cada1 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -638,6 +638,8 @@ static int sc_scda_get_pad_to_mod (const char *padded_data, size_t padded_len, size_t raw_len, char *raw_data) { + const char *last_byte; + SC_ASSERT (padded_data != NULL); SC_ASSERT (raw_len == 0 || raw_data != NULL); @@ -646,8 +648,9 @@ sc_scda_get_pad_to_mod (const char *padded_data, size_t padded_len, return -1; } - if (sc_scda_check_pad_to_mod (&padded_data[raw_len - 1], raw_len, - &padded_data[raw_len], padded_len - raw_len)) { + last_byte = (raw_len > 0) ? &padded_data[raw_len - 1] : NULL; + if (sc_scda_check_pad_to_mod (last_byte, raw_len, &padded_data[raw_len], + padded_len - raw_len)) { /* invalid padding bytes */ return -1; } From 868f1fbe8ddb69e287780f13df0a04e11699a355 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 10:53:19 +0200 Subject: [PATCH 27/43] feature-scda: Implement generic data padding function --- src/sc_scda.c | 112 ++++++++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 58 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index ea4cada1..934258f1 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1726,10 +1726,6 @@ sc_scda_fwrite_block_data_serial (sc_scda_fcontext_t *fc, int mpiret; int count; int invalid_block_data; - size_t num_pad_bytes; - /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ - char padding[SC_SCDA_PADDING_MOD + 6]; - const char *last_byte; *count_err = 0; @@ -1746,17 +1742,47 @@ sc_scda_fwrite_block_data_serial (sc_scda_fcontext_t *fc, sc_scda_mpiret_to_errcode (mpiret, errcode, fc); SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing block data"); SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); +} + +/** Internal function to write mod data padding. + * + * This function is only valid to be called on a rank that stores the global + * last data byte. For block sections, this is the root rank and for (v)array + * sections the maximal not empty rank. + * + * \param [in] fc The file context after writing the data that is + * intended to be padded. + * \param [in] last_byte A pointer to the last global data byte. For + * \b byte_count = 0 NULL is valid. + * \param [in] byte_count Number of (collectively) written bytes in the + * preceding data writing I/O operation. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fwrite_mod_padding_serial (sc_scda_fcontext_t *fc, + const char *last_byte, size_t byte_count, + int *count_err, sc_scda_ferror_t *errcode) +{ + int mpiret; + int count; + size_t num_pad_bytes; + /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ + char padding[SC_SCDA_PADDING_MOD + 6]; + + *count_err = 0; /* get the padding bytes */ - num_pad_bytes = sc_scda_pad_to_mod_len (block_size); - last_byte = (block_size > 0) ? &block_data->array[block_size - 1] : NULL; - sc_scda_pad_to_mod (last_byte, block_size, padding); + num_pad_bytes = sc_scda_pad_to_mod_len (byte_count); + sc_scda_pad_to_mod (last_byte, byte_count, padding); /* write the padding bytes */ - mpiret = sc_io_write_at (fc->file, fc->accessed_bytes + block_size, - padding, (int) num_pad_bytes, sc_MPI_BYTE, &count); + mpiret = sc_io_write_at (fc->file, fc->accessed_bytes, padding, + (int) num_pad_bytes, sc_MPI_BYTE, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing block data padding"); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing array data padding"); SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); } @@ -1766,7 +1792,6 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, int root, int encode, sc_scda_ferror_t * errcode) { int count_err; - size_t num_pad_bytes; sc_scda_ret_t ret; SC_ASSERT (fc != NULL); @@ -1799,17 +1824,29 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, /* The block data is written on the the user-defined rank root. */ if (fc->mpirank == root) { - /* This function includes writing the padding. */ sc_scda_fwrite_block_data_serial (fc, block_data, block_size, &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); - /* get number of padding bytes to update internal file pointer */ - num_pad_bytes = sc_scda_pad_to_mod_len (block_size); + /* update internal file pointer */ + fc->accessed_bytes += (sc_MPI_Offset) block_size; + + /* write the padding */ + if (fc->mpirank == root) { + const char *last_byte; + + last_byte = (block_size > 0) ? &block_data->array[block_size - 1] : NULL; + sc_scda_fwrite_mod_padding_serial (fc, last_byte, block_size, &count_err, + errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); + - fc->accessed_bytes += (sc_MPI_Offset) (block_size + num_pad_bytes); + /* get number of padding bytes to update internal file pointer */ + fc->accessed_bytes += (sc_MPI_Offset) sc_scda_pad_to_mod_len (block_size); return fc; } @@ -1895,47 +1932,6 @@ sc_scda_fwrite_array_header_serial (sc_scda_fcontext_t *fc, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (header_len, count, count_err); } -/** Internal function to write array padding. - * - * This function is dedicated to be called in \ref sc_scda_fwrite_array - * or in \ref sc_scda_fwrite_varray. - * This function is only valid to be called on the maximal not empty rank. - * - * \param [in] fc The file context as in \ref sc_scda_fwrite_array - * after running the collective write part. - * \param [in] last_byte A pointer to the last global data byte. - * \param [in] byte_count Number of collectively written bytes in the function - * \ref sc_scda_fwrite_array. - * \param [out] count_err A Boolean indicating if a count error occurred. - * \param [out] errcode An errcode that can be interpreted by \ref - * sc_scda_ferror_string or mapped to an error class - * by \ref sc_scda_ferror_class. - */ -static void -sc_scda_fwrite_array_padding_serial (sc_scda_fcontext_t *fc, - const char *last_byte, size_t byte_count, - int *count_err, sc_scda_ferror_t *errcode) -{ - int mpiret; - int count; - size_t num_pad_bytes; - /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ - char padding[SC_SCDA_PADDING_MOD + 6]; - - *count_err = 0; - - /* get the padding bytes */ - num_pad_bytes = sc_scda_pad_to_mod_len (byte_count); - sc_scda_pad_to_mod (last_byte, byte_count, padding); - - /* write the padding bytes */ - mpiret = sc_io_write_at (fc->file, fc->accessed_bytes, padding, - (int) num_pad_bytes, sc_MPI_BYTE, &count); - sc_scda_mpiret_to_errcode (mpiret, errcode, fc); - SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing array data padding"); - SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); -} - /** Determine the maximal rank that is not empty. * * \param [in] fc A file context with filled MPI data. @@ -2135,8 +2131,8 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, last_byte = (elem_count > 0) ? &array_data->array[bytes_to_write - 1] : NULL; /* the padding depends on the last data byte */ - sc_scda_fwrite_array_padding_serial (fc, last_byte, collective_byte_count, - &count_err, errcode); + sc_scda_fwrite_mod_padding_serial (fc, last_byte, collective_byte_count, + &count_err, errcode); } SC_SCDA_HANDLE_NONCOLL_ERR (errcode, last_byte_owner, fc); SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, last_byte_owner, fc); From d38dd5706cdfd7ab593eeb701353db01f38f71a5 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 11:05:12 +0200 Subject: [PATCH 28/43] feature-scda: Stricter semantics in write_mod_padding --- src/sc_scda.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 934258f1..8e3c0df4 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1746,14 +1746,14 @@ sc_scda_fwrite_block_data_serial (sc_scda_fcontext_t *fc, /** Internal function to write mod data padding. * - * This function is only valid to be called on a rank that stores the global + * This function is intended to be called on a rank that stores the global * last data byte. For block sections, this is the root rank and for (v)array * sections the maximal not empty rank. * * \param [in] fc The file context after writing the data that is * intended to be padded. * \param [in] last_byte A pointer to the last global data byte. For - * \b byte_count = 0 NULL is valid. + * \b byte_count = 0 \b last_byte must be NULL. * \param [in] byte_count Number of (collectively) written bytes in the * preceding data writing I/O operation. * \param [out] count_err A Boolean indicating if a count error occurred. @@ -1772,6 +1772,8 @@ sc_scda_fwrite_mod_padding_serial (sc_scda_fcontext_t *fc, /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ char padding[SC_SCDA_PADDING_MOD + 6]; + SC_ASSERT (byte_count != 0 || last_byte == NULL); + *count_err = 0; /* get the padding bytes */ From 9d427bf4847a1f5995f9817d3c3406745f49e434 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 11:07:57 +0200 Subject: [PATCH 29/43] feature-scda: Doc. fread_mod_padding --- src/sc_scda.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 8e3c0df4..ada26c0b 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1744,7 +1744,7 @@ sc_scda_fwrite_block_data_serial (sc_scda_fcontext_t *fc, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); } -/** Internal function to write mod data padding. +/** Internal function to write mod data padding in serial. * * This function is intended to be called on a rank that stores the global * last data byte. For block sections, this is the root rank and for (v)array @@ -2831,15 +2831,25 @@ sc_scda_fread_inline_data (sc_scda_fcontext_t *fc, sc_array_t *data, int root, return fc; } -/** TODO: Documentation - * @brief +/** Internal function to read mod data padding in serial. * - * @param fc - * @param last_byte If \b last_byte is NULL and byte_count > 0 the last data - * byte is read from file. - * @param byte_count - * @param count_err - * @param errcode + * The recommended usage of this function is that for (v)array sections one + * passes NULL for \b last_byte to read the last global data byte. + * For block sections, we can read on the root rank and explicitly pass the + * last global byte. + * + * \param [in] fc The file context after reading the data that is + * padded. + * \param [in] last_byte A pointer to the last global data byte. For + * \b byte_count = 0 \b last_byte must be NULL. + * If \b last_byte is NULL and \b byte_count > 0, + * the function reads the last data byte from the file. + * \param [in] byte_count Number of (collectively) read bytes in the + * preceding data read I/O operation. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. */ static void sc_scda_fread_mod_padding_serial (sc_scda_fcontext_t *fc, @@ -2988,8 +2998,9 @@ sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, /* read and check the data padding */ if (fc->mpirank == root) { - const char *last_byte = (block_size > 0) ? &block_data->array[block_size - 1] : NULL; + const char *last_byte; + last_byte = (block_size > 0) ? &block_data->array[block_size - 1] : NULL; sc_scda_fread_mod_padding_serial (fc, last_byte, block_size, &count_err, errcode); } From 37adb155312e876e7c6bc10abd9e83c72ab6cd5a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 16:07:37 +0200 Subject: [PATCH 30/43] feature-scda: Implement indirect array writing --- src/sc_scda.c | 61 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index ada26c0b..05357433 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2019,17 +2019,18 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, { int mpiret; int invalid_elem_counts, global_invalid_elem_counts; + int invalid_array_data, global_invalid_array_data; int ret, count_err; int bytes_to_write; int count; int last_byte_owner; size_t elem_count, si; + size_t local_elem_count; size_t collective_byte_count; size_t num_pad_bytes; sc_MPI_Offset offset; -#if 0 + sc_array_t contig_arr; const void *local_array_data; -#endif SC_ASSERT (fc != NULL); SC_ASSERT (user_string != NULL); @@ -2055,7 +2056,7 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, sc_MPI_Allreduce (&invalid_elem_counts, &global_invalid_elem_counts, 1, sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); SC_CHECK_MPI (mpiret); - sc_scda_scdaret_to_errcode (invalid_elem_counts ? SC_SCDA_FERR_ARG : + sc_scda_scdaret_to_errcode (global_invalid_elem_counts ? SC_SCDA_FERR_ARG : SC_SCDA_FERR_SUCCESS, errcode, fc); SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Invalid elem_counts array"); @@ -2074,8 +2075,27 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, "fwrite_array: elem_counts or elem_size" " is not collective"); - /* TODO: check array_data; depends on indirect parameter */ - /* Call Allreduce to synchronize on check of array_data or collective test? */ + /* check array_data; depends on indirect parameter */ + local_elem_count = + (size_t) *((sc_scda_ulong *) + sc_array_index_int (elem_counts, fc->mpirank)); + if (indirect) { + invalid_array_data = !(array_data->elem_count == local_elem_count && + array_data->elem_size == sizeof (sc_array_t)); + /* The arrays with the actual data are checked in an asserttion below. */ + } + else { + invalid_array_data = !(array_data->elem_count == local_elem_count && + array_data->elem_size == elem_size); + } + /* synchronize */ + mpiret = + sc_MPI_Allreduce (&invalid_array_data, &global_invalid_array_data, 1, + sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); + SC_CHECK_MPI (mpiret); + sc_scda_scdaret_to_errcode (global_invalid_array_data ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Invalid array_data array"); /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { @@ -2089,11 +2109,24 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* add number of written bytes */ fc->accessed_bytes += SC_SCDA_COMMON_FIELD + 2 * SC_SCDA_COUNT_FIELD; -#if 0 /* get pointer to local array data */ if (indirect) { + sc_array_t *data_arr; + char *data_src, *data_dest; + /* indirect addressing */ - /* TODO: copy the data or use muliple write calls; maybe in batches? */ + /* TODO: Use batches */ + sc_array_init_size (&contig_arr, elem_size, elem_count); + for (si = 0; si < array_data->elem_count; ++si) { + data_arr = (sc_array_t *) sc_array_index (array_data, si); + SC_ASSERT (data_arr->elem_size == elem_size + && data_arr->elem_count == 1); + data_src = (char *) sc_array_index (data_arr, 0); + data_dest = (char *) sc_array_index (&contig_arr, si); + + sc_scda_copy_bytes (data_dest, data_src, elem_size); + } + local_array_data = (const void *) contig_arr.array; } else { /* direct addressing */ @@ -2101,17 +2134,17 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, } /* write array data in parallel */ -#endif - - /* TODO: temporary */ - SC_ASSERT (!indirect); + /* retrieve partition information */ sc_scda_get_local_partition_index (fc, elem_counts, elem_size, &offset, &bytes_to_write); mpiret = sc_io_write_at_all (fc->file, fc->accessed_bytes + offset, - array_data->array, bytes_to_write, sc_MPI_BYTE, + local_array_data, bytes_to_write, sc_MPI_BYTE, &count); + if (indirect) { + sc_array_reset (&contig_arr); + } sc_scda_mpiret_to_errcode (mpiret, errcode, fc); SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Writing fixed-length array data"); /* check for count error of the collective I/O operation */ @@ -2126,12 +2159,12 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, /* get and write padding bytes in serial */ if (fc->mpirank == last_byte_owner) { - const char *last_byte; + const char *last_byte; /* get last local/global byte */ SC_ASSERT (elem_count == 0 || bytes_to_write > 0); last_byte = (elem_count > 0) ? - &array_data->array[bytes_to_write - 1] : NULL; + &array_data->array[bytes_to_write - 1] : NULL; /* the padding depends on the last data byte */ sc_scda_fwrite_mod_padding_serial (fc, last_byte, collective_byte_count, &count_err, errcode); From dcaacf9ebe2e630b37c7cbcea2b472294e931e41 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 16:09:02 +0200 Subject: [PATCH 31/43] feature-scda: Test indirect array writing --- test/test_scda.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/test/test_scda.c b/test/test_scda.c index 32ccbc4d..08591848 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -94,6 +94,64 @@ test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, sc_array_reset (&data); } +static void +test_scda_write_indirect_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, + int mpisize) +{ + int i; + const int indirect = 1; + const size_t elem_size = SC_SCDA_ARRAY_SIZE; + const sc_scda_ulong global_elem_count = SC_SCDA_GLOBAL_ARRAY_COUNT; + char *data_ptr; + size_t si; + size_t local_elem_count; + sc_scda_ulong per_proc_count, remainder_count; + sc_array_t elem_counts, data, *curr; + sc_scda_ferror_t errcode; + + sc_array_init_count (&elem_counts, sizeof (sc_scda_ulong), + (size_t) mpisize); + + /* get the counts per process */ + per_proc_count = global_elem_count / (sc_scda_ulong) mpisize; + remainder_count = global_elem_count % (sc_scda_ulong) mpisize; + + /* set elem_counts */ + for (i = 0; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = + per_proc_count; + } + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpisize - 1)) += + remainder_count; + + /* create local data */ + local_elem_count = + (size_t) *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpirank)); + sc_array_init_size (&data, sizeof (sc_array_t), local_elem_count); + for (si = 0; si < local_elem_count; ++si) { + curr = (sc_array_t *) sc_array_index (&data, si); + sc_array_init_size (curr, elem_size, 1); + data_ptr = (char *) sc_array_index (curr, 0); + data_ptr[0] = 'c'; + data_ptr[1] = 'b'; + data_ptr[2] = 'a'; + } + + fc = sc_scda_fwrite_array (fc, "Another fixed-length array section", NULL, + &data, &elem_counts, elem_size, indirect, 0, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fwrite_array failed"); + + /* free memory */ + for (si = 0; si < local_elem_count; ++si) { + curr = (sc_array_t *) sc_array_index (&data, si); + sc_array_reset (curr); + } + sc_array_reset (&data); + sc_array_reset (&elem_counts); +} + static void test_scda_read_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, int mpisize) @@ -347,6 +405,8 @@ main (int argc, char **argv) /* write a fixed-size array section */ test_scda_write_fixed_size_array (fc, mpirank, mpisize); + test_scda_write_indirect_fixed_size_array (fc, mpirank, mpisize); + /* intentionally try to write with non-collective block size */ if (mpisize > 1) { SC_GLOBAL_ESSENTIAL From 53085d2780f1629d0a782c75b88ab31f856d270f Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 17:05:09 +0200 Subject: [PATCH 32/43] feature-scda: Implement indirect array reading --- src/sc_scda.c | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 05357433..d50dd3e0 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -3063,14 +3063,12 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, int count; int count_err; void *data; + sc_array_t conti_arr; SC_ASSERT (fc != NULL); SC_ASSERT (elem_counts != NULL); SC_ASSERT (errcode != NULL); - /* TODO: Temporary */ - SC_ASSERT (!indirect); - /* check array_data; depends on indirect parameter */ /* check elem_counts */ @@ -3105,10 +3103,19 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, data = NULL; } + if (indirect) { + /* TODO: batches */ + /* read to contiguous temporary buffer */ + sc_array_init_size (&conti_arr, elem_size, elem_count); + data = (void *) conti_arr.array; + } + else { + /* read to passed data buffer, i.e. data stays unchanged */ + } + /* collective read */ mpiret = sc_io_read_at_all (fc->file, fc->accessed_bytes + offset, - data, bytes_to_read, sc_MPI_BYTE, - &count); + data, bytes_to_read, sc_MPI_BYTE, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Reading fixed-length array data"); /* check for count error of the collective I/O operation */ @@ -3129,7 +3136,24 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, fc); /* update internal file pointer */ - fc->accessed_bytes += (sc_MPI_Offset) sc_scda_pad_to_mod_len (collective_byte_count); + fc->accessed_bytes += + (sc_MPI_Offset) sc_scda_pad_to_mod_len (collective_byte_count); + + if (indirect) { + sc_array_t *curr; + const char *src; + char *dest; + + /* copy contiguous data to indirect array */ + for (si = 0; si < array_data->elem_count; ++si) { + src = (const char *) sc_array_index (&conti_arr, si); + curr = (sc_array_t *) sc_array_index (array_data, si); + dest = (char *) sc_array_index (curr, 0); + + sc_scda_copy_bytes (dest, src, elem_size); + } + sc_array_reset (&conti_arr); + } /* last function call can not be \ref sc_scda_fread_section_header anymore */ fc->header_before = 0; From fa7f0a91d478a510500ac868524713dcc1cd2d00 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 8 Oct 2024 17:08:08 +0200 Subject: [PATCH 33/43] feature-scda: Test indirect array reading --- test/test_scda.c | 86 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index 08591848..279bbe81 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -241,6 +241,81 @@ test_scda_read_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, sc_array_reset (&elem_counts); } +static void +test_scda_read_indirect_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, + int mpisize) +{ + const int indirect = 1; + int i; + int decode; + char read_user_string[SC_SCDA_USER_STRING_BYTES + 1]; + char section_type; + char *data_ptr; + size_t len; + size_t elem_count, elem_size, si; + size_t num_local_elements; + sc_array_t array_data, elem_counts, *curr; + sc_scda_ulong per_proc_count, remainder_count; + sc_scda_ferror_t errcode; + + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'A' + && elem_count == SC_SCDA_GLOBAL_ARRAY_COUNT + && elem_size == SC_SCDA_ARRAY_SIZE, + "Identifying section type"); + + /* read array data */ + sc_array_init (&array_data, sizeof (sc_array_t)); + sc_array_init_size (&elem_counts, sizeof (sc_scda_ulong), (size_t) mpisize); + + /* get the counts per process */ + per_proc_count = elem_count / (sc_scda_ulong) mpisize; + remainder_count = elem_count % (sc_scda_ulong) mpisize; + + /* set elem_counts, i.e. the reading partition */ + for (i = 0; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = + per_proc_count; + } + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpisize - 1)) += + remainder_count; + + /* allocate space for data that will be read */ + num_local_elements = + (size_t) *((sc_scda_ulong *) sc_array_index_int (&elem_counts, mpirank)); + sc_array_resize (&array_data, num_local_elements); + for (si = 0; si < num_local_elements; ++si) { + curr = (sc_array_t *) sc_array_index (&array_data, si); + sc_array_init_size (curr, elem_size, 1); + } + + fc = sc_scda_fread_array_data (fc, &array_data, &elem_counts, elem_size, + indirect, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_array_data skip empty array failed"); + + /* check read data */ + for (si = 0; si < num_local_elements; ++si) { + curr = (sc_array_t *) sc_array_index (&array_data, si); + data_ptr = (char *) sc_array_index (curr, 0); + SC_CHECK_ABORT (data_ptr[0] == 'c' && data_ptr[1] == 'b' && + data_ptr[2] == 'a', + "sc_scda_fread_array_data indirect data mismatch"); + } + + /* free memory */ + for (si = 0; si < num_local_elements; ++si) { + curr = (sc_array_t *) sc_array_index (&array_data, si); + sc_array_reset (curr); + } + sc_array_reset (&array_data); + sc_array_reset (&elem_counts); +} + int main (int argc, char **argv) { @@ -378,8 +453,9 @@ main (int argc, char **argv) /* write a block section to the file */ block_size = strlen (block_data); sc_array_init_data (&data, (void *) block_data, block_size, 1); - fc = sc_scda_fwrite_block (fc, "Block section test", NULL, &data, block_size, - mpisize - 1, 0, &errcode); + fc = + sc_scda_fwrite_block (fc, "Block section test", NULL, &data, block_size, + mpisize - 1, 0, &errcode); SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "scda_fwrite_block failed"); @@ -414,8 +490,8 @@ main (int argc, char **argv) " This is just for testing purposes and do not imply" " erroneous code behavior.\n"); fc = sc_scda_fwrite_block (fc, "A block section", NULL, &data, - (mpirank == 0) ? 32 : 33, mpisize - 1, 0, - &errcode); + (mpirank == 0) ? 32 : 33, mpisize - 1, 0, + &errcode); SC_CHECK_ABORT (!sc_scda_ferror_is_success (errcode) && errcode.scdaret == SC_SCDA_FERR_ARG, "scda_fwrite_block " "check catch non-collective block size"); @@ -517,6 +593,8 @@ main (int argc, char **argv) test_scda_read_fixed_size_array (fc, mpirank, mpisize); + test_scda_read_indirect_fixed_size_array (fc, mpirank, mpisize); + sc_scda_fclose (fc, &errcode); /* TODO: check errcode and return value */ SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), From 16095939b400ecad30ac6796bfb83949f0796c8f Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 9 Oct 2024 15:17:37 +0200 Subject: [PATCH 34/43] feature-scda: Fix buffer merge --- src/sc_scda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index d50dd3e0..4bf3c282 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -302,7 +302,7 @@ sc_scda_merge_data_to_buf (const char *d1, size_t len1, const char *d2, } if (d3 != NULL) { SC_ASSERT (d2 != NULL); - sc_scda_copy_bytes (&out[len2], d3, len3); + sc_scda_copy_bytes (&out[len1 + len2], d3, len3); } } From 3b4b05cbd96cd39d01f052e83ab46bda8af62a08 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 9 Oct 2024 15:27:57 +0200 Subject: [PATCH 35/43] feature-scda: Generic array param. check function --- src/sc_scda.c | 176 +++++++++++++++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 65 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 4bf3c282..4ea4607b 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2011,42 +2011,43 @@ sc_scda_get_local_partition_index (sc_scda_fcontext_t *fc, *num_bytes = (int) elem_size * num_local_elements; } -sc_scda_fcontext_t * -sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, - size_t *len, sc_array_t *array_data, - sc_array_t *elem_counts, size_t elem_size, int indirect, - int encode, sc_scda_ferror_t *errcode) +/** Collective check of array function parameters. + * + * This function is dedicated to be called in \ref sc_scda_fwrite_array and + * \ref sc_scda_fread_array_data. + * + * \param [in] fc The file context as passed to \ref + * sc_scda_fwrite_array or \ref + * sc_scda_fread_array_data. + * \param [in] array_data As passed to \ref sc_scda_fwrite_array + * \ref sc_scda_fread_array_data, respectively. + * \param [in] indirect As passed to \ref sc_scda_fwrite_array + * \ref sc_scda_fread_array_data, respectively. + * \param [in] elem_size As passed to \ref sc_scda_fwrite_array + * \ref sc_scda_fread_array_data, respectively. + * \param [out] elem_count On successful output the global element count. + * \return \ref SC_SCDA_FERR_ARG if the parameters are not + * valid, and \ref SC_SCDA_FERR_SUCCESS otherwise. + */ +static sc_scda_ret_t +sc_scda_check_array_params (sc_scda_fcontext_t *fc, sc_array_t *array_data, + int indirect, sc_array_t *elem_counts, + size_t elem_size, size_t *elem_count) { int mpiret; int invalid_elem_counts, global_invalid_elem_counts; int invalid_array_data, global_invalid_array_data; - int ret, count_err; - int bytes_to_write; - int count; - int last_byte_owner; - size_t elem_count, si; - size_t local_elem_count; - size_t collective_byte_count; - size_t num_pad_bytes; - sc_MPI_Offset offset; - sc_array_t contig_arr; - const void *local_array_data; + int ret; + int skip_data; + size_t si, local_elem_count; SC_ASSERT (fc != NULL); - SC_ASSERT (user_string != NULL); - SC_ASSERT (array_data != NULL); SC_ASSERT (elem_counts != NULL); - SC_ASSERT (errcode != NULL); - /* TODO: Also check elem_counts and/or indirect? */ - /* check if elem_size is collective */ - ret = sc_scda_check_coll_params (fc, (const char *) &elem_size, - sizeof (size_t), NULL, 0, NULL, 0); - sc_scda_scdaret_to_errcode (ret, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_array: elem_size is not " - "collective"); + *elem_count = 0; - /* TODO: respect encode parameter */ + /* check if array_data is skipped */ + skip_data = array_data == NULL; /* check elem_counts array */ invalid_elem_counts = !(elem_counts->elem_size == sizeof (sc_scda_ulong) && @@ -2056,46 +2057,95 @@ sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, sc_MPI_Allreduce (&invalid_elem_counts, &global_invalid_elem_counts, 1, sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); SC_CHECK_MPI (mpiret); - sc_scda_scdaret_to_errcode (global_invalid_elem_counts ? SC_SCDA_FERR_ARG : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Invalid elem_counts array"); + if (global_invalid_elem_counts) { + /* invalid elem_counts array */ + return SC_SCDA_FERR_ARG; + } /* compute the global element count */ /* TODO: Use a checksum for the colletivness test? */ - elem_count = 0; for (si = 0; si < elem_counts->elem_count; ++si) { - elem_count += *((size_t *) sc_array_index (elem_counts, si)); + *elem_count += *((size_t *) sc_array_index (elem_counts, si)); } - ret = sc_scda_check_coll_params (fc, (const char *) &elem_count, + /* check if elem_count, elem_size and indirect are collective */ + ret = sc_scda_check_coll_params (fc, (const char *) elem_count, sizeof (size_t), (const char *) &elem_size, - sizeof (size_t), NULL, 0); - sc_scda_scdaret_to_errcode (ret, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, - "fwrite_array: elem_counts or elem_size" - " is not collective"); - - /* check array_data; depends on indirect parameter */ - local_elem_count = - (size_t) *((sc_scda_ulong *) - sc_array_index_int (elem_counts, fc->mpirank)); - if (indirect) { - invalid_array_data = !(array_data->elem_count == local_elem_count && - array_data->elem_size == sizeof (sc_array_t)); - /* The arrays with the actual data are checked in an asserttion below. */ + sizeof (size_t), (const char *) &indirect, + sizeof (int)); + if (ret != SC_SCDA_FERR_SUCCESS) { + return SC_SCDA_FERR_ARG; } - else { - invalid_array_data = !(array_data->elem_count == local_elem_count && - array_data->elem_size == elem_size); + + if (!skip_data) { + /* check array_data; depends on indirect parameter */ + local_elem_count = + (size_t) *((sc_scda_ulong *) + sc_array_index_int (elem_counts, fc->mpirank)); + if (indirect) { + invalid_array_data = !(array_data->elem_count == local_elem_count && + array_data->elem_size == sizeof (sc_array_t)); + /* The arrays with the actual data are checked later in an assertion. */ + } + else { + invalid_array_data = !(array_data->elem_count == local_elem_count && + array_data->elem_size == elem_size); + } + /* synchronize */ + mpiret = + sc_MPI_Allreduce (&invalid_array_data, &global_invalid_array_data, 1, + sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); + SC_CHECK_MPI (mpiret); + if (global_invalid_array_data) { + /* invalid array_data array */ + return SC_SCDA_FERR_ARG; + } } - /* synchronize */ - mpiret = + else { + /* no data is read on this rank and hence there is no array_data array */ + /* never causes invalid array_data */ + invalid_array_data = 0; + mpiret = sc_MPI_Allreduce (&invalid_array_data, &global_invalid_array_data, 1, sc_MPI_INT, sc_MPI_LOR, fc->mpicomm); - SC_CHECK_MPI (mpiret); - sc_scda_scdaret_to_errcode (global_invalid_array_data ? SC_SCDA_FERR_ARG : - SC_SCDA_FERR_SUCCESS, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Invalid array_data array"); + SC_CHECK_MPI (mpiret); + } + + return SC_SCDA_FERR_SUCCESS; +} + +sc_scda_fcontext_t * +sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, + size_t *len, sc_array_t *array_data, + sc_array_t *elem_counts, size_t elem_size, int indirect, + int encode, sc_scda_ferror_t *errcode) +{ + int mpiret; + int count_err; + int bytes_to_write; + int count; + int last_byte_owner; + sc_scda_ret_t scdaret; + size_t elem_count, si; + size_t collective_byte_count; + size_t num_pad_bytes; + sc_MPI_Offset offset; + sc_array_t contig_arr; + const void *local_array_data; + + SC_ASSERT (fc != NULL); + SC_ASSERT (user_string != NULL); + SC_ASSERT (array_data != NULL); + SC_ASSERT (elem_counts != NULL); + SC_ASSERT (errcode != NULL); + + /* TODO: respect encode parameter */ + + /* check function parameters */ + scdaret = sc_scda_check_array_params (fc, array_data, indirect, elem_counts, + elem_size, &elem_count); + sc_scda_scdaret_to_errcode (scdaret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_array: Invalid parameters"); /* section header is always written and read on rank SC_SCDA_HEADER_ROOT */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { @@ -3058,6 +3108,7 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, size_t elem_count, si; size_t collective_byte_count; sc_MPI_Offset offset; + sc_scda_ret_t scdaret; int bytes_to_read; int mpiret; int count; @@ -3069,15 +3120,10 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, SC_ASSERT (elem_counts != NULL); SC_ASSERT (errcode != NULL); - /* check array_data; depends on indirect parameter */ - - /* check elem_counts */ - elem_count = 0; - for (si = 0; si < elem_counts->elem_count; ++si) { - elem_count += *((size_t *) sc_array_index (elem_counts, si)); - } - - /* check elem_size */ + scdaret = sc_scda_check_array_params (fc, array_data, indirect, elem_counts, + elem_size, &elem_count); + sc_scda_scdaret_to_errcode (scdaret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fread_array: Invalid parameters"); /* It is necessary that sc_scda_fread_section_header was called as last * function call on fc and that it returned the array (A) section type. From 5bc0bc8be098fc7f3abfe063b5bf9ebb74e02939 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 9 Oct 2024 16:00:52 +0200 Subject: [PATCH 36/43] feature-scda: Fix last byte in case of skipping --- src/sc_scda.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 4ea4607b..085f7fc6 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -3083,7 +3083,9 @@ sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, if (fc->mpirank == root) { const char *last_byte; - last_byte = (block_size > 0) ? &block_data->array[block_size - 1] : NULL; + last_byte = (block_size > 0 + && block_data != + NULL) ? &block_data->array[block_size - 1] : NULL; sc_scda_fread_mod_padding_serial (fc, last_byte, block_size, &count_err, errcode); } From 80ca18004dea4cdc773f728443ff3d36b90592ca Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 9 Oct 2024 16:34:23 +0200 Subject: [PATCH 37/43] feature-scda: Fix indirect and skipping combination --- src/sc_scda.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 085f7fc6..0d98dde3 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -3151,7 +3151,7 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, data = NULL; } - if (indirect) { + if (indirect && array_data != NULL) { /* TODO: batches */ /* read to contiguous temporary buffer */ sc_array_init_size (&conti_arr, elem_size, elem_count); @@ -3187,7 +3187,7 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, fc->accessed_bytes += (sc_MPI_Offset) sc_scda_pad_to_mod_len (collective_byte_count); - if (indirect) { + if (indirect && array_data != NULL) { sc_array_t *curr; const char *src; char *dest; From 9fc55fbf12bb8e649787150df09d66b6b1d0c738 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 9 Oct 2024 16:37:14 +0200 Subject: [PATCH 38/43] feature-scda: Extend testing --- test/test_scda.c | 171 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/test/test_scda.c b/test/test_scda.c index 279bbe81..7312b9bd 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -30,6 +30,173 @@ #define SC_SCDA_GLOBAL_ARRAY_COUNT 12 #define SC_SCDA_ARRAY_SIZE 3 +static void +test_scda_skip_through_file (sc_MPI_Comm mpicomm, const char *filename, + sc_scda_fopen_options_t *opt, int mpirank, + int mpisize) +{ + sc_scda_fcontext_t *fc; + char read_user_string[SC_SCDA_USER_STRING_BYTES + 1]; + char section_type; + size_t len; + size_t elem_count, elem_size; + int i; + int decode; + sc_scda_ferror_t errcode; + sc_array_t elem_counts, data; + + /* TODO, also non-collective skip */ + /* open the file for reading */ + fc = sc_scda_fopen_read (mpicomm, filename, read_user_string, &len, opt, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "sc_scda_fopen_read " + "failed"); + + /* skip first file section */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'B' && elem_count == 0 + && elem_size == 15, "Identifying section type"); + + /* skip block data */ + fc = sc_scda_fread_block_data (fc, NULL, elem_size, 0, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_block_data failed"); + + /* skip second file section */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'I' && elem_count == 0 + && elem_size == 0, "Identifying section type"); + + fc = sc_scda_fread_inline_data (fc, NULL, mpisize - 1, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_inline_data failed"); + + /* skip third file section */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'I' && elem_count == 0 + && elem_size == 0, "Identifying section type"); + + fc = sc_scda_fread_inline_data (fc, NULL, mpisize - 1, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_inline_data failed"); + + /* skip fourth file section */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'B' && elem_count == 0 + && elem_size == 32, "Identifying section type"); + + fc = sc_scda_fread_block_data (fc, NULL, elem_size, 0, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_block_data failed"); + + /* skip fifth file section */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'A' && elem_count == 12 + && elem_size == 3, "Identifying section type"); + + /* define reading partition */ + sc_array_init_size (&elem_counts, sizeof (sc_scda_ulong), (size_t) mpisize); + SC_ASSERT (elem_count > 2); + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, 0)) = + (mpisize > 1) ? elem_count - 2 : elem_count; + if (mpisize > 1) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, 1)) = 2; + } + for (i = 2; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 0; + } + + /* non-collective skipping */ + if (mpirank == 1) { + sc_array_init_size (&data, elem_size, 2); + } + fc = sc_scda_fread_array_data (fc, (mpirank == 1) ? &data : NULL, + &elem_counts, elem_size, 0, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_block_data failed"); + + /* check data on rank 1 */ + if (mpirank == 1) { + size_t si; + char *data_ptr; + + for (si = 0; si < data.elem_count; ++si) { + data_ptr = (char *) sc_array_index (&data, si); + SC_CHECK_ABORT (data_ptr[0] == 'a' && data_ptr[1] == 'b' && + data_ptr[2] == 'c', "data mismatch for non-collective" + " skipping"); + } + sc_array_reset (&data); + } + + /* skip sixth file section */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'A' && elem_count == 0 + && elem_size == 3, "Identifying section type"); + + /* define reading partition */ + for (i = 0; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 0; + } + + fc = sc_scda_fread_array_data (fc, NULL, &elem_counts, elem_size, 1, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_array_data failed"); + + /* skip sixth file section */ + fc = + sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'A' && elem_count == 12 + && elem_size == 3, "Identifying section type"); + + /* define reading partition */ + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, 0)) = + (sc_scda_ulong) elem_count; + for (i = 1; i < mpisize; ++i) { + *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 0; + } + + fc = sc_scda_fread_array_data (fc, NULL, &elem_counts, elem_size, 1, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_array_data failed"); + + sc_array_reset (&elem_counts); + + sc_scda_fclose (fc, &errcode); + /* TODO: check errcode and return value */ + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "scda_fclose after read failed"); +} + static void test_scda_write_fixed_size_array (sc_scda_fcontext_t *fc, int mpirank, int mpisize) @@ -616,6 +783,10 @@ main (int argc, char **argv) "sc_scda_fread_section_header error detection failed"); /* fc is closed and deallocated due to the occurred error */ + /* skip through file and test non-collective skipping */ + test_scda_skip_through_file (mpicomm, filename, &scda_opt, mpirank, + mpisize); + sc_options_destroy (opt); sc_finalize (); From d6f04e3f336ba663ad5debef3bf7e33f65fb77b0 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 9 Oct 2024 16:43:16 +0200 Subject: [PATCH 39/43] feature-scda: Comment on impossible empty block --- src/sc_scda.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sc_scda.h b/src/sc_scda.h index e5aead2e..5178ffaf 100644 --- a/src/sc_scda.h +++ b/src/sc_scda.h @@ -429,6 +429,10 @@ sc_scda_fcontext_t *sc_scda_fwrite_inline (sc_scda_fcontext_t * fc, * \param [in] block_data On rank \b root a sc_array with one element and * element size equals to \b block_size. On all * other ranks the parameter is ignored. + * Since \ref sc_array_init and the other \ref + * sc_array_t initialization functions do not allow + * an element size of 0, block sections can not be + * empty. * \param [in] block_size The size of the data block in bytes. Must be * less or equal than 10^{26} - 1. * \param [in] root An integer between 0 and mpisize of the MPI From 7f9965861cb00af206f792d65d7d76974d5b2522 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 10 Oct 2024 10:02:53 +0200 Subject: [PATCH 40/43] feature-scda: Polish comments --- src/sc_scda.c | 2 +- test/test_scda.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 0d98dde3..d6a24c62 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -3152,7 +3152,7 @@ sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, } if (indirect && array_data != NULL) { - /* TODO: batches */ + /* TODO: batches? */ /* read to contiguous temporary buffer */ sc_array_init_size (&conti_arr, elem_size, elem_count); data = (void *) conti_arr.array; diff --git a/test/test_scda.c b/test/test_scda.c index 7312b9bd..e467ede8 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -45,7 +45,6 @@ test_scda_skip_through_file (sc_MPI_Comm mpicomm, const char *filename, sc_scda_ferror_t errcode; sc_array_t elem_counts, data; - /* TODO, also non-collective skip */ /* open the file for reading */ fc = sc_scda_fopen_read (mpicomm, filename, read_user_string, &len, opt, &errcode); @@ -159,6 +158,7 @@ test_scda_skip_through_file (sc_MPI_Comm mpicomm, const char *filename, && elem_size == 3, "Identifying section type"); /* define reading partition */ + /* Even if all array data is skipped, a valid reading partition is required. */ for (i = 0; i < mpisize; ++i) { *((sc_scda_ulong *) sc_array_index_int (&elem_counts, i)) = 0; } From 436e58a124e387d44d3e63d326147e16a6f1f27a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 10 Oct 2024 10:18:32 +0200 Subject: [PATCH 41/43] feature-scda: Missing type cast --- src/sc_scda.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index d6a24c62..d0569ba3 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -563,7 +563,7 @@ sc_scda_pad_to_mod_inplace (const char *input_data, size_t input_len, * requires this byte. * * \param [in] last_byte Pointer to the last data byte. - * This byte is required since the padding byte + * This byte is required since the padding * content depends on the trailing data byte. * \param [in] data_len The raw data length in number of bytes. * \param [in] pad The padding bytes with byte count \b pad_len. @@ -2051,7 +2051,7 @@ sc_scda_check_array_params (sc_scda_fcontext_t *fc, sc_array_t *array_data, /* check elem_counts array */ invalid_elem_counts = !(elem_counts->elem_size == sizeof (sc_scda_ulong) && - elem_counts->elem_count == fc->mpisize); + elem_counts->elem_count == (size_t) fc->mpisize); /* synchronize */ mpiret = sc_MPI_Allreduce (&invalid_elem_counts, &global_invalid_elem_counts, 1, From 18b13a1162078b47dee5078d9421b6ac83aed7ff Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 10 Oct 2024 10:45:29 +0200 Subject: [PATCH 42/43] feature-scda: Stricter assert and polishing --- src/sc_scda.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index d0569ba3..83d31c8a 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1940,6 +1940,8 @@ sc_scda_fwrite_array_header_serial (sc_scda_fcontext_t *fc, * \param [in] elem_counts As the parameter \b elem_counts in \ref * sc_scda_fwrite_array or \ref sc_scda_fwrite_varray. * This array stores the number of elements per rank. + * The element count of \b elem_counts must be equal + * to \b fc->mpisize. * \return The maximal rank that is not empty. If all ranks * are empty, 0 is returned. */ @@ -1951,6 +1953,8 @@ sc_scda_get_last_byte_owner (sc_scda_fcontext_t *fc, sc_array_t *elem_counts) SC_ASSERT (fc != NULL); SC_ASSERT (elem_counts != NULL); + SC_ASSERT (elem_counts->elem_size == sizeof (sc_scda_ulong)); + SC_ASSERT ((size_t) fc->mpisize == elem_counts->elem_count); /* determine the rank that holds the last byte */ last_byte_owner = 0; @@ -1993,7 +1997,7 @@ sc_scda_get_local_partition_index (sc_scda_fcontext_t *fc, SC_ASSERT (fc != NULL); SC_ASSERT (elem_counts != NULL); SC_ASSERT (elem_counts->elem_size == sizeof (sc_scda_ulong)); - SC_ASSERT ((int) elem_counts->elem_count == fc->mpisize); + SC_ASSERT ((size_t) fc->mpisize == elem_counts->elem_count); /* compute rank-dependent offset */ *offset = 0; @@ -2005,7 +2009,7 @@ sc_scda_get_local_partition_index (sc_scda_fcontext_t *fc, } *offset *= (sc_MPI_Offset) elem_size; - /* computer number of local array data bytes */ + /* compute the number of local array data bytes */ num_local_elements = (int) *((sc_scda_ulong *) sc_array_index_int (elem_counts, fc->mpirank)); *num_bytes = (int) elem_size * num_local_elements; From 7d11bb8e9e62c17638ca46b25e301b693b5dcf8f Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 10 Oct 2024 11:21:54 +0200 Subject: [PATCH 43/43] feature-scda: check_count_entry name and doc. --- src/sc_scda.c | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 83d31c8a..7cbc712c 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2513,28 +2513,24 @@ sc_scda_fread_section_header_common_serial (sc_scda_fcontext_t *fc, /** Internal function to check a count entry in a section header. * - * \param [in] fc The file context as in \ref - * sc_scda_fread_section_header before running the - * second serial code part. - * \param [out] ident The character that identifies the count entry. - * If this character is not conforming to the scda - * convention, the function call is not completed and - * results in \ref SC_SCDA_FERR_FORMAT as error, - * cf. \b errcode. + * This function checks if the count entry is conforming to the scda format. + * + * \param [in] count_entry A pointer to a count entry that was read from file. + * The count entry has exactly \ref SC_SCDA_COUNT_FIELD + * bytes. * \param [in] expc_ident The expected count entry identifier. If the read * identifier is not as expected the function returns - * false. + * true. Note that the function also returns true if + * the count entry identifier coincides with + * \b expc_ident but is not in the list of supported + * count identifiers (currently 'E' and 'N'). * \param [out] count_var The count variable read from the count entry. - * \param [out] count_err A Boolean indicating if a count error occurred. - * For this parameter the term count refers to the - * expected byte count for reading count (different - * count) entry. - * \return True if the count entry is valid and has the - * expected identifier. + * \return False if the count entry is valid and has the + * expected identifier. True, otherwise. */ static int -sc_scda_check_count_entry_internal (const char *count_entry, char expc_ident, - size_t *count_var) +sc_scda_check_count_entry (const char *count_entry, char expc_ident, + size_t *count_var) { char var_str[SC_SCDA_COUNT_MAX_DIGITS + 1]; char ident; @@ -2654,8 +2650,8 @@ sc_scda_fread_block_header_serial (sc_scda_fcontext_t *fc, size_t *elem_size, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); /* check read count entry */ - invalid_count_entry = sc_scda_check_count_entry_internal (count_entry, 'E', - elem_size); + invalid_count_entry = sc_scda_check_count_entry (count_entry, 'E', + elem_size); sc_scda_scdaret_to_errcode (invalid_count_entry ? SC_SCDA_FERR_FORMAT : SC_SCDA_FERR_SUCCESS, errcode, fc); @@ -2705,8 +2701,8 @@ sc_scda_fread_array_header_serial (sc_scda_fcontext_t *fc, size_t *elem_count, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); /* check read count entry */ - invalid_count_entry = sc_scda_check_count_entry_internal (count_entry,'N', - elem_count); + invalid_count_entry = sc_scda_check_count_entry (count_entry,'N', + elem_count); sc_scda_scdaret_to_errcode (invalid_count_entry ? SC_SCDA_FERR_FORMAT : SC_SCDA_FERR_SUCCESS, errcode, fc); @@ -2724,8 +2720,8 @@ sc_scda_fread_array_header_serial (sc_scda_fcontext_t *fc, size_t *elem_count, SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); /* check read count entry */ - invalid_count_entry = sc_scda_check_count_entry_internal (count_entry, 'E', - elem_size); + invalid_count_entry = sc_scda_check_count_entry (count_entry, 'E', + elem_size); sc_scda_scdaret_to_errcode (invalid_count_entry ? SC_SCDA_FERR_FORMAT : SC_SCDA_FERR_SUCCESS, errcode, fc);