Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue96 #110

Merged
merged 5 commits into from
Feb 8, 2021
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions examples/Makefile
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
all: alltoallv_c alltoallv_multicomms_c alltoallv_f alltoallv_bigcounts_c
all: alltoallv_c alltoallv_dt_c alltoallv_multicomms_c alltoallv_f alltoallv_bigcounts_c

alltoallv_c: alltoallv.c
mpicc -g alltoallv.c -o alltoallv_c

alltoallv_dt_c: alltoallv_dt.c
mpicc -g alltoallv_dt.c -o alltoallv_dt_c

alltoallv_multicomms_c: alltoallv_multicomms.c
mpicc -g alltoallv_multicomms.c -o alltoallv_multicomms_c

Expand All @@ -13,4 +16,4 @@ alltoallv_bigcounts_c: alltoallv_bigcounts.c
mpicc -g alltoallv_bigcounts.c -o alltoallv_bigcounts_c

clean:
@rm -f alltoallv_c alltoallv_f alltoallv_multicomms_c alltoallv_bigcounts_c
@rm -f alltoallv_c alltoallv_dt_c alltoallv_f alltoallv_multicomms_c alltoallv_bigcounts_c
106 changes: 106 additions & 0 deletions examples/alltoallv_dt.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*************************************************************************
* Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include "mpi.h"

#define MPICHECK(c) \
do \
{ \
if (c != MPI_SUCCESS) \
{ \
fprintf(stderr, "MPI command failed\n"); \
return 1; \
} \
} while (0);

int main(int argc, char **argv)
{
int i;
int world_size;
int world_rank;
int *send_buffer_int;
int *recv_buffer_int;
double *send_buffer_double;
double *recv_buffer_double;
int *send_count;
int *recv_count;
int *recv_displ;
int *send_displ;

MPICHECK(MPI_Init(&argc, &argv));
MPICHECK(MPI_Comm_size(MPI_COMM_WORLD, &world_size));
MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &world_rank));

send_buffer_int = (int *)calloc(world_size * world_size, sizeof(int));
assert(send_buffer_int);
recv_buffer_int = (int *)calloc(world_size * world_size, sizeof(int));
assert(recv_buffer_int);
send_buffer_double = (double *)calloc(world_size * world_size, sizeof(double));
assert(send_buffer_double);
recv_buffer_double = (double *)calloc(world_size * world_size, sizeof(double));
assert(recv_buffer_double);
send_count = calloc(world_size, sizeof(int));
assert(send_count);
recv_count = calloc(world_size, sizeof(int));
assert(recv_count);
send_displ = calloc(world_size, sizeof(int));
assert(send_displ);
recv_displ = calloc(world_size, sizeof(int));
assert(recv_displ);

for (i = 0; i < world_size * world_size; i++)
{
send_buffer_int[i] = i + 10 * world_rank;
}

for (i = 0; i < world_size * world_size; i++)
{
send_buffer_double[i] = i + 10 * world_rank;
}

for (i = 0; i < world_size; i++)
{
send_count[i] = 1;
recv_count[i] = 1;
recv_displ[i] = 0;
send_displ[i] = 0;
}

if (world_rank == 0)
{
int s;
MPI_Type_size(MPI_INT, &s);
fprintf(stdout, "Size of MPI_INT: %d\n", s);
MPI_Type_size(MPI_DOUBLE, &s);
fprintf(stdout, "Size of MPI_DOUBLE: %d\n", s);
}

MPICHECK(MPI_Alltoallv(send_buffer_int, send_count, send_displ, MPI_INT,
recv_buffer_int, recv_count, recv_displ, MPI_INT,
MPI_COMM_WORLD));

MPICHECK(MPI_Alltoallv(send_buffer_double, send_count, send_displ, MPI_DOUBLE,
recv_buffer_double, recv_count, recv_displ, MPI_DOUBLE,
MPI_COMM_WORLD));

free(send_buffer_int);
free(recv_buffer_int);
free(send_buffer_double);
free(recv_buffer_double);
free(send_count);
free(recv_count);
free(send_displ);
free(recv_displ);
MPI_Finalize();
return EXIT_SUCCESS;

exit_on_failure:
MPI_Finalize();
return EXIT_FAILURE;
}
10 changes: 8 additions & 2 deletions src/alltoall/mpi_alltoall.c
Original file line number Diff line number Diff line change
Expand Up @@ -1280,15 +1280,21 @@ int _mpi_alltoall(const void *sendbuf, const int sendcount, MPI_Datatype sendtyp
#endif

#if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)
if (insert_sendrecv_data(sbuf, rbuf, comm_size, sizeof(sendtype), sizeof(recvtype)))
int s_dt_size, r_dt_size;
MPI_Type_size(sendtype, &s_dt_size);
MPI_Type_size(recvtype, &r_dt_size);
if (insert_sendrecv_data(sbuf, rbuf, comm_size, s_dt_size, r_dt_size))
{
fprintf(stderr, "[%s:%d][ERROR] unable to insert send/recv counts\n", __FILE__, __LINE__);
MPI_Abort(MPI_COMM_WORLD, 1);
}
#endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)

#if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)
save_counts(sbuf, rbuf, sizeof(sendtype), sizeof(recvtype), comm_size, avCalls);
int s_dt_size, r_dt_size;
MPI_Type_size(sendtype, &s_dt_size);
MPI_Type_size(recvtype, &r_dt_size);
save_counts(sbuf, rbuf, s_dt_size, r_dt_size, comm_size, avCalls);
#endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)

#if ENABLE_PATTERN_DETECTION
Expand Down
10 changes: 8 additions & 2 deletions src/alltoallv/mpi_alltoallv.c
Original file line number Diff line number Diff line change
Expand Up @@ -1270,15 +1270,21 @@ int _mpi_alltoallv(const void *sendbuf, const int *sendcounts, const int *sdispl
#endif

#if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)
if (insert_sendrecv_data(sbuf, rbuf, comm_size, sizeof(sendtype), sizeof(recvtype)))
int s_dt_size, r_dt_size;
MPI_Type_size(sendtype, &s_dt_size);
MPI_Type_size(recvtype, &r_dt_size);
if (insert_sendrecv_data(sbuf, rbuf, comm_size, s_dt_size, r_dt_size))
{
fprintf(stderr, "[%s:%d][ERROR] unable to insert send/recv counts\n", __FILE__, __LINE__);
MPI_Abort(MPI_COMM_WORLD, 1);
}
#endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)

#if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)
save_counts(sbuf, rbuf, sizeof(sendtype), sizeof(recvtype), comm_size, avCalls);
int s_dt_size, r_dt_size;
MPI_Type_size(sendtype, &s_dt_size);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the same as for alltoall.c, which may of course be what you want. For alltoall it is not necesary to gather the send types because the counts are gathered and bocksize = sendtypesize * sendcount is a constant, so having gathered the sendcounts the sendtypesize used at each node can be inferred. Does something similar apply to the more complicated alltollv - I have not worked it out yet but man for alltoallv says this, which may be helpful:

"When a pair of processes exchanges data, each may pass different element count and datatype arguments so long as the sender specifies the same amount of data to send (in bytes) as the receiver expects to receive.

"Note that process i may send a different amount of data to process j than it receives from process j. Also, a process may send entirely different amounts of data to different processes in the communicator. "

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I am missing something:

  • i did the same change for alltoall
  • this issue is not about collecting the datatypes, it is about fixing a problem with the current implementation. If we decide to collect all the datatypes (which i personally think should be done), it should be a separate issue; not this one.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree it should be a separate issue. Just thought it worth a precautionary heads up before you work on the golang consumer of this data. Observation - the github interface is not so good a drawing attention to motivation for and decisions on scope of changes - or am I still a newbie looking in the wrong place?!

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Issue #111

MPI_Type_size(recvtype, &r_dt_size);
save_counts(sbuf, rbuf, s_dt_size, r_dt_size, comm_size, avCalls);
#endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)

#if ENABLE_PATTERN_DETECTION
Expand Down