Merge pull request #110 from gvallee/issue96

Issue96

examples/Makefile

@@ -1,8 +1,17 @@
-all: alltoallv_c alltoallv_multicomms_c alltoallv_f alltoallv_bigcounts_c
+#
+# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
+#
+# See LICENSE.txt for license information
+#
+
+all: alltoallv_c alltoallv_dt_c alltoallv_multicomms_c alltoallv_f alltoallv_bigcounts_c
 
 alltoallv_c: alltoallv.c
 	mpicc -g alltoallv.c -o alltoallv_c
 
+alltoallv_dt_c: alltoallv_dt.c
+	mpicc -g alltoallv_dt.c -o alltoallv_dt_c
+
 alltoallv_multicomms_c: alltoallv_multicomms.c
 	mpicc -g alltoallv_multicomms.c -o alltoallv_multicomms_c
 
@@ -13,4 +22,4 @@ alltoallv_bigcounts_c: alltoallv_bigcounts.c
 	mpicc -g alltoallv_bigcounts.c -o alltoallv_bigcounts_c
 
 clean:
-	@rm -f alltoallv_c alltoallv_f alltoallv_multicomms_c alltoallv_bigcounts_c
+	@rm -f alltoallv_c alltoallv_dt_c alltoallv_f alltoallv_multicomms_c alltoallv_bigcounts_c

examples/alltoallv_dt.c

@@ -0,0 +1,106 @@
+/*************************************************************************
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include "mpi.h"
+
+#define MPICHECK(c)                                  \
+    do                                               \
+    {                                                \
+        if (c != MPI_SUCCESS)                        \
+        {                                            \
+            fprintf(stderr, "MPI command failed\n"); \
+            return 1;                                \
+        }                                            \
+    } while (0);
+
+int main(int argc, char **argv)
+{
+    int i;
+    int world_size;
+    int world_rank;
+    int *send_buffer_int;
+    int *recv_buffer_int;
+    double *send_buffer_double;
+    double *recv_buffer_double;
+    int *send_count;
+    int *recv_count;
+    int *recv_displ;
+    int *send_displ;
+
+    MPICHECK(MPI_Init(&argc, &argv));
+    MPICHECK(MPI_Comm_size(MPI_COMM_WORLD, &world_size));
+    MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &world_rank));
+
+    send_buffer_int = (int *)calloc(world_size * world_size, sizeof(int));
+    assert(send_buffer_int);
+    recv_buffer_int = (int *)calloc(world_size * world_size, sizeof(int));
+    assert(recv_buffer_int);
+    send_buffer_double = (double *)calloc(world_size * world_size, sizeof(double));
+    assert(send_buffer_double);
+    recv_buffer_double = (double *)calloc(world_size * world_size, sizeof(double));
+    assert(recv_buffer_double);
+    send_count = calloc(world_size, sizeof(int));
+    assert(send_count);
+    recv_count = calloc(world_size, sizeof(int));
+    assert(recv_count);
+    send_displ = calloc(world_size, sizeof(int));
+    assert(send_displ);
+    recv_displ = calloc(world_size, sizeof(int));
+    assert(recv_displ);
+
+    for (i = 0; i < world_size * world_size; i++)
+    {
+        send_buffer_int[i] = i + 10 * world_rank;
+    }
+
+    for (i = 0; i < world_size * world_size; i++)
+    {
+        send_buffer_double[i] = i + 10 * world_rank;
+    }
+
+    for (i = 0; i < world_size; i++)
+    {
+        send_count[i] = 1;
+        recv_count[i] = 1;
+        recv_displ[i] = 0;
+        send_displ[i] = 0;
+    }
+
+    if (world_rank == 0)
+    {
+        int s;
+        MPI_Type_size(MPI_INT, &s);
+        fprintf(stdout, "Size of MPI_INT: %d\n", s);
+        MPI_Type_size(MPI_DOUBLE, &s);
+        fprintf(stdout, "Size of MPI_DOUBLE: %d\n", s);
+    }
+
+    MPICHECK(MPI_Alltoallv(send_buffer_int, send_count, send_displ, MPI_INT,
+                           recv_buffer_int, recv_count, recv_displ, MPI_INT,
+                           MPI_COMM_WORLD));
+
+    MPICHECK(MPI_Alltoallv(send_buffer_double, send_count, send_displ, MPI_DOUBLE,
+                           recv_buffer_double, recv_count, recv_displ, MPI_DOUBLE,
+                           MPI_COMM_WORLD));
+
+    free(send_buffer_int);
+    free(recv_buffer_int);
+    free(send_buffer_double);
+    free(recv_buffer_double);
+    free(send_count);
+    free(recv_count);
+    free(send_displ);
+    free(recv_displ);
+    MPI_Finalize();
+    return EXIT_SUCCESS;
+
+exit_on_failure:
+    MPI_Finalize();
+    return EXIT_FAILURE;
+}

src/alltoall/mpi_alltoall.c

@@ -1,6 +1,6 @@
 /*************************************************************************
  * Copyright (c) 2019-2010, Mellanox Technologies, Inc. All rights reserved.
- * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
  *
  * See LICENSE.txt for license information
  ************************************************************************/
@@ -1280,15 +1280,21 @@ int _mpi_alltoall(const void *sendbuf, const int sendcount, MPI_Datatype sendtyp
 #endif
 
 #if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)
-			if (insert_sendrecv_data(sbuf, rbuf, comm_size, sizeof(sendtype), sizeof(recvtype)))
+			int s_dt_size, r_dt_size;
+			MPI_Type_size(sendtype, &s_dt_size);
+			MPI_Type_size(recvtype, &r_dt_size);
+			if (insert_sendrecv_data(sbuf, rbuf, comm_size, s_dt_size, r_dt_size))
 			{
 				fprintf(stderr, "[%s:%d][ERROR] unable to insert send/recv counts\n", __FILE__, __LINE__);
 				MPI_Abort(MPI_COMM_WORLD, 1);
 			}
 #endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)
 
 #if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)
-			save_counts(sbuf, rbuf, sizeof(sendtype), sizeof(recvtype), comm_size, avCalls);
+			int s_dt_size, r_dt_size;
+			MPI_Type_size(sendtype, &s_dt_size);
+			MPI_Type_size(recvtype, &r_dt_size);
+			save_counts(sbuf, rbuf, s_dt_size, r_dt_size, comm_size, avCalls);
 #endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)
 
 #if ENABLE_PATTERN_DETECTION

src/alltoallv/mpi_alltoallv.c

@@ -1,6 +1,6 @@
 /*************************************************************************
  * Copyright (c) 2019-2010, Mellanox Technologies, Inc. All rights reserved.
- * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
  *
  * See LICENSE.txt for license information
  ************************************************************************/
@@ -1270,15 +1270,21 @@ int _mpi_alltoallv(const void *sendbuf, const int *sendcounts, const int *sdispl
 #endif
 
 #if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)
-			if (insert_sendrecv_data(sbuf, rbuf, comm_size, sizeof(sendtype), sizeof(recvtype)))
+			int s_dt_size, r_dt_size;
+			MPI_Type_size(sendtype, &s_dt_size);
+			MPI_Type_size(recvtype, &r_dt_size);
+			if (insert_sendrecv_data(sbuf, rbuf, comm_size, s_dt_size, r_dt_size))
 			{
 				fprintf(stderr, "[%s:%d][ERROR] unable to insert send/recv counts\n", __FILE__, __LINE__);
 				MPI_Abort(MPI_COMM_WORLD, 1);
 			}
 #endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && ENABLE_COMPACT_FORMAT)
 
 #if ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)
-			save_counts(sbuf, rbuf, sizeof(sendtype), sizeof(recvtype), comm_size, avCalls);
+			int s_dt_size, r_dt_size;
+			MPI_Type_size(sendtype, &s_dt_size);
+			MPI_Type_size(recvtype, &r_dt_size);
+			save_counts(sbuf, rbuf, s_dt_size, r_dt_size, comm_size, avCalls);
 #endif // ((ENABLE_RAW_DATA || ENABLE_PER_RANK_STATS || ENABLE_VALIDATION) && !ENABLE_COMPACT_FORMAT)
 
 #if ENABLE_PATTERN_DETECTION

tests/alltoallv_bigcounts_c/expectedOutput/stats-job0-rank0.md

@@ -2,8 +2,8 @@ Total number of alltoallv calls: 1000000
 
 # Datatypes
 
-1000000/1000000 calls use a datatype of size 8 while sending data
-1000000/1000000 calls use a datatype of size 8 while receiving data
+1000000/1000000 calls use a datatype of size 4 while sending data
+1000000/1000000 calls use a datatype of size 4 while receiving data
 
 # Communicator size(s)
 

tests/alltoallv_c/expectedOutput/recv-counters.job0.rank0.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 4
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-0
 Count: 1 calls - 0
 

tests/alltoallv_c/expectedOutput/send-counters.job0.rank0.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 4
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-0
 Count: 1 calls - 0
 

tests/alltoallv_c/expectedOutput/stats-job0-rank0.md

@@ -2,8 +2,8 @@ Total number of alltoallv calls: 1
 
 # Datatypes
 
-1/1 calls use a datatype of size 8 while sending data
-1/1 calls use a datatype of size 8 while receiving data
+1/1 calls use a datatype of size 4 while sending data
+1/1 calls use a datatype of size 4 while receiving data
 
 # Communicator size(s)
 

tests/alltoallv_dt_c/expectedOutput/patterns-job0-rank0.md

@@ -0,0 +1,10 @@
+# Patterns
+## Pattern #0 (2/2 alltoallv calls)
+
+Alltoallv calls: 0-1
+
+4 ranks sent to 4 other ranks
+
+4 ranks recv'd from 4 other ranks
+
+

tests/alltoallv_dt_c/expectedOutput/patterns-summary-job0-rank0.md

@@ -0,0 +1,12 @@
+
+# N to n patterns
+
+## Pattern #0 (2/2 alltoallv calls)
+
+Alltoallv calls: 0-1
+
+4 ranks sent to 4 other ranks
+
+4 ranks recv'd from 4 other ranks
+
+

tests/alltoallv_dt_c/expectedOutput/profile_alltoallv_job0.rank0.md

@@ -0,0 +1,3 @@
+# Summary
+COMM_WORLD size: 4
+Total number of Alltoallv calls = 2 (limit is 0; -1 means no limit)

tests/alltoallv_dt_c/expectedOutput/stats-job0-rank0.md

@@ -0,0 +1,37 @@
+Total number of alltoallv calls: 2
+
+# Datatypes
+
+1/2 calls use a datatype of size 4 while sending data
+1/2 calls use a datatype of size 8 while sending data
+1/2 calls use a datatype of size 4 while receiving data
+1/2 calls use a datatype of size 8 while receiving data
+
+# Communicator size(s)
+
+2/2 calls use a communicator size of 4
+
+# Message sizes
+
+0/32 of all messages are large (threshold = 200)
+32/32 of all messages are small (threshold = 200)
+32/32 of all messages are small, but not 0-size (threshold = 200)
+
+# Sparsity
+
+2/2 of all calls have 0 send counts equals to zero
+2/2 of all calls have 0 recv counts equals to zero
+
+# Min/max
+2/2 calls have a send count min of 1
+
+2/2 calls have a recv count min of 1
+
+2/2 calls have a send count min of 0 (excluding zero)
+
+2/2 calls have a recv count min of 0 (excluding zero)
+
+2/2 calls have a send count max of 1
+
+2/2 calls have a recv count max of 1
+

tests/alltoallv_f/expectedOutput/recv-counters.job0.rank0.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 3
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-1
 Count: 2 calls - 0-1
 

tests/alltoallv_f/expectedOutput/send-counters.job0.rank0.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 3
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-1
 Count: 2 calls - 0-1
 

tests/alltoallv_f/expectedOutput/stats-job0-rank0.md

@@ -2,8 +2,8 @@ Total number of alltoallv calls: 2
 
 # Datatypes
 
-2/2 calls use a datatype of size 8 while sending data
-2/2 calls use a datatype of size 8 while receiving data
+2/2 calls use a datatype of size 4 while sending data
+2/2 calls use a datatype of size 4 while receiving data
 
 # Communicator size(s)
 

tests/alltoallv_multicomms_c/expectedOutput/recv-counters.job0.rank0.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 2
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-2
 Count: 2 calls - 0, 2
 
@@ -13,7 +13,7 @@ END DATA
 # Raw counters
 
 Number of ranks: 4
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-2
 Count: 1 calls - 1
 

tests/alltoallv_multicomms_c/expectedOutput/recv-counters.job0.rank2.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 2
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-1
 Count: 2 calls - 0, 2
 

tests/alltoallv_multicomms_c/expectedOutput/send-counters.job0.rank0.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 2
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-2
 Count: 2 calls - 0, 2
 
@@ -12,7 +12,7 @@ END DATA
 # Raw counters
 
 Number of ranks: 4
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-2
 Count: 1 calls - 1
 

tests/alltoallv_multicomms_c/expectedOutput/send-counters.job0.rank2.txt

@@ -1,7 +1,7 @@
 # Raw counters
 
 Number of ranks: 2
-Datatype size: 8
+Datatype size: 4
 Alltoallv calls 0-1
 Count: 2 calls - 0, 2
 

tests/alltoallv_multicomms_c/expectedOutput/stats-job0-rank0.md

@@ -2,8 +2,8 @@ Total number of alltoallv calls: 3
 
 # Datatypes
 
-3/3 calls use a datatype of size 8 while sending data
-3/3 calls use a datatype of size 8 while receiving data
+3/3 calls use a datatype of size 4 while sending data
+3/3 calls use a datatype of size 4 while receiving data
 
 # Communicator size(s)