TL/UCP: fix completed counter race

Sergei-Lebedev · Feb 28, 2024 · 48eb6a8 · 48eb6a8
1 parent ce9821c
commit 48eb6a8
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 11 deletions.
diff --git a/src/components/tl/ucp/alltoallv/alltoallv_hybrid.c b/src/components/tl/ucp/alltoallv/alltoallv_hybrid.c
@@ -13,6 +13,7 @@
 #include "tl_ucp_sendrecv.h"
 #include "components/mc/ucc_mc.h"
 #include "coll_patterns/bruck_alltoall.h"
+#include "utils/ucc_atomic.h"
 
 /*
 scratch structure
@@ -122,7 +123,7 @@ static void send_completion(void *request, ucs_status_t status,
         ucp_request_free(request);
     }
 
-    bin->task->tagged.send_completed++;
+    ucc_atomic_add32(&bin->task->tagged.send_completed, 1);
     bin->len = 0;
 }
 

diff --git a/src/components/tl/ucp/bcast/bcast_dbt.c b/src/components/tl/ucp/bcast/bcast_dbt.c
@@ -40,7 +40,7 @@ static void recv_completion_common(void *request, ucs_status_t status,
                  ucs_status_string(status));
         task->super.status = ucs_status_to_ucc_status(status);
     }
-    task->tagged.recv_completed++;
+    ucc_atomic_add32(&task->tagged.recv_completed, 1);
     if (request) {
         ucp_request_free(request);
     }

diff --git a/src/components/tl/ucp/reduce/reduce_dbt.c b/src/components/tl/ucp/reduce/reduce_dbt.c
@@ -41,7 +41,7 @@ static void recv_completion_common(void *request, ucs_status_t status,
                  ucs_status_string(status));
         task->super.status = ucs_status_to_ucc_status(status);
     }
-    task->tagged.recv_completed++;
+    ucc_atomic_add32(&task->tagged.recv_completed, 1);
     if (request) {
         ucp_request_free(request);
     }

diff --git a/src/components/tl/ucp/reduce_scatter/reduce_scatter_ring.c b/src/components/tl/ucp/reduce_scatter/reduce_scatter_ring.c
@@ -11,6 +11,7 @@
 #include "utils/ucc_math.h"
 #include "utils/ucc_coll_utils.h"
 #include "utils/ucc_dt_reduce.h"
+#include "utils/ucc_atomic.h"
 
 #define REVERSED_FRAG 1
 
@@ -24,7 +25,7 @@ static inline void send_completion_common(void *request, ucs_status_t status,
                  ucs_status_string(status));
         task->super.status = ucs_status_to_ucc_status(status);
     }
-    task->tagged.send_completed++;
+    ucc_atomic_add32(&task->tagged.send_completed, 1);
     if (request) {
         ucp_request_free(request);
     }

diff --git a/src/components/tl/ucp/reduce_scatterv/reduce_scatterv_ring.c b/src/components/tl/ucp/reduce_scatterv/reduce_scatterv_ring.c
@@ -24,7 +24,7 @@ static inline void send_completion_common(void *request, ucs_status_t status,
                  ucs_status_string(status));
         task->super.status = ucs_status_to_ucc_status(status);
     }
-    task->tagged.send_completed++;
+    ucc_atomic_add32(&task->tagged.send_completed, 1);
     if (request) {
         ucp_request_free(request);
     }

diff --git a/src/components/tl/ucp/tl_ucp_coll.c b/src/components/tl/ucp/tl_ucp_coll.c
@@ -102,7 +102,7 @@ void ucc_tl_ucp_send_completion_cb(void *request, ucs_status_t status,
                  ucs_status_string(status));
         task->super.status = ucs_status_to_ucc_status(status);
     }
-    task->tagged.send_completed++;
+    ucc_atomic_add32(&task->tagged.send_completed, 1);
     ucp_request_free(request);
 }
 
@@ -142,7 +142,7 @@ void ucc_tl_ucp_recv_completion_cb(void *request, ucs_status_t status,
                  ucs_status_string(status));
         task->super.status = ucs_status_to_ucc_status(status);
     }
-    task->tagged.recv_completed++;
+    ucc_atomic_add32(&task->tagged.recv_completed, 1);
     ucp_request_free(request);
 }
 

diff --git a/src/components/tl/ucp/tl_ucp_sendrecv.h b/src/components/tl/ucp/tl_ucp_sendrecv.h
@@ -107,7 +107,7 @@ ucc_tl_ucp_send_nb(void *buffer, size_t msglen, ucc_memory_type_t mtype,
     if (UCS_OK != ucp_status) {
         UCC_TL_UCP_CHECK_REQ_STATUS();
     } else {
-        task->tagged.send_completed++;
+        ucc_atomic_add32(&task->tagged.send_completed, 1);
     }
     return UCC_OK;
 }
@@ -170,7 +170,7 @@ ucc_tl_ucp_recv_nb(void *buffer, size_t msglen, ucc_memory_type_t mtype,
     if (UCS_OK != ucp_status) {
         UCC_TL_UCP_CHECK_REQ_STATUS();
     } else {
-        task->tagged.recv_completed++;
+        ucc_atomic_add32(&task->tagged.recv_completed, 1);
     }
     return UCC_OK;
 
@@ -202,7 +202,7 @@ static inline ucc_status_t ucc_tl_ucp_recv_nz(void *buffer, size_t msglen,
 {
     if (msglen == 0) {
         task->tagged.recv_posted++;
-        task->tagged.recv_completed++;
+        ucc_atomic_add32(&task->tagged.recv_completed, 1);
         return UCC_OK;
     }
     return ucc_tl_ucp_recv_nb(buffer, msglen, mtype,
@@ -218,7 +218,7 @@ static inline ucc_status_t ucc_tl_ucp_send_nz(void *buffer, size_t msglen,
 {
     if (msglen == 0) {
         task->tagged.send_posted++;
-        task->tagged.send_completed++;
+        ucc_atomic_add32(&task->tagged.send_completed, 1);
         return UCC_OK;
     }
     return ucc_tl_ucp_send_nb(buffer, msglen, mtype,