Merge pull request #10255 from evgeny-leksikov/ucp_tl_alias_gga

UCP: add gga_mlx5 to UCP transport aliases
openucx · Nov 29, 2024 · 1c22ffc · 1c22ffc
2 parents 6ba246d + ae50f8d
commit 1c22ffc
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 10 deletions.
diff --git a/contrib/lsan.supp b/contrib/lsan.supp
@@ -2,3 +2,5 @@ leak:libcuda
 leak:dlsym
 leak:nvmlInitWithFlags
 leak:bfd_map_over_sections
+# see https://github.com/openucx/ucx/blob/ec4407f08dbed639409d796e42bfab1ff718fda5/src/uct/ib/base/ib_md.c#L1392
+leak:ibv_alloc_pd 
diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c
@@ -659,7 +659,8 @@ static ucp_tl_alias_t ucp_tl_aliases[] = {
   { "mm",    { "posix", "sysv", "xpmem", NULL } }, /* for backward compatibility */
   { "sm",    { "posix", "sysv", "xpmem", "knem", "cma", NULL } },
   { "shm",   { "posix", "sysv", "xpmem", "knem", "cma", NULL } },
-  { "ib",    { "rc_verbs", "ud_verbs", "rc_mlx5", "ud_mlx5", "dc_mlx5", NULL } },
+  { "ib",    { "rc_verbs", "ud_verbs", "rc_mlx5", "ud_mlx5", "dc_mlx5",
+               "gga_mlx5", NULL } },
   { "ud_v",  { "ud_verbs", NULL } },
   { "ud_x",  { "ud_mlx5", NULL } },
   { "ud",    { "ud_mlx5", "ud_verbs", NULL } },
@@ -673,6 +674,7 @@ static ucp_tl_alias_t ucp_tl_aliases[] = {
   { "cuda",  { "cuda_copy", "cuda_ipc", "gdr_copy", NULL } },
   { "rocm",  { "rocm_copy", "rocm_ipc", "rocm_gdr", NULL } },
   { "ze",    { "ze_copy", "ze_ipc", "ze_gdr", NULL } },
+  { "gga",   { "gga_mlx5", NULL } },
   { NULL }
 };
 

diff --git a/src/ucp/rndv/rndv.c b/src/ucp/rndv/rndv.c
@@ -594,6 +594,8 @@ ucp_rndv_progress_rma_zcopy_common(ucp_request_t *req, ucp_lane_index_t lane,
     remaining = (uintptr_t)req->send.buffer % align;
 
     if ((offset == 0) && (remaining > 0) && (req->send.length > ucp_mtu)) {
+        ucs_assertv(ucp_mtu > remaining, "ucp_mtu %lu, remaining %lu",
+                    ucp_mtu, remaining);
         length = ucp_mtu - remaining;
     } else {
         lanes_count = ucs_popcount(req->send.rndv.zcopy.lanes_map_all);

diff --git a/src/uct/ib/mlx5/gga/gga_mlx5.c b/src/uct/ib/mlx5/gga/gga_mlx5.c
@@ -257,6 +257,7 @@ static ucs_status_t
 uct_gga_mlx5_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr)
 {
     uct_rc_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_iface_t);
+    size_t iface_mtu      = uct_ib_mtu_value(iface->super.config.path_mtu);
     ucs_status_t status;
 
     status = uct_ib_iface_query(&iface->super, UCT_IB_RETH_LEN, iface_attr);
@@ -280,11 +281,13 @@ uct_gga_mlx5_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr)
     iface_attr->cap.put.max_zcopy       = UCT_GGA_MAX_MSG_SIZE;
     iface_attr->cap.put.max_iov         = 1;
     iface_attr->cap.put.opt_zcopy_align = UCS_SYS_PCI_MAX_PAYLOAD;
+    iface_attr->cap.put.align_mtu       = iface_mtu;
 
     iface_attr->cap.get.min_zcopy       = 1;
     iface_attr->cap.get.max_zcopy       = iface->config.max_get_zcopy;
     iface_attr->cap.get.max_iov         = 1;
     iface_attr->cap.get.opt_zcopy_align = UCS_SYS_PCI_MAX_PAYLOAD;
+    iface_attr->cap.get.align_mtu       = iface_mtu;
 
     iface_attr->latency.c += 200e-9;
 

diff --git a/test/gtest/ucp/test_ucp_request.cc b/test/gtest/ucp/test_ucp_request.cc
@@ -389,6 +389,7 @@ class test_proto_reset : public ucp_test {
         std::vector<void*> reqs;
         send_requests(1, reqs, op, sync);
         wait_reqs(op, reqs);
+        flush_workers();
 
         /* Send all messages */
         send_requests(reqs_count, reqs, op, sync);

diff --git a/test/gtest/ucp/ucp_test.h b/test/gtest/ucp/ucp_test.h
@@ -440,15 +440,16 @@ std::vector<ucp_test_param> enum_test_params(const std::string& tls)
  * @param _test_case  Test case class, derived from ucp_test.
  */
 #define UCP_INSTANTIATE_TEST_CASE(_test_case) \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, dcx,    "dc_x") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, ud,     "ud_v") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, udx,    "ud_x") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, rc,     "rc_v") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, rcx,    "rc_x") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, shm_ib, "shm,ib") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, ugni,   "ugni") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, self,   "self") \
-    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, tcp,    "tcp")
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, dcx,     "dc_x") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, ud,      "ud_v") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, udx,     "ud_x") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, rc,      "rc_v") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, rcx,     "rc_x") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, shm_ib,  "shm,ib") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, ugni,    "ugni") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, self,    "self") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, tcp,     "tcp") \
+    UCP_INSTANTIATE_TEST_CASE_TLS(_test_case, shm_gga, "shm,gga")
 
 
 /**