UTIL: properly init rcache params

Sergei-Lebedev · Jan 25, 2024 · dad2dc5 · dad2dc5
1 parent 75ecf74
commit dad2dc5
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 37 deletions.
diff --git a/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c b/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  *
  * See file LICENSE for terms.
  */
@@ -130,7 +130,7 @@ ucc_tl_mlx5_mcast_mem_deregister(ucc_tl_mlx5_mcast_coll_context_t *ctx,
     return UCC_OK;
 }
 
-static ucc_rcache_ops_t ucc_rcache_ops = {
+static ucc_rcache_ops_t ucc_tl_mlx5_rcache_ops = {
     .mem_reg     = ucc_tl_mlx5_mcast_rcache_mem_reg_cb,
     .mem_dereg   = ucc_tl_mlx5_mcast_rcache_mem_dereg_cb,
     .dump_region = ucc_tl_mlx5_mcast_rcache_dump_region_cb
@@ -140,15 +140,12 @@ ucc_status_t ucc_tl_mlx5_mcast_setup_rcache(ucc_tl_mlx5_mcast_coll_context_t *ct
 {
     ucc_rcache_params_t rcache_params;
 
-    rcache_params.ucm_event_priority = 1000;
-    rcache_params.max_regions        = ULONG_MAX;
-    rcache_params.max_size           = SIZE_MAX;
+    ucc_rcache_set_default_params(&rcache_params);
     rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_mcast_rcache_region_t);
+    rcache_params.context            = ctx;
+    rcache_params.ops                = &ucc_tl_mlx5_rcache_ops;
     rcache_params.ucm_events         = UCM_EVENT_VM_UNMAPPED |
                                        UCM_EVENT_MEM_TYPE_FREE;
-    rcache_params.context            = ctx;
-    rcache_params.ops                = &ucc_rcache_ops;
-    rcache_params.flags              = 0;
 
-    return ucc_rcache_create(&rcache_params, "MCAST", &ctx->rcache);
+    return ucc_rcache_create(&rcache_params, "MLX5_MCAST", &ctx->rcache);
 }
diff --git a/src/components/tl/mlx5/tl_mlx5_rcache.c b/src/components/tl/mlx5/tl_mlx5_rcache.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  *
  * See file LICENSE for terms.
  */
@@ -52,22 +52,22 @@ static void ucc_tl_mlx5_rcache_dump_region_cb(void *context, //NOLINT
     snprintf(buf, max, "bar ptr:%p", mlx5_rregion->reg.mr);
 }
 
-static ucc_rcache_ops_t ucc_rcache_ops = {
+static ucc_rcache_ops_t ucc_tl_mlx5_rcache_ops = {
     .mem_reg     = rcache_reg_mr,
     .mem_dereg   = rcache_dereg_mr,
     .dump_region = ucc_tl_mlx5_rcache_dump_region_cb
 };
 
 ucc_status_t tl_mlx5_rcache_create(ucc_tl_mlx5_context_t *ctx)
 {
-    ucc_rcache_params_t     rcache_params;
+    ucc_rcache_params_t rcache_params;
 
+    ucc_rcache_set_default_params(&rcache_params);
     rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_rcache_region_t);
-    rcache_params.ucm_event_priority = 1000;
-    rcache_params.context            = (void *)ctx;
-    rcache_params.ops                = &ucc_rcache_ops;
-    rcache_params.ucm_events         = UCM_EVENT_VM_UNMAPPED
-                                            | UCM_EVENT_MEM_TYPE_FREE;
+    rcache_params.context            = ctx;
+    rcache_params.ops                = &ucc_tl_mlx5_rcache_ops;
+    rcache_params.ucm_events         = UCM_EVENT_VM_UNMAPPED |
+                                       UCM_EVENT_MEM_TYPE_FREE;
 
-    return ucc_rcache_create(&rcache_params, "MLX5", &ctx->rcache);
+    return ucc_rcache_create(&rcache_params, "MLX5_A2A", &ctx->rcache);
 }
diff --git a/src/components/tl/sharp/tl_sharp_context.c b/src/components/tl/sharp/tl_sharp_context.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  *
  * See file LICENSE for terms.
  */
@@ -269,15 +269,12 @@ ucc_status_t ucc_tl_sharp_rcache_create(struct sharp_coll_context *context,
 {
     ucc_rcache_params_t rcache_params;
 
-    rcache_params.ucm_event_priority = 1000;
-    rcache_params.max_regions        = ULONG_MAX;
-    rcache_params.max_size           = SIZE_MAX;
+    ucc_rcache_set_default_params(&rcache_params);
     rcache_params.region_struct_size = sizeof(ucc_tl_sharp_rcache_region_t);
-    rcache_params.ucm_events         = UCM_EVENT_VM_UNMAPPED |
-        UCM_EVENT_MEM_TYPE_FREE;
     rcache_params.context            = context;
     rcache_params.ops                = &ucc_tl_sharp_rcache_ops;
-    rcache_params.flags              = 0;
+    rcache_params.ucm_events         = UCM_EVENT_VM_UNMAPPED |
+                                       UCM_EVENT_MEM_TYPE_FREE;
 
     return ucc_rcache_create(&rcache_params, "SHARP", rcache);
 }

diff --git a/src/utils/ucc_rcache.h b/src/utils/ucc_rcache.h
@@ -1,5 +1,6 @@
 /**
- * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *
  * See file LICENSE for terms.
  */
 
@@ -16,10 +17,23 @@
 #define ucc_rcache_params_t          ucs_rcache_params_t
 #define ucc_rcache_region_t          ucs_rcache_region_t
 
-#define ucc_rcache_destroy           ucs_rcache_destroy
-#define ucc_rcache_region_hold       ucs_rcache_region_hold
-#define ucc_rcache_region_put        ucs_rcache_region_put
-#define ucc_rcache_region_invalidate ucs_rcache_region_invalidate
+static inline void ucc_rcache_set_default_params(ucs_rcache_params_t *rcache_params)
+{
+    rcache_params->region_struct_size = sizeof(ucs_rcache_region_t);
+    rcache_params->ucm_events         = 0;
+    rcache_params->ucm_event_priority = 1000;
+    rcache_params->ops                = NULL;
+    rcache_params->context            = NULL;
+    rcache_params->flags              = 0;
+    rcache_params->max_regions        = UCS_MEMUNITS_INF;
+    rcache_params->max_size           = UCS_MEMUNITS_INF;
+    rcache_params->max_unreleased     = UCS_MEMUNITS_INF;
+}
+
+#define ucc_rcache_destroy            ucs_rcache_destroy
+#define ucc_rcache_region_hold        ucs_rcache_region_hold
+#define ucc_rcache_region_put         ucs_rcache_region_put
+#define ucc_rcache_region_invalidate  ucs_rcache_region_invalidate
 
 /* Wrapper functions for status conversion */
 static inline ucc_status_t
@@ -46,16 +60,17 @@ static inline ucc_status_t
 ucc_rcache_get(ucc_rcache_t *rcache, void *address, size_t length,
                void *arg, ucc_rcache_region_t **region_p)
 {
+    ucs_status_t status;
+
 #ifdef UCS_HAVE_RCACHE_REGION_ALIGNMENT
-    return ucs_status_to_ucc_status(ucs_rcache_get(
-                                       rcache, address, length,
-                                       ucc_get_page_size(),
-                                       PROT_READ | PROT_WRITE, arg, region_p));
+    status = ucs_rcache_get(rcache, address, length, ucc_get_page_size(),
+                            PROT_READ | PROT_WRITE, arg, region_p);
 #else
-    return ucs_status_to_ucc_status(ucs_rcache_get(
-                                       rcache, address, length,
-                                       PROT_READ | PROT_WRITE, arg, region_p));
+    status = ucs_rcache_get(rcache, address, length,
+                            PROT_READ | PROT_WRITE, arg, region_p);
 #endif
+
+    return ucs_status_to_ucc_status(status);
 }
 
 #endif