Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UTIL: properly init rcache params #896

Merged
merged 1 commit into from
Jan 25, 2024
Merged

UTIL: properly init rcache params #896

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*
* See file LICENSE for terms.
*/
Expand Down Expand Up @@ -130,7 +130,7 @@ ucc_tl_mlx5_mcast_mem_deregister(ucc_tl_mlx5_mcast_coll_context_t *ctx,
return UCC_OK;
}

static ucc_rcache_ops_t ucc_rcache_ops = {
static ucc_rcache_ops_t ucc_tl_mlx5_rcache_ops = {
.mem_reg = ucc_tl_mlx5_mcast_rcache_mem_reg_cb,
.mem_dereg = ucc_tl_mlx5_mcast_rcache_mem_dereg_cb,
.dump_region = ucc_tl_mlx5_mcast_rcache_dump_region_cb
Expand All @@ -140,15 +140,12 @@ ucc_status_t ucc_tl_mlx5_mcast_setup_rcache(ucc_tl_mlx5_mcast_coll_context_t *ct
{
ucc_rcache_params_t rcache_params;

rcache_params.ucm_event_priority = 1000;
rcache_params.max_regions = ULONG_MAX;
rcache_params.max_size = SIZE_MAX;
ucc_rcache_set_default_params(&rcache_params);
rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_mcast_rcache_region_t);
rcache_params.context = ctx;
rcache_params.ops = &ucc_tl_mlx5_rcache_ops;
rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED |
UCM_EVENT_MEM_TYPE_FREE;
rcache_params.context = ctx;
rcache_params.ops = &ucc_rcache_ops;
rcache_params.flags = 0;

return ucc_rcache_create(&rcache_params, "MCAST", &ctx->rcache);
return ucc_rcache_create(&rcache_params, "MLX5_MCAST", &ctx->rcache);
}
18 changes: 9 additions & 9 deletions src/components/tl/mlx5/tl_mlx5_rcache.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*
* See file LICENSE for terms.
*/
Expand Down Expand Up @@ -52,22 +52,22 @@ static void ucc_tl_mlx5_rcache_dump_region_cb(void *context, //NOLINT
snprintf(buf, max, "bar ptr:%p", mlx5_rregion->reg.mr);
}

static ucc_rcache_ops_t ucc_rcache_ops = {
static ucc_rcache_ops_t ucc_tl_mlx5_rcache_ops = {
.mem_reg = rcache_reg_mr,
.mem_dereg = rcache_dereg_mr,
.dump_region = ucc_tl_mlx5_rcache_dump_region_cb
};

ucc_status_t tl_mlx5_rcache_create(ucc_tl_mlx5_context_t *ctx)
{
ucc_rcache_params_t rcache_params;
ucc_rcache_params_t rcache_params;

ucc_rcache_set_default_params(&rcache_params);
rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_rcache_region_t);
rcache_params.ucm_event_priority = 1000;
rcache_params.context = (void *)ctx;
rcache_params.ops = &ucc_rcache_ops;
rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED
| UCM_EVENT_MEM_TYPE_FREE;
rcache_params.context = ctx;
rcache_params.ops = &ucc_tl_mlx5_rcache_ops;
rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED |
UCM_EVENT_MEM_TYPE_FREE;

return ucc_rcache_create(&rcache_params, "MLX5", &ctx->rcache);
return ucc_rcache_create(&rcache_params, "MLX5_A2A", &ctx->rcache);
}
11 changes: 4 additions & 7 deletions src/components/tl/sharp/tl_sharp_context.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*
* See file LICENSE for terms.
*/
Expand Down Expand Up @@ -269,15 +269,12 @@ ucc_status_t ucc_tl_sharp_rcache_create(struct sharp_coll_context *context,
{
ucc_rcache_params_t rcache_params;

rcache_params.ucm_event_priority = 1000;
rcache_params.max_regions = ULONG_MAX;
rcache_params.max_size = SIZE_MAX;
ucc_rcache_set_default_params(&rcache_params);
rcache_params.region_struct_size = sizeof(ucc_tl_sharp_rcache_region_t);
rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED |
UCM_EVENT_MEM_TYPE_FREE;
rcache_params.context = context;
rcache_params.ops = &ucc_tl_sharp_rcache_ops;
rcache_params.flags = 0;
rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED |
UCM_EVENT_MEM_TYPE_FREE;

return ucc_rcache_create(&rcache_params, "SHARP", rcache);
}
Expand Down
39 changes: 27 additions & 12 deletions src/utils/ucc_rcache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*
* See file LICENSE for terms.
*/

Expand All @@ -16,10 +17,23 @@
#define ucc_rcache_params_t ucs_rcache_params_t
#define ucc_rcache_region_t ucs_rcache_region_t

#define ucc_rcache_destroy ucs_rcache_destroy
#define ucc_rcache_region_hold ucs_rcache_region_hold
#define ucc_rcache_region_put ucs_rcache_region_put
#define ucc_rcache_region_invalidate ucs_rcache_region_invalidate
static inline void ucc_rcache_set_default_params(ucs_rcache_params_t *rcache_params)
{
rcache_params->region_struct_size = sizeof(ucs_rcache_region_t);
rcache_params->ucm_events = 0;
rcache_params->ucm_event_priority = 1000;
rcache_params->ops = NULL;
rcache_params->context = NULL;
rcache_params->flags = 0;
rcache_params->max_regions = UCS_MEMUNITS_INF;
rcache_params->max_size = UCS_MEMUNITS_INF;
rcache_params->max_unreleased = UCS_MEMUNITS_INF;
}

#define ucc_rcache_destroy ucs_rcache_destroy
#define ucc_rcache_region_hold ucs_rcache_region_hold
#define ucc_rcache_region_put ucs_rcache_region_put
#define ucc_rcache_region_invalidate ucs_rcache_region_invalidate

/* Wrapper functions for status conversion */
static inline ucc_status_t
Expand All @@ -46,16 +60,17 @@ static inline ucc_status_t
ucc_rcache_get(ucc_rcache_t *rcache, void *address, size_t length,
void *arg, ucc_rcache_region_t **region_p)
{
ucs_status_t status;

#ifdef UCS_HAVE_RCACHE_REGION_ALIGNMENT
return ucs_status_to_ucc_status(ucs_rcache_get(
rcache, address, length,
ucc_get_page_size(),
PROT_READ | PROT_WRITE, arg, region_p));
status = ucs_rcache_get(rcache, address, length, ucc_get_page_size(),
PROT_READ | PROT_WRITE, arg, region_p);
#else
return ucs_status_to_ucc_status(ucs_rcache_get(
rcache, address, length,
PROT_READ | PROT_WRITE, arg, region_p));
status = ucs_rcache_get(rcache, address, length,
PROT_READ | PROT_WRITE, arg, region_p);
#endif

return ucs_status_to_ucc_status(status);
}

#endif
Loading