Skip to content

Commit

Permalink
CL/DOCA_UROM: Add CL_DOCA_UROM
Browse files Browse the repository at this point in the history
  • Loading branch information
@nsarka
Nicholas Sarkauskas authored and nsarka committed May 31, 2024
1 parent b62b472 commit 5585525
Show file tree
Hide file tree
Showing 16 changed files with 1,475 additions and 2 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/codestyle.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
fi
fi
H1="CODESTYLE|REVIEW|CORE|UTIL|TEST|API|DOCS|TOOLS|BUILD|MC|EC|SCHEDULE|TOPO"
H2="CI|CL/|TL/|MC/|EC/|UCP|SHM|NCCL|SHARP|BASIC|HIER|CUDA|CPU|EE|RCCL|ROCM|SELF|MLX5"
H2="CI|CL/|TL/|MC/|EC/|UCP|SHM|NCCL|SHARP|BASIC|HIER|DOCA_UROM|CUDA|CPU|EE|RCCL|ROCM|SELF|MLX5"
if ! echo $msg | grep -qP '^Merge |^'"(($H1)|($H2))"'+: \w'
then
echo "Wrong header"
Expand Down
77 changes: 77 additions & 0 deletions config/m4/doca_urom.m4
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#
# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# See file LICENSE for terms.
#

AC_DEFUN([CHECK_DOCA_UROM],[
AS_IF([test "x$doca_urom_checked" != "xyes"],[
doca_urom_happy="no"
AC_ARG_WITH([doca_urom],
[AS_HELP_STRING([--with-doca_urom=(DIR)], [Enable the use of DOCA_UROM (default is guess).])],
[], [with_doca_urom=guess])
AS_IF([test "x$with_doca_urom" != "xno"],
[
save_CPPFLAGS="$CPPFLAGS $UCS_CPPFLAGS"
save_CFLAGS="$CFLAGS"
save_LDFLAGS="$LDFLAGS"
AS_IF([test ! -z "$with_doca_urom" -a "x$with_doca_urom" != "xyes" -a "x$with_doca_urom" != "xguess"],
[
AS_IF([test ! -d $with_doca_urom],
[AC_MSG_ERROR([Provided "--with-doca_urom=${with_doca_urom}" location does not exist])])
check_doca_urom_dir="$with_doca_urom"
check_doca_urom_libdir="$with_doca_urom/lib64"
CPPFLAGS="-I$with_doca_urom/include $save_CPPFLAGS"
LDFLAGS="-L$check_doca_urom_libdir $save_LDFLAGS"
])
AS_IF([test ! -z "$with_doca_urom_libdir" -a "x$with_doca_urom_libdir" != "xyes"],
[
check_doca_urom_libdir="$with_doca_urom_libdir"
LDFLAGS="-L$check_doca_urom_libdir $save_LDFLAGS"
])
AC_CHECK_HEADERS([doca_urom.h],
[
AC_CHECK_LIB([doca_urom], [doca_urom_service_create],
[
doca_urom_happy="yes"
],
[
echo "CPPFLAGS: $CPPFLAGS"
doca_urom_happy="no"
], [-ldoca_common -ldoca_argp -ldoca_urom])
],
[
doca_urom_happy="no"
])
AS_IF([test "x$doca_urom_happy" = "xyes"],
[
AS_IF([test "x$check_doca_urom_dir" != "x"],
[
AC_MSG_RESULT([DOCA_UROM dir: $check_doca_urom_dir])
AC_SUBST(DOCA_UROM_CPPFLAGS, "-I$check_doca_urom_dir/include/ $doca_urom_old_headers")
])
AS_IF([test "x$check_doca_urom_libdir" != "x"],
[
AC_SUBST(DOCA_UROM_LDFLAGS, "-L$check_doca_urom_libdir")
])
AC_SUBST(DOCA_UROM_LIBADD, "-ldoca_common -ldoca_argp -ldoca_urom")
AC_DEFINE([HAVE_DOCA_UROM], 1, [Enable DOCA_UROM support])
],
[
AS_IF([test "x$with_doca_urom" != "xguess"],
[
AC_MSG_ERROR([DOCA_UROM support is requested but DOCA_UROM packages cannot be found! $CPPFLAGS $LDFLAGS])
],
[
AC_MSG_WARN([DOCA_UROM not found])
])
])
CFLAGS="$save_CFLAGS"
CPPFLAGS="$save_CPPFLAGS"
LDFLAGS="$save_LDFLAGS"
],
[
AC_MSG_WARN([DOCA_UROM was explicitly disabled])
])
doca_urom_checked=yes
AM_CONDITIONAL([HAVE_DOCA_UROM], [test "x$doca_urom_happy" != xno])
])])
84 changes: 84 additions & 0 deletions config/m4/doca_urom_ucc.m4
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#
# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# See file LICENSE for terms.
#

AC_DEFUN([CHECK_DOCA_UROM_UCC],[
AS_IF([test "x$doca_urom_ucc_checked" != "xyes"],[
doca_urom_ucc_happy="no"
AC_ARG_WITH([doca_urom_ucc],
[AS_HELP_STRING([--with-doca_urom_ucc=(DIR)], [Enable the use of DOCA_UROM_UCC (default is no).])],
[], [with_doca_urom_ucc=no])
AS_IF([test "x$with_doca_urom_ucc" != "xno"],
[
AS_IF([test "x$doca_urom_happy" == "xyes"],
[
save_CPPFLAGS="$CPPFLAGS $UCS_CPPFLAGS $DOCA_UROM_CPPFLAGS"
save_CFLAGS="$CFLAGS"
save_LDFLAGS="$LDFLAGS $DOCA_UROM_LDFLAGS"
AS_IF([test ! -z "$with_doca_urom_ucc" -a "x$with_doca_urom_ucc" != "xyes" -a "x$with_doca_urom_ucc" != "xguess"],
[
AS_IF([test ! -d $with_doca_urom_ucc],
[AC_MSG_ERROR([Provided "--with-doca_urom_ucc=${with_doca_urom_ucc}" location does not exist])])
check_doca_urom_ucc_dir="$with_doca_urom_ucc"
check_doca_urom_ucc_libdir="$with_doca_urom_ucc/lib64"
CPPFLAGS="-I$with_doca_urom_ucc/include $save_CPPFLAGS"
LDFLAGS="-L$check_doca_urom_ucc_libdir $save_LDFLAGS"
])
AS_IF([test ! -z "$with_doca_urom_ucc_libdir" -a "x$with_doca_urom_ucc_libdir" != "xyes"],
[
check_doca_urom_ucc_libdir="$with_doca_urom_ucc_libdir"
LDFLAGS="-L$check_doca_urom_ucc_libdir $save_LDFLAGS"
])
AC_CHECK_HEADERS([urom_common.h],
[
AC_CHECK_LIB([doca_urom_ucc], [start_urom_service],
[
doca_urom_ucc_happy="yes"
],
[
echo "CPPFLAGS: $CPPFLAGS"
doca_urom_ucc_happy="no"
], [-ldoca_urom_ucc -ldoca_common -ldoca_argp -ldoca_urom])
],
[
doca_urom_ucc_happy="no"
])
AS_IF([test "x$doca_urom_ucc_happy" = "xyes"],
[
AS_IF([test "x$check_doca_urom_ucc_dir" != "x"],
[
AC_MSG_RESULT([DOCA_UROM_UCC dir: $check_doca_urom_ucc_dir])
AC_SUBST(DOCA_UROM_UCC_CPPFLAGS, "-I$check_doca_urom_ucc_dir/include/ $doca_urom_ucc_old_headers")
])
AS_IF([test "x$check_doca_urom_ucc_libdir" != "x"],
[
AC_SUBST(DOCA_UROM_UCC_LDFLAGS, "-L$check_doca_urom_ucc_libdir")
])
AC_SUBST(DOCA_UROM_UCC_LIBADD, "-ldoca_urom_ucc")
AC_DEFINE([HAVE_DOCA_UROM_UCC], 1, [Enable DOCA_UROM_UCC support])
],
[
AS_IF([test "x$with_doca_urom_ucc" != "xguess"],
[
AC_MSG_ERROR([DOCA_UROM_UCC support is requested but DOCA_UROM_UCC packages cannot be found! $CPPFLAGS $LDFLAGS])
],
[
AC_MSG_WARN([DOCA_UROM_UCC not found])
])
])
CFLAGS="$save_CFLAGS"
CPPFLAGS="$save_CPPFLAGS"
LDFLAGS="$save_LDFLAGS"
],
[
AC_MSG_ERROR([DOCA_UROM_UCC was enabled, but DOCA_UROM was not])
doca_urom_ucc_happy="no"
])
],
[
AC_MSG_WARN([DOCA_UROM_UCC was explicitly disabled])
])
doca_urom_ucc_checked=yes
AM_CONDITIONAL([HAVE_DOCA_UROM_UCC], [test "x$doca_urom_ucc_happy" != xno])
])])
9 changes: 9 additions & 0 deletions configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ AS_IF([test "x$with_docs_only" = xyes],
m4_include([config/m4/cuda.m4])
m4_include([config/m4/nccl.m4])
m4_include([config/m4/rocm.m4])
m4_include([config/m4/doca_urom.m4])
m4_include([config/m4/doca_urom_ucc.m4])
m4_include([config/m4/rccl.m4])
m4_include([config/m4/sharp.m4])
m4_include([config/m4/mpi.m4])
Expand Down Expand Up @@ -205,6 +207,12 @@ AS_IF([test "x$with_docs_only" = xyes],
mc_modules="${mc_modules}:rocm"
fi
CHECK_DOCA_UROM
AC_MSG_RESULT([DOCA_UROM support: $doca_urom_happy])
CHECK_DOCA_UROM_UCC
AC_MSG_RESULT([DOCA_UROM_UCC support: $doca_urom_ucc_happy])
CHECK_GTEST
AC_MSG_RESULT([GTEST support: $gtest_happy])
Expand All @@ -229,6 +237,7 @@ AC_CONFIG_FILES([
src/core/ucc_version.c
src/components/cl/basic/Makefile
src/components/cl/hier/Makefile
src/components/cl/doca_urom/Makefile
src/components/mc/cpu/Makefile
src/components/mc/cuda/Makefile
src/components/ec/cpu/Makefile
Expand Down
4 changes: 4 additions & 0 deletions src/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ mc_dirs += components/mc/cuda
ec_dirs += components/ec/cuda
endif

if HAVE_DOCA_UROM_UCC
cl_dirs += components/cl/doca_urom
endif

if HAVE_ROCM
mc_dirs += components/mc/rocm
ec_dirs += components/ec/rocm
Expand Down
20 changes: 20 additions & 0 deletions src/components/cl/doca_urom/Makefile.am
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#
# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#

sources = \
cl_doca_urom.h \
cl_doca_urom.c \
cl_doca_urom_lib.c \
cl_doca_urom_context.c \
cl_doca_urom_team.c \
cl_doca_urom_coll.c

module_LTLIBRARIES = libucc_cl_doca_urom.la
libucc_cl_doca_urom_la_SOURCES = $(sources)
libucc_cl_doca_urom_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(DOCA_UROM_CPPFLAGS) $(DOCA_UROM_UCC_CPPFLAGS)
libucc_cl_doca_urom_la_CFLAGS = $(BASE_CFLAGS)
libucc_cl_doca_urom_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(DOCA_UROM_UCC_LDFLAGS) $(DOCA_UROM_LDFLAGS)
libucc_cl_doca_urom_la_LIBADD = $(DOCA_UROM_UCC_LIBADD) $(DOCA_UROM_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la

include $(top_srcdir)/config/module.am
76 changes: 76 additions & 0 deletions src/components/cl/doca_urom/cl_doca_urom.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/**
* Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*
* See file LICENSE for terms.
*/

#include "cl_doca_urom.h"
#include "utils/ucc_malloc.h"

ucc_status_t ucc_cl_doca_urom_get_lib_attr(const ucc_base_lib_t *lib,
ucc_base_lib_attr_t *base_attr);

ucc_status_t ucc_cl_doca_urom_get_context_attr(const ucc_base_context_t *context,
ucc_base_ctx_attr_t *base_attr);

ucc_status_t ucc_cl_doca_urom_get_lib_properties(ucc_base_lib_properties_t *prop);

static ucc_config_field_t ucc_cl_doca_urom_lib_config_table[] = {
{"", "", NULL, ucc_offsetof(ucc_cl_doca_urom_lib_config_t, super),
UCC_CONFIG_TYPE_TABLE(ucc_cl_lib_config_table)},

{"DEVICE", "mlx5_0",
"DPU device",
ucc_offsetof(ucc_cl_doca_urom_lib_config_t, device),
UCC_CONFIG_TYPE_STRING},

{"PLUGIN_NAME", "libdoca_urom_ucc_plugin",
"Name of plugin library",
ucc_offsetof(ucc_cl_doca_urom_lib_config_t, plugin_name),
UCC_CONFIG_TYPE_STRING},

{"DOCA_LOG_LEVEL", "10",
"DOCA log level",
ucc_offsetof(ucc_cl_doca_urom_lib_config_t, doca_log_level),
UCC_CONFIG_TYPE_INT},

{NULL}};

static ucs_config_field_t ucc_cl_doca_urom_context_config_table[] = {
{"", "", NULL, ucc_offsetof(ucc_cl_doca_urom_context_config_t, super),
UCC_CONFIG_TYPE_TABLE(ucc_cl_context_config_table)},

{"PLUGIN_ENVS", "",
"Comma separated envs to pass to the worker plugin",
ucc_offsetof(ucc_cl_doca_urom_context_config_t, plugin_envs),
UCC_CONFIG_TYPE_STRING_ARRAY},

{NULL}};

UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_doca_urom_lib_t, ucc_base_lib_t,
const ucc_base_lib_params_t *,
const ucc_base_config_t *);

UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_doca_urom_lib_t, ucc_base_lib_t);

UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_doca_urom_context_t, ucc_base_context_t,
const ucc_base_context_params_t *,
const ucc_base_config_t *);

UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_doca_urom_context_t, ucc_base_context_t);

UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_doca_urom_team_t, ucc_base_team_t,
ucc_base_context_t *, const ucc_base_team_params_t *);

ucc_status_t ucc_cl_doca_urom_team_create_test(ucc_base_team_t *cl_team);

ucc_status_t ucc_cl_doca_urom_team_destroy(ucc_base_team_t *cl_team);

ucc_status_t ucc_cl_doca_urom_coll_init(ucc_base_coll_args_t *coll_args,
ucc_base_team_t *team,
ucc_coll_task_t **task);

ucc_status_t ucc_cl_doca_urom_team_get_scores(ucc_base_team_t *cl_team,
ucc_coll_score_t **score);

UCC_CL_IFACE_DECLARE(doca_urom, DOCA_UROM);
Loading

0 comments on commit 5585525

Please sign in to comment.