From b03e2fed1dc3e604e20c04310d2c7ff6ef096ce4 Mon Sep 17 00:00:00 2001 From: Florian Weik Date: Wed, 14 Sep 2016 12:54:18 -0600 Subject: [PATCH 1/4] Added index parameter to get_comm functions --- src/quo-mpi.c | 40 ++++++++++++++++++++++------------------ src/quo-mpi.h | 1 + src/quo.c | 3 ++- src/quo.h | 3 +++ 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/quo-mpi.c b/src/quo-mpi.c index 5e55dc8..17193ed 100644 --- a/src/quo-mpi.c +++ b/src/quo-mpi.c @@ -768,24 +768,28 @@ quo_mpi_sm_barrier(const quo_mpi_t *mpi) int quo_mpi_get_comm_by_type(const quo_mpi_t *mpi, QUO_obj_type_t target_type, + int index, MPI_Comm *out_comm) { - if (!mpi || !out_comm) return QUO_ERR_INVLD_ARG; - - switch (target_type) { - case QUO_OBJ_MACHINE: - { - /* this case is easy. just return a dup of the smp communicator that - * we already maintain internally. */ - if (MPI_SUCCESS != MPI_Comm_dup(mpi->smpcomm, out_comm)) { - return QUO_ERR_MPI; - } - break; - } - /* TODO add support for other obj types */ - default: - return QUO_ERR_NOT_SUPPORTED; - } - - return QUO_SUCCESS; + if (!mpi || !out_comm) return QUO_ERR_INVLD_ARG; + + switch (target_type) { + case QUO_OBJ_MACHINE: + { + /* this case is easy. just return a dup of the smp communicator that + o * we already maintain internally. */ + if (MPI_SUCCESS != MPI_Comm_dup(mpi->smpcomm, out_comm)) { + return QUO_ERR_MPI; + } + break; + } + case QUO_OBJ_NODE: + case QUO_OBJ_SOCKET: + case QUO_OBJ_CORE: + case QUO_OBJ_PU: + default: + return QUO_ERR_NOT_SUPPORTED; + } + + return QUO_SUCCESS; } diff --git a/src/quo-mpi.h b/src/quo-mpi.h index 19ea9de..32c538d 100644 --- a/src/quo-mpi.h +++ b/src/quo-mpi.h @@ -94,5 +94,6 @@ quo_mpi_sm_barrier(const quo_mpi_t *mpi); int quo_mpi_get_comm_by_type(const quo_mpi_t *mpi, QUO_obj_type_t target_type, + int index, MPI_Comm *out_comm); #endif diff --git a/src/quo.c b/src/quo.c index 56fb43b..4828eef 100644 --- a/src/quo.c +++ b/src/quo.c @@ -582,13 +582,14 @@ QUO_auto_distrib(QUO_t *q, int QUO_get_mpi_comm_by_type(QUO_t *q, QUO_obj_type_t target_type, + int index, MPI_Comm *out_comm) { if (!q || !out_comm) return QUO_ERR_INVLD_ARG; /* make sure we are initialized before we continue */ noinit_action(q); - return quo_mpi_get_comm_by_type(q->mpi, target_type, out_comm); + return quo_mpi_get_comm_by_type(q->mpi, target_type, index, out_comm); } /* ////////////////////////////////////////////////////////////////////////// */ diff --git a/src/quo.h b/src/quo.h index 614b1a8..4be68ff 100644 --- a/src/quo.h +++ b/src/quo.h @@ -611,6 +611,8 @@ QUO_auto_distrib(QUO_context q, * * @param target_type - target hardware object type. (IN) * + * @param indes - the ressource index + * * @param out_comm - MPI_Comm_dup'd communicator containing processes that match * the target request. Returned resources must be freed with a * call to MPI_Comm_free. @@ -620,6 +622,7 @@ QUO_auto_distrib(QUO_context q, int QUO_get_mpi_comm_by_type(QUO_context q, QUO_obj_type_t target_type, + int index, MPI_Comm *out_comm); /** From afb9199042c7fc3fefadbc09cface15edc4d7a48 Mon Sep 17 00:00:00 2001 From: Florian Weik Date: Wed, 14 Sep 2016 12:56:27 -0600 Subject: [PATCH 2/4] Adaped to new signature of get_comm --- demos/quo-mpi-sub-comm.c | 2 +- src/quo.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/demos/quo-mpi-sub-comm.c b/demos/quo-mpi-sub-comm.c index be5651a..4bf29c2 100644 --- a/demos/quo-mpi-sub-comm.c +++ b/demos/quo-mpi-sub-comm.c @@ -182,7 +182,7 @@ test_QUO_get_mpi_comm_by_type(QUO_context quoc, // MPI_Comm quo_node_comm; if (QUO_SUCCESS != (qrc = - QUO_get_mpi_comm_by_type(quoc, QUO_OBJ_MACHINE, &quo_node_comm))) { + QUO_get_mpi_comm_by_type(quoc, QUO_OBJ_MACHINE, 0, &quo_node_comm))) { bad_func = "QUO_get_mpi_comm_by_type"; goto out; } diff --git a/src/quo.c b/src/quo.c index 4828eef..fac2746 100644 --- a/src/quo.c +++ b/src/quo.c @@ -600,10 +600,11 @@ QUO_get_mpi_comm_by_type(QUO_t *q, int QUO_get_mpi_comm_by_type_f2c(QUO_t *q, QUO_obj_type_t target_type, + int index, MPI_Fint *out_comm) { MPI_Comm c_comm; - int rc = QUO_get_mpi_comm_by_type(q, target_type, &c_comm); + int rc = QUO_get_mpi_comm_by_type(q, target_type, index, &c_comm); *out_comm = MPI_Comm_c2f(c_comm); return rc; From 2ce9ddb8903476fa39dc78d5968e45ab8559a77e Mon Sep 17 00:00:00 2001 From: Florian Weik Date: Wed, 14 Sep 2016 13:23:42 -0600 Subject: [PATCH 3/4] Added implementation for get_comm for types other than machine. --- src/quo-mpi.c | 20 +++++++++++++++++++- src/quo-mpi.h | 3 +++ src/quo.c | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/quo-mpi.c b/src/quo-mpi.c index 17193ed..875796e 100644 --- a/src/quo-mpi.c +++ b/src/quo-mpi.c @@ -767,11 +767,13 @@ quo_mpi_sm_barrier(const quo_mpi_t *mpi) /* ////////////////////////////////////////////////////////////////////////// */ int quo_mpi_get_comm_by_type(const quo_mpi_t *mpi, + const quo_hwloc_t *hwloc, QUO_obj_type_t target_type, + int pid, int index, MPI_Comm *out_comm) { - if (!mpi || !out_comm) return QUO_ERR_INVLD_ARG; + if (!mpi || !hwloc || !out_comm) return QUO_ERR_INVLD_ARG; switch (target_type) { case QUO_OBJ_MACHINE: @@ -787,6 +789,22 @@ quo_mpi_get_comm_by_type(const quo_mpi_t *mpi, case QUO_OBJ_SOCKET: case QUO_OBJ_CORE: case QUO_OBJ_PU: + { + /* Check if we are on the selected ressource, and if so include + this rank in the result communicator. Otherwise set the result + to MPI_COMM_NULL (behavior of MPI_Comm_split for color = MPI_UNDEFINED). + */ + int in_resource; + + quo_hwloc_is_in_cpuset_by_type_id(hwloc, target_type, pid, + (unsigned)index, + &in_resource); + + const int color = in_resource ? 0 : MPI_UNDEFINED; + + MPI_Comm_split(mpi->smpcomm, color, mpi->smprank, out_comm); + } + break; default: return QUO_ERR_NOT_SUPPORTED; } diff --git a/src/quo-mpi.h b/src/quo-mpi.h index 32c538d..3048543 100644 --- a/src/quo-mpi.h +++ b/src/quo-mpi.h @@ -49,6 +49,7 @@ #endif #include "quo-private.h" +#include "quo-hwloc.h" #include "quo.h" #include "mpi.h" @@ -93,7 +94,9 @@ quo_mpi_sm_barrier(const quo_mpi_t *mpi); int quo_mpi_get_comm_by_type(const quo_mpi_t *mpi, + const quo_hwloc_t *hwloc, QUO_obj_type_t target_type, + int pid, int index, MPI_Comm *out_comm); #endif diff --git a/src/quo.c b/src/quo.c index fac2746..fa38601 100644 --- a/src/quo.c +++ b/src/quo.c @@ -589,7 +589,7 @@ QUO_get_mpi_comm_by_type(QUO_t *q, /* make sure we are initialized before we continue */ noinit_action(q); - return quo_mpi_get_comm_by_type(q->mpi, target_type, index, out_comm); + return quo_mpi_get_comm_by_type(q->mpi, q->hwloc, target_type, q->pid, index, out_comm); } /* ////////////////////////////////////////////////////////////////////////// */ From 82444481ff971e66e0b577613d7f3b3de1acccbd Mon Sep 17 00:00:00 2001 From: Florian Weik Date: Wed, 14 Sep 2016 13:28:53 -0600 Subject: [PATCH 4/4] Added error handling. --- src/quo-mpi.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/quo-mpi.c b/src/quo-mpi.c index 875796e..756bbaa 100644 --- a/src/quo-mpi.c +++ b/src/quo-mpi.c @@ -795,14 +795,18 @@ quo_mpi_get_comm_by_type(const quo_mpi_t *mpi, to MPI_COMM_NULL (behavior of MPI_Comm_split for color = MPI_UNDEFINED). */ int in_resource; - - quo_hwloc_is_in_cpuset_by_type_id(hwloc, target_type, pid, - (unsigned)index, - &in_resource); + int rc; + if(QUO_SUCCESS != (rc = quo_hwloc_is_in_cpuset_by_type_id(hwloc, target_type, pid, + (unsigned)index, + &in_resource))) { + return rc; + } const int color = in_resource ? 0 : MPI_UNDEFINED; - MPI_Comm_split(mpi->smpcomm, color, mpi->smprank, out_comm); + if (MPI_SUCCESS != MPI_Comm_split(mpi->smpcomm, color, mpi->smprank, out_comm)) { + return QUO_ERR_MPI; + } } break; default: