Mpi stuff

demos/quo-mpi-sub-comm.c

@@ -182,7 +182,7 @@ test_QUO_get_mpi_comm_by_type(QUO_context quoc,
     //
     MPI_Comm quo_node_comm;
     if (QUO_SUCCESS != (qrc =
-        QUO_get_mpi_comm_by_type(quoc, QUO_OBJ_MACHINE, &quo_node_comm))) {
+                        QUO_get_mpi_comm_by_type(quoc, QUO_OBJ_MACHINE, 0, &quo_node_comm))) {
         bad_func = "QUO_get_mpi_comm_by_type";
         goto out;
     }

src/quo-mpi.c

@@ -767,25 +767,51 @@ quo_mpi_sm_barrier(const quo_mpi_t *mpi)
 /* ////////////////////////////////////////////////////////////////////////// */
 int
 quo_mpi_get_comm_by_type(const quo_mpi_t *mpi,
+                         const quo_hwloc_t *hwloc,
                          QUO_obj_type_t target_type,
+                         int pid,
+                         int index,
                          MPI_Comm *out_comm)
 {
-    if (!mpi || !out_comm) return QUO_ERR_INVLD_ARG;
-
-    switch (target_type) {
-        case QUO_OBJ_MACHINE:
-        {
-            /* this case is easy. just return a dup of the smp communicator that
-             * we already maintain internally. */
-            if (MPI_SUCCESS != MPI_Comm_dup(mpi->smpcomm, out_comm)) {
-                return QUO_ERR_MPI;
-            }
-            break;
-        }
-        /* TODO add support for other obj types */
-        default:
-            return QUO_ERR_NOT_SUPPORTED;
+  if (!mpi || !hwloc || !out_comm) return QUO_ERR_INVLD_ARG;
+
+  switch (target_type) {
+  case QUO_OBJ_MACHINE:
+    {
+      /* this case is easy. just return a dup of the smp communicator that
+      o * we already maintain internally. */
+      if (MPI_SUCCESS != MPI_Comm_dup(mpi->smpcomm, out_comm)) {
+        return QUO_ERR_MPI;
+      }
+      break;
+    }
+  case QUO_OBJ_NODE:
+  case QUO_OBJ_SOCKET:
+  case QUO_OBJ_CORE:
+  case QUO_OBJ_PU:
+    {
+      /* Check if we are on the selected ressource, and if so include
+         this rank in the result communicator. Otherwise set the result
+         to MPI_COMM_NULL (behavior of MPI_Comm_split for color = MPI_UNDEFINED).
+      */
+      int in_resource;
+      int rc;
+      if(QUO_SUCCESS != (rc = quo_hwloc_is_in_cpuset_by_type_id(hwloc, target_type, pid,
+                                                                (unsigned)index,
+                                                                &in_resource))) {
+        return rc;
+      }
+
+      const int color = in_resource ? 0 : MPI_UNDEFINED;
+
+      if (MPI_SUCCESS != MPI_Comm_split(mpi->smpcomm, color, mpi->smprank, out_comm)) {
+        return QUO_ERR_MPI;
+      }
     }
+    break;
+  default:
+    return QUO_ERR_NOT_SUPPORTED;
+  }
 
-    return QUO_SUCCESS;
+  return QUO_SUCCESS;
 }

src/quo-mpi.h

@@ -49,6 +49,7 @@
 #endif
 
 #include "quo-private.h"
+#include "quo-hwloc.h"
 #include "quo.h"
 
 #include "mpi.h"
@@ -93,6 +94,9 @@ quo_mpi_sm_barrier(const quo_mpi_t *mpi);
 
 int
 quo_mpi_get_comm_by_type(const quo_mpi_t *mpi,
+                         const quo_hwloc_t *hwloc,
                          QUO_obj_type_t target_type,
+                         int pid,
+                         int index,
                          MPI_Comm *out_comm);
 #endif

src/quo.c

@@ -582,13 +582,14 @@ QUO_auto_distrib(QUO_t *q,
 int
 QUO_get_mpi_comm_by_type(QUO_t *q,
                          QUO_obj_type_t target_type,
+                         int index,
                          MPI_Comm *out_comm)
 {
     if (!q || !out_comm) return QUO_ERR_INVLD_ARG;
     /* make sure we are initialized before we continue */
     noinit_action(q);
 
-    return quo_mpi_get_comm_by_type(q->mpi, target_type, out_comm);
+    return quo_mpi_get_comm_by_type(q->mpi, q->hwloc, target_type, q->pid, index, out_comm);
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
@@ -599,10 +600,11 @@ QUO_get_mpi_comm_by_type(QUO_t *q,
 int
 QUO_get_mpi_comm_by_type_f2c(QUO_t *q,
                              QUO_obj_type_t target_type,
+                             int index,
                              MPI_Fint *out_comm)
 {
     MPI_Comm c_comm;
-    int rc = QUO_get_mpi_comm_by_type(q, target_type, &c_comm);
+    int rc = QUO_get_mpi_comm_by_type(q, target_type, index, &c_comm);
     *out_comm = MPI_Comm_c2f(c_comm);
 
     return rc;

src/quo.h

@@ -611,6 +611,8 @@ QUO_auto_distrib(QUO_context q,
  *
  * @param target_type - target hardware object type. (IN)
  *
+ * @param indes - the ressource index
+ *
  * @param out_comm - MPI_Comm_dup'd communicator containing processes that match
  *                   the target request. Returned resources must be freed with a
  *                   call to MPI_Comm_free.
@@ -620,6 +622,7 @@ QUO_auto_distrib(QUO_context q,
 int
 QUO_get_mpi_comm_by_type(QUO_context q,
                          QUO_obj_type_t target_type,
+                         int index,
                          MPI_Comm *out_comm);
 
 /**