Add support for detecting when dynamic add_procs is not possible

This commit adds support to the pml, mtl, and btl frameworks for
components to indicate at runtime that they do not support the new
dynamic add_procs behavior. At the high end the lack of dynamic
add_procs support is signalled by the pml using the new pml_flags
member to the pml module structure. If the
MCA_PML_BASE_FLAG_REQUIRE_WORLD flag is set MPI_Init will generate the
ompi_proc_t array passed to add_proc from ompi_proc_world () instead
of ompi_proc_get_allocated ().

Both cm and ob1 have been updated to detect if the underlying mtl and
btl components support dynamic add_procs.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>

ompi/mca/mtl/mtl.h

@@ -56,6 +56,12 @@ struct mca_mtl_request_t {
 };
 typedef struct mca_mtl_request_t mca_mtl_request_t;
 
+
+/**
+ * MTL module flags
+ */
+#define MCA_MTL_BASE_FLAG_REQUIRE_WORLD 0x00000001
+
 /**
  * Initialization routine for MTL component
  *

ompi/mca/mtl/portals4/mtl_portals4_component.c

@@ -232,6 +232,8 @@ ompi_mtl_portals4_component_open(void)
     return OMPI_SUCCESS;
 }
 
+#define NEED_ALL_PROCS (ompi_mtl_portals4.use_logical || ompi_mtl_portals4.use_flowctl)
+
 static int
 ompi_mtl_portals4_component_query(mca_base_module_t **module, int *priority)
 {
@@ -241,6 +243,13 @@ ompi_mtl_portals4_component_query(mca_base_module_t **module, int *priority)
 
     *priority = param_priority;
     *module = (mca_base_module_t *)&ompi_mtl_portals4.base;
+
+    if (NEED_ALL_PROCS) {
+        /* let the pml know we need add_procs to be calls with all the
+         * procs in the job */
+        ompi_mtl_portals4.base.mtl_flags |= MCA_MTL_BASE_FLAG_REQUIRE_WORLD;
+    }
+
     return OMPI_SUCCESS;
 }
 

ompi/mca/pml/cm/pml_cm_component.c

@@ -169,6 +169,10 @@ mca_pml_cm_component_init(int* priority,
     }
 
 
+    if (ompi_mtl->mtl_flags & MCA_MTL_BASE_FLAG_REQUIRE_WORLD) {
+        ompi_pml_cm.super.pml_flags |= MCA_PML_BASE_FLAG_REQUIRE_WORLD;
+    }
+
     /* update our tag / context id max values based on MTL
        information */
     ompi_pml_cm.super.pml_max_contextid = ompi_mtl->mtl_max_contextid;

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -38,6 +38,7 @@
 #include "opal/mca/allocator/base/base.h"
 #include "opal/mca/base/mca_base_pvar.h"
 #include "opal/runtime/opal_params.h"
+#include "opal/mca/btl/base/base.h"
 
 OBJ_CLASS_INSTANCE( mca_pml_ob1_pckt_pending_t,
                     opal_free_list_item_t,
@@ -278,6 +279,17 @@ mca_pml_ob1_component_init( int* priority,
         return NULL;
     }
 
+    /* check if any btls do not support dynamic add_procs */
+    mca_btl_base_selected_module_t* selected_btl;
+    OPAL_LIST_FOREACH(selected_btl, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
+        mca_btl_base_module_t *btl = selected_btl->btl_module;
+
+        if (btl->btl_flags & MCA_BTL_FLAGS_SINGLE_ADD_PROCS) {
+            mca_pml_ob1.super.pml_flags |= MCA_PML_BASE_FLAG_REQUIRE_WORLD;
+            break;
+        }
+    }
+
     /* Set this here (vs in component_open()) because
        opal_leave_pinned* may have been set after MCA params were
        read (e.g., by the openib btl) */

ompi/mca/pml/pml.h

@@ -93,7 +93,7 @@ struct ompi_proc_t;
  * indicates whether multiple threads may invoke this component
  * simultaneously or not.
  */
-typedef struct mca_pml_base_module_1_0_0_t * (*mca_pml_base_component_init_fn_t)(
+typedef struct mca_pml_base_module_1_0_1_t * (*mca_pml_base_component_init_fn_t)(
     int *priority,
     bool enable_progress_threads,
     bool enable_mpi_threads);
@@ -479,13 +479,18 @@ typedef int (*mca_pml_base_module_dump_fn_t)(
  */
 typedef int (*mca_pml_base_module_ft_event_fn_t) (int status);
 
-
+/**
+ * pml module flags
+ */
+/** PML requires requires all procs in the job on the first call to
+ * add_procs */
+#define MCA_PML_BASE_FLAG_REQUIRE_WORLD 0x00000001
 
 /**
  *  PML instance.
  */
 
-struct mca_pml_base_module_1_0_0_t {
+struct mca_pml_base_module_1_0_1_t {
 
     /* downcalls from MCA to PML */
     mca_pml_base_module_add_procs_fn_t    pml_add_procs;
@@ -519,9 +524,10 @@ struct mca_pml_base_module_1_0_0_t {
     /* maximum constant sizes */
     uint32_t                              pml_max_contextid;
     int                                   pml_max_tag;
+    int                                   pml_flags;
 };
-typedef struct mca_pml_base_module_1_0_0_t mca_pml_base_module_1_0_0_t;
-typedef mca_pml_base_module_1_0_0_t mca_pml_base_module_t;
+typedef struct mca_pml_base_module_1_0_1_t mca_pml_base_module_1_0_1_t;
+typedef mca_pml_base_module_1_0_1_t mca_pml_base_module_t;
 
 /*
  * Macro for use in components that are of type pml
@@ -546,6 +552,10 @@ typedef mca_pml_base_module_1_0_0_t mca_pml_base_module_t;
 
 OMPI_DECLSPEC extern mca_pml_base_module_t mca_pml;
 
+static inline bool mca_pml_base_requires_world (void)
+{
+    return !!(mca_pml.pml_flags & MCA_PML_BASE_FLAG_REQUIRE_WORLD);
+}
 
 END_C_DECLS
 #endif /* MCA_PML_H */

ompi/runtime/ompi_mpi_init.c

@@ -739,11 +739,21 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
         goto error;
     }
 
-    /* add all allocated ompi_proc_t's to PML (below the add_procs limit this
-     * behaves identically to ompi_proc_world ()) */
-    if (NULL == (procs = ompi_proc_get_allocated (&nprocs))) {
-        error = "ompi_proc_get_allocated () failed";
-        goto error;
+    /* some btls/mtls require we call add_procs with all procs in the job.
+     * since the btls/mtls have no visibility here it is up to the pml to
+     * convey this requirement */
+    if (mca_pml_base_requires_world ()) {
+        if (NULL == (procs = ompi_proc_world (&nprocs))) {
+            error = "ompi_proc_get_allocated () failed";
+            goto error;
+        }
+    } else {
+        /* add all allocated ompi_proc_t's to PML (below the add_procs limit this
+         * behaves identically to ompi_proc_world ()) */
+        if (NULL == (procs = ompi_proc_get_allocated (&nprocs))) {
+            error = "ompi_proc_get_allocated () failed";
+            goto error;
+        }
     }
     ret = MCA_PML_CALL(add_procs(procs, nprocs));
     free(procs);

opal/mca/btl/btl.h

@@ -231,12 +231,15 @@ typedef uint8_t mca_btl_base_tag_t;
  */
 #define MCA_BTL_FLAGS_SIGNALED        0x4000
 
-
 /** The BTL supports network atomic operations */
 #define MCA_BTL_FLAGS_ATOMIC_OPS      0x08000
 /** The BTL supports fetching network atomic operations */
 #define MCA_BTL_FLAGS_ATOMIC_FOPS     0x10000
 
+/** The BTL requires add_procs to be with all procs including non-local. Shared-memory
+ * BTLs should not set this flag. */
+#define MCA_BTL_FLAGS_SINGLE_ADD_PROCS 0x20000
+
 /* Default exclusivity levels */
 #define MCA_BTL_EXCLUSIVITY_HIGH     (64*1024) /* internal loopback */
 #define MCA_BTL_EXCLUSIVITY_DEFAULT  1024      /* GM/IB/etc. */