Fix sm BTL initialization for MPI_Comm_spawn and friends. Thanks to Jeff for

finding the issue.

This commit was SVN r28094.

ompi/mca/btl/sm/btl_sm_component.c

@@ -671,14 +671,18 @@ out:
  */
 static int
 backing_store_init(mca_btl_sm_component_t *comp_ptr,
-                   ompi_node_rank_t node_rank)
+                   orte_local_rank_t local_rank)
 {
     int rc = OMPI_SUCCESS;
 
+    opal_output(0, "hi from: %d\n", (int)local_rank);
+
     if (OMPI_SUCCESS != (rc = set_uniq_paths_for_init_rndv(comp_ptr))) {
         goto out;
     }
-    if (0 == node_rank) {
+    /* only let the lowest rank setup the metadata */
+    if (0 == local_rank) {
+        opal_output(0, "%d creating metadata!!!!\n", (int)local_rank);
         /* === sm mpool === */
         if (OMPI_SUCCESS != (rc =
             create_rndv_file(comp_ptr, MCA_BTL_SM_RNDV_MOD_MPOOL))) {
@@ -705,7 +709,7 @@ mca_btl_sm_component_init(int *num_btls,
 {
     int num_local_procs = 0;
     mca_btl_base_module_t **btls = NULL;
-    ompi_node_rank_t my_node_rank = OMPI_NODE_RANK_INVALID;
+    orte_local_rank_t my_local_rank = OMPI_NODE_RANK_INVALID;
 #if OMPI_BTL_SM_HAVE_KNEM
     int rc;
 #endif /* OMPI_BTL_SM_HAVE_KNEM */
@@ -723,9 +727,13 @@ mca_btl_sm_component_init(int *num_btls,
         return NULL;
     }
     /* if we don't have locality information, then we cannot be used because we
-     * need to know who the respective node ranks for initialization. */
-    if (OMPI_NODE_RANK_INVALID ==
-        (my_node_rank = ompi_process_info.my_node_rank)) {
+     * need to know who the respective node ranks for initialization. note the
+     * use of my_local_rank here. we use this instead of my_node_rank because in
+     * the spawn case we need to designate a metadata creator rank within the
+     * set of processes that are initializing the btl, and my_local_rank seems
+     * to provide that for us. */
+    if (OMPI_LOCAL_RANK_INVALID ==
+        (my_local_rank = ompi_process_info.my_local_rank)) {
         opal_show_help("help-mpi-btl-sm.txt", "no locality", true);
         return NULL;
     }
@@ -742,7 +750,7 @@ mca_btl_sm_component_init(int *num_btls,
      * other local procs can read from it during add_procs. The rest will just
      * stash the known paths for use later in init. */
     if (OMPI_SUCCESS != backing_store_init(&mca_btl_sm_component,
-                                           my_node_rank)) {
+                                           my_local_rank)) {
         return NULL;
     }
 

ompi/mca/rte/orte/rte_orte.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012      Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
  *
  * $COPYRIGHT$
@@ -45,6 +45,7 @@ typedef orte_ns_cmp_bitmask_t ompi_rte_cmp_bitmask_t;
 #define ompi_rte_compare_name_fields(a, b, c) orte_util_compare_name_fields(a, b, c)
 #define OMPI_NAME_WILDCARD  ORTE_NAME_WILDCARD
 #define OMPI_NODE_RANK_INVALID ORTE_NODE_RANK_INVALID
+#define OMPI_LOCAL_RANK_INVALID ORTE_LOCAL_RANK_INVALID
 #define OMPI_RTE_CMP_JOBID  ORTE_NS_CMP_JOBID
 #define OMPI_RTE_CMP_VPID   ORTE_NS_CMP_VPID
 #define OMPI_RTE_CMP_ALL    ORTE_NS_CMP_ALL

ompi/mca/rte/pmi/rte_pmi.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012      Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2013      Sandia National Laboratories. All rights reserved.
  *
@@ -92,6 +92,7 @@ struct ompi_process_info_t {
 };
 typedef struct ompi_process_info_t ompi_process_info_t;
 #define OMPI_NODE_RANK_INVALID (-1)
+#define OMPI_LOCAL_RANK_INVALID (-1)
 
 
 OMPI_DECLSPEC extern ompi_process_info_t ompi_process_info;