* add mode for utcp compat code where modex is not used. Instead, use the

"run-time" api for the reference implementation.
* Make the non-modex utcp and redstorm compat code do the same things in
  the same order

This commit was SVN r6556.

ompi/mca/btl/portals/src/btl_portals_compat_redstorm.c

@@ -30,16 +30,6 @@ mca_btl_portals_init_compat(mca_btl_portals_component_t *comp)
     uint32_t i;
     struct mca_btl_portals_module_t *btl;
 
-    /*
-     * Initialize Portals interface
-     */
-    ret = PtlInit(&max_interfaces);
-    if (PTL_OK != ret) {
-        opal_output_verbose(10, mca_btl_portals_component.portals_output,
-                            "PtlInit failed, returning %d\n", ret);
-        return OMPI_ERR_FATAL;
-    }
-
     /*
      * create module - only ever one "NIC" on red storm
      */
@@ -64,6 +54,16 @@ mca_btl_portals_init_compat(mca_btl_portals_component_t *comp)
         /* the defaults are good enough for the rest */
     }
 
+    /*
+     * Initialize Portals interface
+     */
+    ret = PtlInit(&max_interfaces);
+    if (PTL_OK != ret) {
+        opal_output_verbose(10, mca_btl_portals_component.portals_output,
+                            "PtlInit failed, returning %d\n", ret);
+        return OMPI_ERR_FATAL;
+    }
+
     /*
      * Initialize a network device
      */

ompi/mca/btl/portals/src/btl_portals_compat_utcp.c

@@ -39,13 +39,16 @@ extern unsigned int utcp_my_nid(const char *if_str);
 FILE* utcp_api_out;
 FILE* utcp_lib_out;
 
+static bool use_modex = true;
+
 int
 mca_btl_portals_init_compat(mca_btl_portals_component_t *comp)
 {
     ptl_process_id_t info;
-    int ret;
+    int ret, max_interfaces;
     uint32_t i;
-#if 0
+    struct mca_btl_portals_module_t *btl;
+#if 0 /* send all the portals internal debug to a file or stderr */
     FILE *output;
     char *tmp;
 
@@ -60,6 +63,41 @@ mca_btl_portals_init_compat(mca_btl_portals_component_t *comp)
     utcp_api_out = stderr;
 #endif
 
+    /* if the environment variables for the utcp implementation are
+       already set, assume the user is running without the full Open
+       RTE and is doing RTE testing for a more tightly-coupled
+       platform (like, say, Red Storm).  Otherwise, be nice and use
+       the modex to setup everything for the user */
+    if (NULL == getenv("PTL_MY_RID")) {
+        use_modex = true;
+    } else {
+        use_modex = false;
+    }
+
+    /* with the utcp interface, only ever one "NIC" */
+    comp->portals_num_modules = 1;
+    comp->portals_modules = calloc(comp->portals_num_modules,
+                                   sizeof(mca_btl_portals_module_t));
+    if (NULL == comp->portals_modules) {
+        opal_output_verbose(10, mca_btl_portals_component.portals_output,
+                            "malloc failed in mca_btl_portals_init");
+        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+    }
+    btl = &(comp->portals_modules[0]);
+
+    /* compat code is responsible for copying over the "template" onto
+       each module instance.  The calling code will create the free
+       lists and the like - we're only responsible for the
+       Portals-specific entries */
+    for (i = 0 ; i < comp->portals_num_modules ; ++i) {
+        memcpy(&(comp->portals_modules[i]), 
+               &mca_btl_portals_module,
+               sizeof(mca_btl_portals_module_t));
+        /* the defaults are good enough for the rest */
+    }
+
+    if (use_modex) {
+        /* post our contact info in the registry */
         info.nid = htonl(utcp_my_nid(mca_btl_portals_component.portals_ifname));
         info.pid = htonl((ptl_pid_t) getpid());
         opal_output_verbose(100, mca_btl_portals_component.portals_output,
@@ -73,26 +111,31 @@ mca_btl_portals_init_compat(mca_btl_portals_component_t *comp)
                                 "mca_base_modex_send failed: %d", ret);
             return ret;
         }
-
+    } else {
-    /* with the utcp interface, only ever one "NIC" */
+        /*
-    comp->portals_num_modules = 1;
+         * Initialize Portals interface
-    comp->portals_modules = calloc(comp->portals_num_modules,
+         */
-                                   sizeof(mca_btl_portals_module_t));
+        ret = PtlInit(&max_interfaces);
-    if (NULL == comp->portals_modules) {
+        if (PTL_OK != ret) {
             opal_output_verbose(10, mca_btl_portals_component.portals_output,
-                            "malloc failed in mca_btl_portals_init");
+                                "PtlInit failed, returning %d\n", ret);
-        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+            return OMPI_ERR_FATAL;
         }
 
-    /* compat code is responsible for copying over the "template" onto
+        /*
-       each module instance.  The calling code will create the free
+         * Initialize a network device
-       lists and the like - we're only responsible for the
+         */
-       Portals-specific entries */
+        ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */
-    for (i = 0 ; i < comp->portals_num_modules ; ++i) {
+                        PTL_PID_ANY,       /* let library assign our pid */
-        memcpy(&(comp->portals_modules[i]), 
+                        NULL,              /* no desired limits */
-               &mca_btl_portals_module,
+                        &(btl->portals_ni_limits),    /* save our limits somewhere */
-               sizeof(mca_btl_portals_module_t));
+                        &(btl->portals_ni_h)  /* our interface handle */
-        /* the defaults are good enough for the rest */
+                        );
+        if (PTL_OK != ret) {
+            opal_output_verbose(10, mca_btl_portals_component.portals_output,
+                                "PtlNIInit failed, returning %d\n", ret);
+            return OMPI_ERR_FATAL;
+        }
     }
 
     return OMPI_SUCCESS;
@@ -116,6 +159,7 @@ mca_btl_portals_add_procs_compat(struct mca_btl_portals_module_t* btl,
     ompi_proc_t* proc_self = ompi_proc_local();
     int max_interfaces;
 
+    if (use_modex) {
         /*
          * Do all the NID/PID map setup
          */
@@ -124,7 +168,8 @@ mca_btl_portals_add_procs_compat(struct mca_btl_portals_module_t* btl,
         pidmap = malloc(map_size);
         nid_str = malloc(12 + 1);
         pid_str = malloc(12 + 1);
-    if (NULL == nidmap || NULL == pidmap || NULL == nid_str || NULL == pid_str)
+        if (NULL == nidmap || NULL == pidmap || 
+            NULL == nid_str || NULL == pid_str)
             return OMPI_ERROR;
 
         /* get space for the portals procs list */
@@ -217,6 +262,44 @@ mca_btl_portals_add_procs_compat(struct mca_btl_portals_module_t* btl,
 #if 0
         PtlNIDebug(btl->portals_ni_h, PTL_DBG_ALL | PTL_DBG_NI_ALL);
 #endif
+    } else { /* use_modex */
+        int nptl_procs = 0;
+
+        /*
+         * FIXME - XXX - FIXME 
+         * BWB - implicit assumption that cnos procs list will match our
+         *       procs list.  Don't know what to do about that...
+         */
+        ret = PtlGetRank(&my_rid, &nptl_procs);
+        if (ret != PTL_OK) {
+            opal_output_verbose(10, mca_btl_portals_component.portals_output,
+                                "PtlGetRank() returned %d", ret);
+            return OMPI_ERR_FATAL;
+        } else if (nptl_procs != nprocs) {
+            opal_output_verbose(10, mca_btl_portals_component.portals_output,
+                                "nptl_procs != nprocs (%d, %d)", nptl_procs,
+                                nprocs);
+            return OMPI_ERR_FATAL;
+        }
+
+        /* create enough space for all the proc info structs */
+        *portals_procs = calloc(nprocs, sizeof(ptl_process_id_t));
+        if (NULL == *portals_procs) {
+            opal_output_verbose(10, mca_btl_portals_component.portals_output,
+                                "calloc(nprocs, sizeof(ptl_process_id_t)) failed");
+            return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+        }
+        /* fill in all the proc info structs */
+        for (i = 0 ; i < nprocs ; ++i) {
+            ret = PtlGetRankId(i, &((*portals_procs)[i]));
+            if (PTL_OK != ret) {
+                opal_output_verbose(10,
+                                    mca_btl_portals_component.portals_output,
+                                    "PtlGetRankId(%d) failed: %d\n", i, ret);
+                return OMPI_ERR_FATAL;
+            }
+        }
+    }
 
     return OMPI_SUCCESS;
 }