initial fixes for ugni btl move to opal

This commit was SVN r32466.

opal/mca/btl/ugni/btl_ugni.h

@@ -29,6 +29,7 @@
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/class/opal_hash_table.h"
 #include "opal/class/ompi_free_list.h"
 #include "opal/mca/common/ugni/common_ugni.h"
 
@@ -288,9 +289,9 @@ OPAL_MODULE_DECLSPEC extern mca_btl_ugni_component_t mca_btl_ugni_component;
 OPAL_MODULE_DECLSPEC extern mca_btl_ugni_module_t mca_btl_ugni_module;
 
 /* Get a unique 64-bit id for the process name */
-static inline uint64_t mca_btl_ugni_proc_name_to_id (ompi_process_name_t name) {
+static inline uint64_t mca_btl_ugni_proc_name_to_id (opal_process_name_t name) {
     /* Throw away the top bit of the jobid for the datagram type */
-    return ((uint64_t) (name.jobid & 0x7fffffff) << 32 | (uint64_t) name.vpid);
+    return ((uint64_t) (opal_process_name_jobid(name) & 0x7fffffff) << 32 | opal_process_name_vpid(name));
 }
 
 #endif

opal/mca/btl/ugni/btl_ugni_add_procs.c

@@ -31,11 +31,14 @@ int mca_btl_ugni_add_procs(struct mca_btl_base_module_t* btl,
                            struct mca_btl_base_endpoint_t **peers,
                            opal_bitmap_t *reachable) {
     mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    opal_proc_t *my_proc = ompi_proc_local ();
+    opal_proc_t *my_proc = opal_proc_local_get();
     size_t ntotal_procs, i;
     int rc;
 
     if (false == ugni_module->initialized) {
+
+        /* TODO: fix me don't do this now that btl has moved in to opal */
+
         ntotal_procs = ompi_comm_size ((ompi_communicator_t *) MPI_COMM_WORLD);
 
         rc = opal_pointer_array_init (&ugni_module->endpoints, ntotal_procs, 1 << 24, 512);
@@ -233,6 +236,7 @@ mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module)
     }
 
     /* determine how many procs are in the job (might want to check universe size here) */
+    /* TODO: need to fix this with something else now that btl is in opal */
     nprocs = ompi_comm_size ((ompi_communicator_t *) MPI_COMM_WORLD);
 
     rc = mca_btl_ugni_smsg_setup (nprocs);

opal/mca/btl/ugni/btl_ugni_component.c

@@ -16,6 +16,7 @@
 #include "btl_ugni_smsg.h"
 
 #include "opal/memoryhooks/memory.h"
+#include "opal/runtime/opal_params.h"
 #include "ompi/runtime/params.h"
 
 static int btl_ugni_component_register(void);
@@ -286,7 +287,7 @@ mca_btl_ugni_component_init (int *num_btl_modules,
     int rc;
 
     /* Currently refuse to run if MPI_THREAD_MULTIPLE is enabled */
-    if (ompi_mpi_thread_multiple && !mca_btl_base_thread_multiple_override) {
+    if (opal_using_threads() && !mca_btl_base_thread_multiple_override) {
         opal_output_verbose(5, opal_btl_base_framework.framework_output,
                             "btl:ugni: MPI_THREAD_MULTIPLE not supported; skipping this component");
         return NULL;

opal/mca/btl/ugni/btl_ugni_endpoint.c

@@ -35,7 +35,7 @@ static inline int mca_btl_ugni_ep_smsg_get_mbox (mca_btl_base_endpoint_t *ep) {
     mca_btl_ugni_module_t *ugni_module = ep->btl;
     ompi_free_list_item_t *mbox;
 
-    OPAL_FREE_LIST_GET_MT(&ugni_module->smsg_mboxes, mbox);
+    OMPI_FREE_LIST_GET_MT(&ugni_module->smsg_mboxes, mbox);
     if (OPAL_UNLIKELY(NULL == mbox)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }

opal/mca/btl/ugni/btl_ugni_prepare.h

@@ -25,7 +25,6 @@ mca_btl_ugni_prepare_src_send_nodata (struct mca_btl_base_module_t *btl,
                                       uint32_t flags)
 {
     mca_btl_ugni_base_frag_t *frag = NULL;
-    int rc;
 
     (void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
     if (OPAL_UNLIKELY(NULL == frag)) {

opal/mca/btl/ugni/btl_ugni_send.c

@@ -37,7 +37,7 @@ int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
     }
 
     BTL_VERBOSE(("btl/ugni sending descriptor %p from %d -> %d. length = %" PRIu64, (void *)descriptor,
-                 opal_process_name_vpid(OPAL_PROC_MY_NAME, endpoint->common->ep_rem_id, frag->segments[0].base.seg_len));
+                 opal_process_name_vpid(OPAL_PROC_MY_NAME), endpoint->common->ep_rem_id, frag->segments[0].base.seg_len));
 
     /* temporarily disable ownership and callback flags so we can reliably check the complete flag */
     frag->base.des_flags &= ~(MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK);

opal/mca/common/ugni/common_ugni.c

@@ -156,7 +156,7 @@ static int opal_common_ugni_device_fini (opal_common_ugni_device_t *dev)
  * Send local device information and other information
  * required for setup
  */
-static int opal_common_ugni_send_modex (int my_rank)
+static int opal_common_ugni_send_modex (int my_cdm_id)
 {
     uint32_t modex_size, total_msg_size, msg_offset;
     struct opal_common_ugni_modex_t modex;
@@ -178,7 +178,7 @@ static int opal_common_ugni_send_modex (int my_rank)
         opal_common_ugni_device_t *dev = opal_common_ugni_module.devices + i;
 
         modex.addr = dev->dev_addr;
-        modex.id   = my_rank;
+        modex.id   = my_cdm_id;
 
         memcpy ((void *)((uintptr_t) modex_msg + msg_offset),
                 (void *)&modex, modex_size);
@@ -228,9 +228,11 @@ int opal_common_ugni_fini (void)
 
 int opal_common_ugni_init (void)
 {
+#if 0
     opal_proc_t *my_proc;
+#endif
     int modes, rc, i;
-    uint32_t my_rank, *ptr;
+    uint32_t my_cdm_id, *ptr;
 
     opal_common_ugni_module_ref_count ++;
 
@@ -238,9 +240,10 @@ int opal_common_ugni_init (void)
         return OPAL_SUCCESS;
     }
 
-    my_proc = opal_proc_get_local ();
-
     /* get a unique id from the runtime */
+    /* the code below is unnecessary.  The cdm_id only needs to be unique
+       within a node for a given ptag/cookie tuple */
+#if 0
 #if defined(OMPI_DB_GLOBAL_RANK)
     {
         opal_list_t myvals;
@@ -265,6 +268,9 @@ int opal_common_ugni_init (void)
 #else
     my_rank = my_proc->proc_name.vpid;
 #endif
+#else
+    my_cdm_id = getpid();   /*TODO: eventually need something else for thread-hot support */
+#endif
 
     /* pull settings from ugni btl */
     opal_common_ugni_module.rdma_max_retries =
@@ -286,7 +292,7 @@ int opal_common_ugni_init (void)
     }
 
     /* create a communication domain */
-    rc = GNI_CdmCreate (my_rank, opal_common_ugni_module.ptag,
+    rc = GNI_CdmCreate (my_cdm_id, opal_common_ugni_module.ptag,
                         opal_common_ugni_module.cookie, modes,
                         &opal_common_ugni_module.cd_handle);
     if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
@@ -308,7 +314,7 @@ int opal_common_ugni_init (void)
     }
 
     /* send ugni modex */
-    opal_common_ugni_send_modex (my_rank);
+    opal_common_ugni_send_modex (my_cdm_id);
 
     return OPAL_SUCCESS;
 }

opal/mca/common/ugni/common_ugni.h

@@ -13,6 +13,7 @@
 #include "opal_config.h"
 
 #include "opal/util/output.h"
+#include "opal/util/proc.h"
 #include "opal/class/opal_list.h"
 #include "opal/include/opal/prefetch.h"
 #include "opal_stdint.h"