btl/ugni: add a request_progress_thread mca param

Replace temporary environment variables with a MCA
parameter for the ugni btl.  A user wishing to
use the ugni btl async. progress thread needs to
set the request_progress_thread param to true.
For example, using env. variable format:

export OMPI_MCA_btl_ugni_request_progress_thread=1

opal/mca/btl/ugni/btl_ugni.h

@@ -47,8 +47,6 @@
 #define MCA_BTL_UGNI_CONNECT_DIRECTED_ID 0x8000000000000000ull
 #define MCA_BTL_UGNI_DATAGRAM_MASK       0x8000000000000000ull
 
-extern int howards_progress_var;
-
 /* ompi and smsg endpoint attributes */
 typedef struct mca_btl_ugni_endpoint_attr_t {
     uint64_t proc_id;
@@ -181,8 +179,11 @@ typedef struct mca_btl_ugni_component_t {
     /* Number of mailboxes to allocate in each block */
     unsigned int mbox_increment;
 
+    /* Indicate whether progress thread requested */
+    bool progress_thread_requested;
+
     /* Indicate whether progress thread allowed */
-    bool progress_thread_allowed;
+    bool progress_thread_enabled;
 
 } mca_btl_ugni_component_t;
 

opal/mca/btl/ugni/btl_ugni_add_procs.c

@@ -20,8 +20,6 @@
 #include "opal/include/opal/align.h"
 #include "opal/mca/dstore/dstore.h"
 
-extern int howards_progress_var;
-
 #define INITIAL_GNI_EPS 10000
 
 static int
@@ -30,8 +28,6 @@ static void
 mca_btl_ugni_module_set_max_reg (mca_btl_ugni_module_t *ugni_module, int nlocal_procs);
 static int mca_btl_ugni_smsg_setup (int nprocs);
 
-void *howards_start_addr;
-
 int mca_btl_ugni_add_procs(struct mca_btl_base_module_t* btl,
                            size_t nprocs,
                            struct opal_proc_t **procs,
@@ -41,6 +37,7 @@ int mca_btl_ugni_add_procs(struct mca_btl_base_module_t* btl,
     opal_proc_t *my_proc = opal_proc_local_get();
     size_t i;
     int rc;
+    void *mmap_start_addr;
 
     if (false == ugni_module->initialized) {
 
@@ -123,7 +120,7 @@ int mca_btl_ugni_add_procs(struct mca_btl_base_module_t* btl,
             return opal_common_rc_ugni_to_opal (rc);
         }
 
-        if (howards_progress_var) {
+        if (mca_btl_ugni_component.progress_thread_enabled) {
             OPAL_THREAD_LOCK(&ugni_module->device->dev_lock);
             rc = GNI_CqCreate (ugni_module->device->dev_handle, mca_btl_ugni_component.local_cq_size,
                                0, GNI_CQ_BLOCKING, NULL, NULL, &ugni_module->rdma_local_irq_cq);
@@ -175,15 +172,24 @@ int mca_btl_ugni_add_procs(struct mca_btl_base_module_t* btl,
             return rc;
         }
 
-        if (howards_progress_var) {
-            howards_start_addr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-            if (howards_start_addr == NULL) {
-                fprintf(stderr,"Hey, mmap returned NULL!\b");
+        /*
+         * If progress thread enabled, registered a page of memory
+         * with the smsg_remote_irq_cq.  This memory handle is passed
+         * to ranks which want to communicate with this rank. A rank which
+         * posts a GNI_PostCqWrite targeting this memory handle generates
+         * an IRQ at the target node, which ultimately causes the progress
+         * thread in the target rank to become schedulable.
+         */
+        if (mca_btl_ugni_component.progress_thread_enabled) {
+            mmap_start_addr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+            if (NULL == mmap_start_addr) {
+                BTL_ERROR(("btl/ugni mmap returned error"));
+                return OPAL_ERR_OUT_OF_RESOURCE;
             }
 
             OPAL_THREAD_LOCK(&ugni_module->device->dev_lock);
             rc = GNI_MemRegister(ugni_module->device->dev_handle,
-                                     (unsigned long)howards_start_addr,
+                                     (unsigned long)mmap_start_addr,
                                      4096,
                                      ugni_module->smsg_remote_irq_cq,
                                      GNI_MEM_READWRITE,

opal/mca/btl/ugni/btl_ugni_component.c

@@ -189,6 +189,16 @@ btl_ugni_component_register(void)
                                            MCA_BASE_VAR_SCOPE_LOCAL,
                                            &mca_btl_ugni_component.smsg_page_size);
 
+    mca_btl_ugni_component.progress_thread_requested = 0;
+    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
+                                           "request_progress_thread",
+                                           "Enable to request ugni btl progress thread - requires MPI_THREAD_MULTIPLE support",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE,
+                                           OPAL_INFO_LVL_3,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
+                                           &mca_btl_ugni_component.progress_thread_requested);
+
     /* btl/ugni can only support only a fixed set of mpools (these mpools have compatible resource
      * structures) */
     rc = mca_base_var_enum_create ("btl_ugni_mpool", mpool_values, &new_enum);
@@ -293,8 +303,8 @@ mca_btl_ugni_component_init (int *num_btl_modules,
         mca_btl_ugni_component.ugni_fma_limit = 65536;
     }
 
-    if (enable_mpi_threads) {
-        mca_btl_ugni_component.progress_thread_allowed = 1;
+    if (enable_mpi_threads && mca_btl_ugni_component.progress_thread_requested) {
+        mca_btl_ugni_component.progress_thread_enabled = 1;
     }
 
     /* Initialize ugni library and create communication domain */
@@ -568,7 +578,7 @@ static int mca_btl_ugni_component_progress (void)
         count += mca_btl_ugni_progress_local_smsg (ugni_module);
         count += mca_btl_ugni_progress_remote_smsg (ugni_module);
         count += mca_btl_ugni_progress_rdma (ugni_module, 0);
-        if (howards_progress_var) {
+        if (mca_btl_ugni_component.progress_thread_enabled) {
             count += mca_btl_ugni_progress_rdma (ugni_module, 1);
         }
     }

opal/mca/btl/ugni/btl_ugni_module.c

@@ -20,8 +20,6 @@
 #include "btl_ugni_prepare.h"
 #include "btl_ugni_smsg.h"
 
-int howards_progress_var = 0;
-
 static int
 mca_btl_ugni_free (struct mca_btl_base_module_t *btl,
                    mca_btl_base_descriptor_t *des);
@@ -116,10 +114,6 @@ mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
         return rc;
     }
 
-    if (mca_btl_ugni_component.progress_thread_allowed && (NULL != getenv("HOWARDS_PROGESS"))) {
-        howards_progress_var = 1;
-    }
-
     return OPAL_SUCCESS;
 }
 
@@ -151,7 +145,7 @@ mca_btl_ugni_module_finalize (struct mca_btl_base_module_t *btl)
             rc = opal_hash_table_get_next_key_uint64 (&ugni_module->id_to_endpoint, &key, (void **) &ep, node, &node);
         }
 
-        if (howards_progress_var) {
+        if (mca_btl_ugni_component.progress_thread_enabled) {
             mca_btl_ugni_kill_progress_thread();
         }
 

opal/mca/btl/ugni/btl_ugni_rdma.h

@@ -68,7 +68,7 @@ static inline int mca_btl_ugni_post_bte (mca_btl_ugni_base_frag_t *frag, gni_pos
     gni_return_t status;
 
     /* Post descriptor */
-    if (howards_progress_var && (getenv("GENERATE_RDMA_IRQS") != NULL)) {
+    if (mca_btl_ugni_component.progress_thread_enabled) {
         init_gni_post_desc (frag, op_type, lcl_seg->base.seg_addr.lval, lcl_seg->memory_handle,
                             rem_seg->base.seg_addr.lval, rem_seg->memory_handle, lcl_seg->base.seg_len,
                             frag->endpoint->btl->rdma_local_irq_cq);

opal/mca/btl/ugni/btl_ugni_smsg.h

@@ -108,7 +108,7 @@ static inline int opal_mca_btl_ugni_smsg_send (mca_btl_ugni_base_frag_t *frag,
         /* increment the active send counter */
         opal_atomic_add_32(&frag->endpoint->btl->active_send_count,1);
 
-        if (howards_progress_var == 1 && (getenv("GENERATE_MDH_IRQS") != NULL)) {
+        if (mca_btl_ugni_component.progress_thread_enabled) {
             if (frag->base.des_flags & MCA_BTL_DES_FLAGS_SIGNAL) {
                 rc = mca_btl_ugni_frag_alloc(frag->endpoint,
                                              &frag->endpoint->btl->rdma_frags,