while the BTL semantics are sorted out, wait for completion of all rdma events before starting the ack

This commit was SVN r15292.

ompi/mca/osc/rdma/osc_rdma.h

@@ -177,6 +177,7 @@ struct ompi_osc_rdma_module_t {
 
     /* RDMA data */
     bool m_use_rdma;
+    bool m_rdma_wait_completion;
     ompi_osc_rdma_setup_info_t *m_setup_info;
     ompi_osc_rdma_peer_info_t *m_peer_info;
     int32_t m_rdma_num_pending;

ompi/mca/osc/rdma/osc_rdma_component.c

@@ -150,6 +150,13 @@ component_open(void)
                            "Info key of same name overrides this value.",
                            false, false, 0, NULL);
 
+     mca_base_param_reg_int(&mca_osc_rdma_component.super.osc_version,
+                            "rdma_completion_wait",
+                            "Wait for all completion of rdma events before "
+                            "sending acknowledgment.  Info key of same name "
+                            "overrides this value.",
+                            false, false, 1, NULL);
+
     mca_base_param_reg_int(&mca_osc_rdma_component.super.osc_version,
                            "no_locks",
                            "Enable optimizations available only if MPI_LOCK is "
@@ -344,6 +351,7 @@ ompi_osc_rdma_component_select(ompi_win_t *win,
 
     /* allocate space for rdma information */
     module->m_use_rdma = check_config_value_bool("use_rdma", info);
+    module->m_rdma_wait_completion = check_config_value_bool("rdma_completion_wait", info);
     module->m_setup_info = NULL;
     module->m_peer_info = NULL;
 

ompi/mca/osc/rdma/osc_rdma_sync.c

@@ -129,11 +129,13 @@ ompi_osc_rdma_module_fence(int assert, ompi_win_t *win)
         }
 
         if (module->m_use_rdma) {
-            OPAL_THREAD_LOCK(&module->m_lock);
-            while (module->m_rdma_num_pending != 0) {
-                opal_condition_wait(&module->m_cond, &module->m_lock);
+            if (module->m_rdma_wait_completion) {
+                OPAL_THREAD_LOCK(&module->m_lock);
+                while (module->m_rdma_num_pending != 0) {
+                    opal_condition_wait(&module->m_cond, &module->m_lock);
+                }
+                OPAL_THREAD_UNLOCK(&module->m_lock);
             }
-            OPAL_THREAD_UNLOCK(&module->m_lock);
 
             for (i = 0 ; i < ompi_comm_size(module->m_comm) ; ++i) {
                 int j;
@@ -289,11 +291,13 @@ ompi_osc_rdma_module_complete(ompi_win_t *win)
     for (i = 0 ; i < ompi_group_size(module->m_sc_group) ; ++i) {
         int comm_rank = module->m_sc_remote_ranks[i];
         if (module->m_use_rdma) {
-            OPAL_THREAD_LOCK(&module->m_lock);
-            while (module->m_rdma_num_pending != 0) {
-                opal_condition_wait(&module->m_cond, &module->m_lock);
+            if (module->m_rdma_wait_completion) {
+                OPAL_THREAD_LOCK(&module->m_lock);
+                while (module->m_rdma_num_pending != 0) {
+                    opal_condition_wait(&module->m_cond, &module->m_lock);
+                }
+                OPAL_THREAD_UNLOCK(&module->m_lock);
             }
-            OPAL_THREAD_UNLOCK(&module->m_lock);
 
             for (j = 0 ; j < module->m_peer_info[comm_rank].peer_num_btls ; ++j) {
                 if (module->m_peer_info[comm_rank].peer_btls[j].num_sent > 0) {