From c9a257f1a083a7ef40b0268c891b6765f3c52caa Mon Sep 17 00:00:00 2001
From: Nathan Hjelm <hjelmn@lanl.gov>
Date: Thu, 24 Apr 2014 18:50:29 +0000
Subject: [PATCH] btl/ugni: always buffer sendi fragments

This commit will improve the message rate when using the sendi function
by not waiting for the send to get to the remote process.

cmr=v1.8.2:reviewer=ompi-rm1.8

This commit was SVN r31526.
---
 ompi/mca/btl/ugni/btl_ugni_send.c | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/ompi/mca/btl/ugni/btl_ugni_send.c b/ompi/mca/btl/ugni/btl_ugni_send.c
index b2f7519ebc..a59594a9d9 100644
--- a/ompi/mca/btl/ugni/btl_ugni_send.c
+++ b/ompi/mca/btl/ugni/btl_ugni_send.c
@@ -101,8 +101,15 @@ mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
             break;
         }
 
-        frag = mca_btl_ugni_prepare_src_send (btl, endpoint, convertor, order, header_size,
-                                              &packed_size, flags);
+        flags |= MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
+
+        if (0 == payload_size) {
+            frag = (mca_btl_ugni_base_frag_t *) mca_btl_ugni_prepare_src_send_nodata (btl, endpoint, order, header_size,
+                                                                                      flags);
+        } else {
+            frag = (mca_btl_ugni_base_frag_t *) mca_btl_ugni_prepare_src_send_buffered (btl, endpoint, convertor, order,
+                                                                                        header_size, &packed_size, flags);
+        }
         assert (packed_size == payload_size);
         if (OPAL_UNLIKELY(NULL == frag)) {
             break;
@@ -111,27 +118,12 @@ mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
         frag->hdr.send.lag = (tag << 24) | total_size;
         memcpy (frag->segments[0].base.seg_addr.pval, header, header_size);
 
-        frag->flags = MCA_BTL_UGNI_FRAG_IGNORE;
-
         rc = mca_btl_ugni_send_frag (endpoint, frag);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
             mca_btl_ugni_frag_return (frag);
             break;
         }
 
-        if (!(frag->flags & MCA_BTL_UGNI_FRAG_BUFFERED)) {
-            /* wait until the frag is received by the remote end */
-            while (!(frag->flags & MCA_BTL_UGNI_FRAG_SMSG_COMPLETE)) {
-                mca_btl_ugni_progress_local_smsg ((mca_btl_ugni_module_t *) btl);
-            }
-        }
-
-        frag->flags &= ~MCA_BTL_UGNI_FRAG_IGNORE;
-
-        if (frag->flags & MCA_BTL_UGNI_FRAG_SMSG_COMPLETE) {
-            mca_btl_ugni_frag_return (frag);
-        }
-
         return OMPI_SUCCESS;
     } while (0);