From 2f9c5b46cf1b4834387297a849e5b600eb3f7987 Mon Sep 17 00:00:00 2001
From: Gleb Natapov <gleb@voltaire.com>
Date: Sun, 9 Dec 2007 14:14:11 +0000
Subject: [PATCH] Return OMPI_ERR_RESOURCE_BUSY from openib_btl_send() if
 fragment is not on wire.

This commit was SVN r16906.
---
 ompi/mca/btl/openib/btl_openib.c          |  4 ++--
 ompi/mca/btl/openib/btl_openib_endpoint.c | 26 ++++++++++-------------
 ompi/mca/btl/openib/btl_openib_endpoint.h |  4 ++--
 3 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/ompi/mca/btl/openib/btl_openib.c b/ompi/mca/btl/openib/btl_openib.c
index a4bbb2955e..51b3165f2c 100644
--- a/ompi/mca/btl/openib/btl_openib.c
+++ b/ompi/mca/btl/openib/btl_openib.c
@@ -1109,7 +1109,7 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
         OPAL_THREAD_LOCK(&ep->endpoint_lock);
         rc = check_endpoint_state(ep, descriptor, &ep->pending_put_frags);
         OPAL_THREAD_UNLOCK(&ep->endpoint_lock);
-        if(OMPI_ERR_TEMP_OUT_OF_RESOURCE == rc)
+        if(OMPI_ERR_RESOURCE_BUSY == rc)
             return OMPI_SUCCESS;
         if(OMPI_SUCCESS != rc)
             return rc;
@@ -1178,7 +1178,7 @@ int mca_btl_openib_get(mca_btl_base_module_t* btl,
         OPAL_THREAD_LOCK(&ep->endpoint_lock);
         rc = check_endpoint_state(ep, descriptor, &ep->pending_get_frags);
         OPAL_THREAD_UNLOCK(&ep->endpoint_lock);
-        if(OMPI_ERR_TEMP_OUT_OF_RESOURCE == rc)
+        if(OMPI_ERR_RESOURCE_BUSY == rc)
             return OMPI_SUCCESS;
         if(OMPI_SUCCESS != rc)
             return rc;
diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.c b/ompi/mca/btl/openib/btl_openib_endpoint.c
index 70a8576614..ba867cc20d 100644
--- a/ompi/mca/btl/openib/btl_openib_endpoint.c
+++ b/ompi/mca/btl/openib/btl_openib_endpoint.c
@@ -190,7 +190,7 @@ int mca_btl_openib_endpoint_post_send(mca_btl_openib_endpoint_t *endpoint,
     qp = des->order;
 
     if(acruire_wqe(endpoint, frag) != OMPI_SUCCESS)
-        return OMPI_ERR_OUT_OF_RESOURCE;
+        return OMPI_ERR_RESOURCE_BUSY;
 
     eager_limit = mca_btl_openib_component.eager_limit +
         sizeof(mca_btl_openib_header_coalesced_t) +
@@ -204,7 +204,7 @@ int mca_btl_openib_endpoint_post_send(mca_btl_openib_endpoint_t *endpoint,
 
     if(!do_rdma && acquire_send_credit(endpoint, frag) != OMPI_SUCCESS) {
         qp_put_wqe(endpoint, qp);
-        return OMPI_ERR_OUT_OF_RESOURCE;
+        return OMPI_ERR_RESOURCE_BUSY;
     }
 
     GET_CREDITS(endpoint->eager_rdma_local.credits, hdr->credits);
@@ -589,13 +589,8 @@ int mca_btl_openib_endpoint_send(mca_btl_base_endpoint_t* ep,
     rc = check_endpoint_state(ep, &to_base_frag(frag)->base,
             &ep->pending_lazy_frags);
 
-    if(OPAL_LIKELY(rc == OMPI_SUCCESS)) {
+    if(OPAL_LIKELY(rc == OMPI_SUCCESS))
         rc = mca_btl_openib_endpoint_post_send(ep, frag);
-        if(OMPI_ERR_OUT_OF_RESOURCE == rc)
-            rc = OMPI_SUCCESS;
-    } else if(OMPI_ERR_TEMP_OUT_OF_RESOURCE == rc) {
-        rc = OMPI_SUCCESS;
-    }
     OPAL_THREAD_UNLOCK(&ep->endpoint_lock);
 
     return rc;
@@ -774,12 +769,13 @@ static int mca_btl_openib_endpoint_send_eager_rdma(
                    rdma_hdr->rdma_start.ival
                    ));
     }
-    if (mca_btl_openib_endpoint_send(endpoint, frag) != OMPI_SUCCESS) {
-        MCA_BTL_IB_FRAG_RETURN(frag);
-        BTL_ERROR(("Error sending RDMA buffer", strerror(errno)));
-        return -1;
-    }
-    return 0;
+    rc = mca_btl_openib_endpoint_send(endpoint, frag); 
+    if (OMPI_SUCCESS == rc ||OMPI_ERR_RESOURCE_BUSY == rc)
+        return OMPI_SUCCESS;
+    
+    MCA_BTL_IB_FRAG_RETURN(frag);
+    BTL_ERROR(("Error sending RDMA buffer", strerror(errno)));
+    return rc;
 }
 
 /* Setup eager RDMA buffers and notify the remote endpoint*/
@@ -854,7 +850,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
     opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, (void*)1,
             buf);
 
-    if(mca_btl_openib_endpoint_send_eager_rdma(endpoint) == 0) {
+    if(mca_btl_openib_endpoint_send_eager_rdma(endpoint) == OMPI_SUCCESS) {
         /* This can never fail because max number of entries allocated
          * at init time */
         OBJ_RETAIN(endpoint);
diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.h b/ompi/mca/btl/openib/btl_openib_endpoint.h
index 961c1ccaa2..3565a79ecc 100644
--- a/ompi/mca/btl/openib/btl_openib_endpoint.h
+++ b/ompi/mca/btl/openib/btl_openib_endpoint.h
@@ -352,13 +352,13 @@ try_send:
 static inline int check_endpoint_state(mca_btl_openib_endpoint_t *ep,
         mca_btl_base_descriptor_t *des, opal_list_t *pending_list)
 {
-    int rc = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+    int rc = ORTE_ERR_RESOURCE_BUSY;
 
     switch(ep->endpoint_state) {
         case MCA_BTL_IB_CLOSED:
             rc = ompi_btl_openib_connect.bcf_start_connect(ep);
             if(rc == OMPI_SUCCESS)
-                rc = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+                rc = ORTE_ERR_RESOURCE_BUSY;
             /*
              * As long as we expect a message from the peer (in order
              * to setup the connection) let the event engine pool the