Sender can start scheduling send fragments immediately after receiving ACK. No

need to wait for RNDV completion.

This commit was SVN r12965.

ompi/mca/pml/ob1/pml_ob1_recvfrag.c

@@ -129,7 +129,14 @@ void mca_pml_ob1_recv_frag_callback(
             sendreq = (mca_pml_ob1_send_request_t*)hdr->hdr_ack.hdr_src_req.pval;
             sendreq->req_recv = hdr->hdr_ack.hdr_dst_req;
             sendreq->req_rdma_offset = (size_t)hdr->hdr_ack.hdr_rdma_offset;
-            MCA_PML_OB1_SEND_REQUEST_ADVANCE(sendreq);
+            if(OPAL_THREAD_ADD32(&sendreq->req_state, 1) == 2 &&
+                    sendreq->req_bytes_delivered >=
+                    sendreq->req_send.req_bytes_packed) {
+                MCA_PML_OB1_SEND_REQUEST_PML_COMPLETE(sendreq);
+            } else {
+                mca_pml_ob1_send_request_schedule(sendreq);
+            }
+                
             break;
         }
     case MCA_PML_OB1_HDR_TYPE_FRAG:

ompi/mca/pml/ob1/pml_ob1_recvreq.c

@@ -235,10 +235,10 @@ static int mca_pml_ob1_recv_request_ack(
 
     bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml; 
 
+    /* by default copy */
+    recvreq->req_rdma_offset = hdr->hdr_msg_length;
     if(hdr->hdr_msg_length > bytes_received) {
         
-        /* by default copy */
-        recvreq->req_rdma_offset = hdr->hdr_msg_length;
 
         /*
          * lookup request buffer to determine if memory is already

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -248,8 +248,11 @@ static void mca_pml_ob1_rndv_completion(
     mca_bml_base_free(bml_btl, descriptor); 
 
     /* advance the request */
-    MCA_PML_OB1_SEND_REQUEST_ADVANCE(sendreq);
+    if(OPAL_THREAD_ADD32(&sendreq->req_state, 1) == 2 &&
-
+            sendreq->req_bytes_delivered >=
+            sendreq->req_send.req_bytes_packed) {
+        MCA_PML_OB1_SEND_REQUEST_PML_COMPLETE(sendreq);
+    }
     /* check for pending requests */
     MCA_PML_OB1_PROGRESS_PENDING(bml_btl);
 }
@@ -871,6 +874,10 @@ int mca_pml_ob1_send_request_schedule_exclusive(
             sendreq->req_send_offset;
         size_t prev_bytes_remaining = 0, num_fail = 0;
 
+        if(bytes_remaining == 0) {
+            OPAL_THREAD_ADD32(&sendreq->req_lock, -sendreq->req_lock);
+            return OMPI_SUCCESS;
+        }
         while((int32_t)bytes_remaining > 0 &&
                 (sendreq->req_pipeline_depth < mca_pml_ob1.send_pipeline_depth
                  ||
@@ -1119,7 +1126,7 @@ void mca_pml_ob1_send_request_put(
     size_t i, size = 0;
 
     if(hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_TYPE_ACK) { 
-        MCA_PML_OB1_SEND_REQUEST_ADVANCE_NO_SCHEDULE(sendreq);
+        OPAL_THREAD_ADD32(&sendreq->req_state, 1);
     }
 
     MCA_PML_OB1_RDMA_FRAG_ALLOC(frag, rc); 

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -211,38 +211,6 @@ static void inline mca_pml_ob1_send_request_schedule(
         mca_pml_ob1_send_request_schedule_exclusive(sendreq);
 }
 
-/*
- * Advance a pending send request. Note that the initial descriptor must complete
- * and the acknowledment received before the request can complete or be scheduled.
- * However, these events may occur in either order.
- */
-
-#define MCA_PML_OB1_SEND_REQUEST_ADVANCE(sendreq)                                 \
-do {                                                                              \
-    /* has an acknowledgment been received */                                     \
-    if(OPAL_THREAD_ADD32(&sendreq->req_state, 1) == 2) {                          \
-        if(sendreq->req_bytes_delivered >= sendreq->req_send.req_bytes_packed) {  \
-            MCA_PML_OB1_SEND_REQUEST_PML_COMPLETE(sendreq);                       \
-        } else {                                                                  \
-            /* additional data to schedule */                                     \
-            mca_pml_ob1_send_request_schedule(sendreq);                           \
-        }                                                                         \
-    }                                                                             \
-} while (0)
-
-
-/*
- * Advance a pending send request. Note that the initial descriptor must complete
- * and the acknowledment received before the request can complete or be scheduled.
- * However, these events may occur in either order.
- */
-
-#define MCA_PML_OB1_SEND_REQUEST_ADVANCE_NO_SCHEDULE(sendreq)                     \
-do {                                                                              \
-    OPAL_THREAD_ADD32(&sendreq->req_state, 1);                                    \
-} while (0)
-
-
 /*
  * Release resources associated with a request
  */