Handle out of order completion/receives when marking completion...

this is a fix for #107... needs to go to the 1.1 branch.. 

This commit was SVN r10331.

ompi/mca/pml/ob1/pml_ob1_recvreq.c

@@ -589,6 +589,7 @@ void mca_pml_ob1_recv_request_schedule(mca_pml_ob1_recv_request_t* recvreq)
         ompi_proc_t* proc = recvreq->req_recv.req_base.req_proc;
         mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_pml; 
         mca_bml_base_btl_t* bml_btl; 
+        bool ack = false;
         do {
             size_t bytes_remaining = recvreq->req_recv.req_bytes_packed - recvreq->req_rdma_offset;
             while(bytes_remaining > 0 && recvreq->req_pipeline_depth < mca_pml_ob1.recv_pipeline_depth) {
@@ -630,6 +631,11 @@ void mca_pml_ob1_recv_request_schedule(mca_pml_ob1_recv_request_t* recvreq)
                     } else {
                         size = (size_t)(bml_btl->btl_weight * bytes_remaining);
                     }
+                    if(recvreq->req_rdma_idx == 0) { 
+                        ack = true; 
+                    } else { 
+                        ack = false;
+                    }
                      
                 } else {
                     char* base; 
@@ -712,7 +718,7 @@ void mca_pml_ob1_recv_request_schedule(mca_pml_ob1_recv_request_t* recvreq)
                 /* fill in rdma header */
                 hdr = (mca_pml_ob1_rdma_hdr_t*)ctl->des_src->seg_addr.pval;
                 hdr->hdr_common.hdr_type = MCA_PML_OB1_HDR_TYPE_PUT;
-                hdr->hdr_common.hdr_flags = 0;
+                hdr->hdr_common.hdr_flags = ack ? MCA_PML_OB1_HDR_TYPE_ACK : 0; 
                 hdr->hdr_req = recvreq->req_send;
                 hdr->hdr_des.pval = dst;
                 hdr->hdr_rdma_offset = recvreq->req_rdma_offset;

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -1032,12 +1032,12 @@ static void mca_pml_ob1_put_completion(
         }
         goto cleanup;
     }
-
+    
     /* check for request completion */
     if( OPAL_THREAD_ADD_SIZE_T(&sendreq->req_bytes_delivered, frag->rdma_length)
         >= sendreq->req_send.req_bytes_packed) {
         /* if we've got completion on rndv packet */
-        if (OPAL_THREAD_ADD32(&sendreq->req_state, 1) == 2) {
+        if (sendreq->req_state == 2) {
             MCA_PML_OB1_SEND_REQUEST_PML_COMPLETE(sendreq);
         }
     }
@@ -1077,6 +1077,10 @@ void mca_pml_ob1_send_request_put(
     
     bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl);  
     MCA_PML_OB1_RDMA_FRAG_ALLOC(frag, rc); 
+    if(hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_TYPE_ACK) { 
+        MCA_PML_OB1_SEND_REQUEST_ADVANCE_NO_SCHEDULE(sendreq);
+    }
+    
     if(NULL == frag) {
         /* TSW - FIX */
         ORTE_ERROR_LOG(rc);

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -318,7 +318,7 @@ do {
 do {                                                                              \
     /* has an acknowledgment been received */                                     \
     if(OPAL_THREAD_ADD32(&sendreq->req_state, 1) == 2) {                          \
-        if(sendreq->req_bytes_delivered == sendreq->req_send.req_bytes_packed) {  \
+        if(sendreq->req_bytes_delivered >= sendreq->req_send.req_bytes_packed) {  \
             MCA_PML_OB1_SEND_REQUEST_PML_COMPLETE(sendreq);                       \
         } else {                                                                  \
             /* additional data to schedule */                                     \
@@ -327,6 +327,19 @@ do {
     }                                                                             \
 } while (0)
 
+
+/*
+ * Advance a pending send request. Note that the initial descriptor must complete
+ * and the acknowledment received before the request can complete or be scheduled.
+ * However, these events may occur in either order.
+ */
+
+#define MCA_PML_OB1_SEND_REQUEST_ADVANCE_NO_SCHEDULE(sendreq)                     \
+do {                                                                              \
+    OPAL_THREAD_ADD32(&sendreq->req_state, 1);                                    \
+} while (0)
+
+
 /*
  * Release resources associated with a request
  */