defer sends for second+ fragments to improve overlap by the scheduler

This commit was SVN r2739.

src/mca/ptl/tcp/src/ptl_tcp.c

@@ -194,7 +194,7 @@ void mca_ptl_tcp_send_frag_return(struct mca_ptl_base_module_t* ptl, struct mca_
         }
         OMPI_THREAD_UNLOCK(&mca_ptl_tcp_component.tcp_lock);
         mca_ptl_tcp_send_frag_init_ack(frag, ptl, pending->frag_recv.frag_base.frag_peer, pending);
-        mca_ptl_tcp_peer_send(pending->frag_recv.frag_base.frag_peer, frag);
+        mca_ptl_tcp_peer_send(pending->frag_recv.frag_base.frag_peer, frag, 0);
         mca_ptl_tcp_recv_frag_return(ptl, pending);
     } else {
         OMPI_FREE_LIST_RETURN(&mca_ptl_tcp_component.tcp_send_frags, (ompi_list_item_t*)frag);
@@ -233,7 +233,7 @@ int mca_ptl_tcp_send(
      * before attempting to send the fragment 
      */
     sendreq->req_offset += size;
-    return mca_ptl_tcp_peer_send(ptl_peer, sendfrag);
+    return mca_ptl_tcp_peer_send(ptl_peer, sendfrag, offset);
 }
 
 
@@ -263,7 +263,7 @@ void mca_ptl_tcp_matched(
             OMPI_THREAD_UNLOCK(&mca_ptl_tcp_component.tcp_lock);
         } else {
             mca_ptl_tcp_send_frag_init_ack(ack, ptl, recv_frag->frag_recv.frag_base.frag_peer, recv_frag);
-            mca_ptl_tcp_peer_send(ack->frag_send.frag_base.frag_peer, ack);
+            mca_ptl_tcp_peer_send(ack->frag_send.frag_base.frag_peer, ack, 0);
         }
     }
 

src/mca/ptl/tcp/src/ptl_tcp_peer.c

@@ -151,7 +151,7 @@ static inline void mca_ptl_tcp_peer_event_init(mca_ptl_base_peer_t* ptl_peer, in
  * queue the fragment and start the connection as required.
  */
 
-int mca_ptl_tcp_peer_send(mca_ptl_base_peer_t* ptl_peer, mca_ptl_tcp_send_frag_t* frag)
+int mca_ptl_tcp_peer_send(mca_ptl_base_peer_t* ptl_peer, mca_ptl_tcp_send_frag_t* frag, int offset)
 {
     int rc = OMPI_SUCCESS;
     OMPI_THREAD_LOCK(&ptl_peer->peer_send_lock);
@@ -169,7 +169,7 @@ int mca_ptl_tcp_peer_send(mca_ptl_base_peer_t* ptl_peer, mca_ptl_tcp_send_frag_t
     case MCA_PTL_TCP_CONNECTED:
         if (NULL != ptl_peer->peer_send_frag) {
             ompi_list_append(&ptl_peer->peer_frags, (ompi_list_item_t*)frag);
-        } else {
+        } else if (offset == 0) {
             if(mca_ptl_tcp_send_frag_handler(frag, ptl_peer->peer_sd)) {
                 OMPI_THREAD_UNLOCK(&ptl_peer->peer_send_lock);
                 mca_ptl_tcp_send_frag_progress(frag);
@@ -178,6 +178,9 @@ int mca_ptl_tcp_peer_send(mca_ptl_base_peer_t* ptl_peer, mca_ptl_tcp_send_frag_t
                 ptl_peer->peer_send_frag = frag;
                 ompi_event_add(&ptl_peer->peer_send_event, 0);
             }
+        } else {
+            ptl_peer->peer_send_frag = frag;
+            ompi_event_add(&ptl_peer->peer_send_event, 0);
         }
         break;
     case MCA_PTL_TCP_SHUTDOWN:

src/mca/ptl/tcp/src/ptl_tcp_peer.h

@@ -61,7 +61,7 @@ typedef struct mca_ptl_base_peer_t mca_ptl_tcp_peer_t;
 
 void mca_ptl_tcp_set_socket_options(int sd);
 void mca_ptl_tcp_peer_close(mca_ptl_base_peer_t*);
-int  mca_ptl_tcp_peer_send(mca_ptl_base_peer_t*, mca_ptl_tcp_send_frag_t*);
+int  mca_ptl_tcp_peer_send(mca_ptl_base_peer_t*, mca_ptl_tcp_send_frag_t*, int);
 bool mca_ptl_tcp_peer_accept(mca_ptl_base_peer_t*, struct sockaddr_in*, int);
 void mca_ptl_tcp_peer_shutdown(mca_ptl_base_peer_t*);