set socket options to improve oob performance

This commit was SVN r7934.

orte/mca/oob/tcp/oob_tcp.c

@@ -200,6 +200,10 @@ int mca_oob_tcp_component_open(void)
         mca_oob_tcp_param_register_str("include", NULL);
     mca_oob_tcp_component.tcp_exclude =
         mca_oob_tcp_param_register_str("exclude", NULL);
+    mca_oob_tcp_component.tcp_sndbuf =
+        mca_oob_tcp_param_register_int("sndbuf", 128*1024);
+    mca_oob_tcp_component.tcp_rcvbuf =
+        mca_oob_tcp_param_register_int("rcvbuf", 128*1024);
 
     /* initialize state */
     mca_oob_tcp_component.tcp_listen_sd = -1;
@@ -260,6 +264,9 @@ static void mca_oob_tcp_accept(void)
             return;
         }
 
+        /* setup socket options */
+        mca_oob_tcp_set_socket_options(sd);
+
         /* log the accept */
         if(mca_oob_tcp_component.tcp_debug) {
             opal_output(0, "[%lu,%lu,%lu] mca_oob_tcp_accept: %s:%d\n",
@@ -291,6 +298,11 @@ static int mca_oob_tcp_create_listen(void)
         opal_output(0,"mca_oob_tcp_component_init: socket() failed with errno=%d", ompi_socket_errno);
         return OMPI_ERROR;
     }
+
+    /* setup socket options */
+    mca_oob_tcp_set_socket_options(mca_oob_tcp_component.tcp_listen_sd);
+
+    /* bind address */
     memset(&inaddr, 0, sizeof(inaddr));
     inaddr.sin_family = AF_INET;
     inaddr.sin_addr.s_addr = INADDR_ANY;

orte/mca/oob/tcp/oob_tcp.h

@@ -299,6 +299,11 @@ void mca_oob_tcp_registry_callback(
      orte_gpr_notify_data_t* data,
      void* cbdata);
 
+/**
+ *  Setup socket options
+ */
+
+void mca_oob_tcp_set_socket_options(int sd);
 
 /**
  *  OOB TCP Component
@@ -316,6 +321,8 @@ struct mca_oob_tcp_component_t {
     opal_free_list_t   tcp_peer_free;        /**< free list of peers */
     int                tcp_peer_limit;       /**< max size of tcp peer cache */
     int                tcp_peer_retries;     /**< max number of retries before declaring peer gone */
+    int                tcp_sndbuf;           /**< socket send buffer size */
+    int                tcp_rcvbuf;           /**< socket recv buffer size */
     opal_free_list_t   tcp_msgs;             /**< free list of messages */
     opal_event_t       tcp_send_event;       /**< event structure for sends */
     opal_event_t       tcp_recv_event;       /**< event structure for recvs */

orte/mca/oob/tcp/oob_tcp_msg.c

@@ -180,8 +180,8 @@ int mca_oob_tcp_msg_complete(mca_oob_tcp_msg_t* msg, orte_process_name_t * peer)
                 msg->msg_ucnt, 
                 msg->msg_hdr.msg_tag, 
                 msg->msg_cbdata);
-            MCA_OOB_TCP_MSG_RETURN(msg);
             OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
+            MCA_OOB_TCP_MSG_RETURN(msg);
         }
         OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
 

orte/mca/oob/tcp/oob_tcp_peer.c

@@ -292,6 +292,9 @@ static int mca_oob_tcp_peer_start_connect(mca_oob_tcp_peer_t* peer)
         return OMPI_ERR_UNREACH;
     }
 
+    /* setup socket options */
+    mca_oob_tcp_set_socket_options(peer->peer_sd);
+
     /* setup event callbacks */
     mca_oob_tcp_peer_event_init(peer);
 
@@ -943,3 +946,32 @@ void mca_oob_tcp_peer_dequeue_msg(mca_oob_tcp_peer_t* peer, mca_oob_tcp_msg_t* m
 }
 
 
+/**
+ * Set socket buffering
+ */
+
+void mca_oob_tcp_set_socket_options(int sd)
+{
+    int optval;
+#if defined(TCP_NODELAY)
+    optval = 1;
+    if(setsockopt(sd, IPPROTO_TCP, TCP_NODELAY, (char *)&optval, sizeof(optval)) < 0) {
+        opal_output(0, "[%s:%d] setsockopt(TCP_NODELAY) failed with errno=%d", __FILE__, __LINE__, ompi_socket_errno);
+    }
+#endif
+#if defined(SO_SNDBUF)
+    if(mca_oob_tcp_component.tcp_sndbuf > 0 &&
+       setsockopt(sd, SOL_SOCKET, SO_SNDBUF, (char *)&mca_oob_tcp_component.tcp_sndbuf, sizeof(int)) < 0) {
+        opal_output(0, "[%s:%d] setsockopt(SO_SNDBUF) failed with errno %d", __FILE__, __LINE__, ompi_socket_errno);
+    }
+#endif
+#if defined(SO_RCVBUF)
+    if(mca_oob_tcp_component.tcp_rcvbuf > 0 &&
+       setsockopt(sd, SOL_SOCKET, SO_RCVBUF, (char *)&mca_oob_tcp_component.tcp_rcvbuf, sizeof(int)) < 0) {
+        opal_output(0, "[%s:%d] setsockopt(SO_RCVBUF) failed with errno %d", __FILE__, __LINE__, ompi_socket_errno);
+    }
+#endif
+}
+
+
+