Fix the TCP performance impact when not used

Based on an idea from Brian move the libevent trigger update to a later
stage instead of the generic add/del procs. So, we are doing the
increment/decrement when we register the recv handler for an endpoint,
so basically when we create and connect a socket to a peer. The benefit
is that as long as TCP is not used, there should be no impact on the
performance of other BTLs. The drawback is that the first TCP connection
will be slightly slower, but then once we have a peer connected over
TCP things go back to normal.

Signed-off-by: George Bosilca <bosilca@icl.utk.edu>

opal/mca/btl/tcp/btl_tcp.c

@@ -135,11 +135,6 @@ int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
         }
 
         peers[i] = tcp_endpoint;
-
-        /* we increase the count of MPI users of the event library
-           once per peer, so that we are used until we aren't
-           connected to a peer */
-        opal_progress_event_users_increment();
     }
 
     return OPAL_SUCCESS;
@@ -158,7 +153,6 @@ int mca_btl_tcp_del_procs(struct mca_btl_base_module_t* btl,
         mca_btl_tcp_endpoint_t* tcp_endpoint = endpoints[i];
         opal_list_remove_item(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint);
         OBJ_RELEASE(tcp_endpoint);
-        opal_progress_event_users_decrement();
     }
     OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex);
     return OPAL_SUCCESS;
@@ -492,7 +486,6 @@ int mca_btl_tcp_finalize(struct mca_btl_base_module_t* btl)
          item = opal_list_remove_first(&tcp_btl->tcp_endpoints)) {
         mca_btl_tcp_endpoint_t *endpoint = (mca_btl_tcp_endpoint_t*)item;
         OBJ_RELEASE(endpoint);
-        opal_progress_event_users_decrement();
     }
     free(tcp_btl);
     return OPAL_SUCCESS;

opal/mca/btl/tcp/btl_tcp_endpoint.c

@@ -464,6 +464,10 @@ static void *mca_btl_tcp_endpoint_complete_accept(int fd, int flags, void *conte
         mca_btl_tcp_endpoint_event_init(btl_endpoint);
         MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(recv) [endpoint_accept]");
         opal_event_add(&btl_endpoint->endpoint_recv_event, 0);
+        if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+            /* If no progress thread then raise the awarness of the default progress engine */
+            opal_progress_event_users_increment();
+        }
         mca_btl_tcp_endpoint_connected(btl_endpoint);
 
         MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "accepted");
@@ -513,6 +517,10 @@ void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t* btl_endpoint)
     btl_endpoint->endpoint_retries++;
     MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, false, "event_del(recv) [close]");
     opal_event_del(&btl_endpoint->endpoint_recv_event);
+    if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+        /* If no progress thread then lower the awarness of the default progress engine */
+        opal_progress_event_users_decrement();
+    }
     MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, false, "event_del(send) [close]");
     opal_event_del(&btl_endpoint->endpoint_send_event);
 
@@ -732,6 +740,10 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
             btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECT_ACK;
             MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(recv) [start_connect]");
             opal_event_add(&btl_endpoint->endpoint_recv_event, 0);
+            if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+                /* If no progress thread then raise the awarness of the default progress engine */
+                opal_progress_event_users_increment();
+            }
             return OPAL_SUCCESS;
         }
         /* We connected to the peer, but he close the socket before we got a chance to send our guid */
@@ -801,6 +813,10 @@ static void mca_btl_tcp_endpoint_complete_connect(mca_btl_base_endpoint_t* btl_e
     if(mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint) == OPAL_SUCCESS) {
         btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECT_ACK;
         opal_event_add(&btl_endpoint->endpoint_recv_event, 0);
+        if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+            /* If no progress thread then raise the awarness of the default progress engine */
+            opal_progress_event_users_increment();
+        }
         MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, false, "event_add(recv) [complete_connect]");
         return;
     }