From bd5650d68021dec40131a5bb10783bb1f3a9c7c6 Mon Sep 17 00:00:00 2001 From: George Bosilca Date: Fri, 23 Jun 2017 11:15:45 +0200 Subject: [PATCH] Fix the TCP performance impact when not used Based on an idea from Brian move the libevent trigger update to a later stage instead of the generic add/del procs. So, we are doing the increment/decrement when we register the recv handler for an endpoint, so basically when we create and connect a socket to a peer. The benefit is that as long as TCP is not used, there should be no impact on the performance of other BTLs. The drawback is that the first TCP connection will be slightly slower, but then once we have a peer connected over TCP things go back to normal. Signed-off-by: George Bosilca --- opal/mca/btl/tcp/btl_tcp.c | 7 ------- opal/mca/btl/tcp/btl_tcp_endpoint.c | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/opal/mca/btl/tcp/btl_tcp.c b/opal/mca/btl/tcp/btl_tcp.c index ac6289cf1f..40f99c6742 100644 --- a/opal/mca/btl/tcp/btl_tcp.c +++ b/opal/mca/btl/tcp/btl_tcp.c @@ -135,11 +135,6 @@ int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl, } peers[i] = tcp_endpoint; - - /* we increase the count of MPI users of the event library - once per peer, so that we are used until we aren't - connected to a peer */ - opal_progress_event_users_increment(); } return OPAL_SUCCESS; @@ -158,7 +153,6 @@ int mca_btl_tcp_del_procs(struct mca_btl_base_module_t* btl, mca_btl_tcp_endpoint_t* tcp_endpoint = endpoints[i]; opal_list_remove_item(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint); OBJ_RELEASE(tcp_endpoint); - opal_progress_event_users_decrement(); } OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex); return OPAL_SUCCESS; @@ -492,7 +486,6 @@ int mca_btl_tcp_finalize(struct mca_btl_base_module_t* btl) item = opal_list_remove_first(&tcp_btl->tcp_endpoints)) { mca_btl_tcp_endpoint_t *endpoint = (mca_btl_tcp_endpoint_t*)item; OBJ_RELEASE(endpoint); - opal_progress_event_users_decrement(); } free(tcp_btl); return OPAL_SUCCESS; diff --git a/opal/mca/btl/tcp/btl_tcp_endpoint.c b/opal/mca/btl/tcp/btl_tcp_endpoint.c index 9cd97e34b2..0001ff49f3 100644 --- a/opal/mca/btl/tcp/btl_tcp_endpoint.c +++ b/opal/mca/btl/tcp/btl_tcp_endpoint.c @@ -464,6 +464,10 @@ static void *mca_btl_tcp_endpoint_complete_accept(int fd, int flags, void *conte mca_btl_tcp_endpoint_event_init(btl_endpoint); MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(recv) [endpoint_accept]"); opal_event_add(&btl_endpoint->endpoint_recv_event, 0); + if( mca_btl_tcp_event_base == opal_sync_event_base ) { + /* If no progress thread then raise the awarness of the default progress engine */ + opal_progress_event_users_increment(); + } mca_btl_tcp_endpoint_connected(btl_endpoint); MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "accepted"); @@ -513,6 +517,10 @@ void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t* btl_endpoint) btl_endpoint->endpoint_retries++; MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, false, "event_del(recv) [close]"); opal_event_del(&btl_endpoint->endpoint_recv_event); + if( mca_btl_tcp_event_base == opal_sync_event_base ) { + /* If no progress thread then lower the awarness of the default progress engine */ + opal_progress_event_users_decrement(); + } MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, false, "event_del(send) [close]"); opal_event_del(&btl_endpoint->endpoint_send_event); @@ -732,6 +740,10 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECT_ACK; MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(recv) [start_connect]"); opal_event_add(&btl_endpoint->endpoint_recv_event, 0); + if( mca_btl_tcp_event_base == opal_sync_event_base ) { + /* If no progress thread then raise the awarness of the default progress engine */ + opal_progress_event_users_increment(); + } return OPAL_SUCCESS; } /* We connected to the peer, but he close the socket before we got a chance to send our guid */ @@ -801,6 +813,10 @@ static void mca_btl_tcp_endpoint_complete_connect(mca_btl_base_endpoint_t* btl_e if(mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint) == OPAL_SUCCESS) { btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECT_ACK; opal_event_add(&btl_endpoint->endpoint_recv_event, 0); + if( mca_btl_tcp_event_base == opal_sync_event_base ) { + /* If no progress thread then raise the awarness of the default progress engine */ + opal_progress_event_users_increment(); + } MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, false, "event_add(recv) [complete_connect]"); return; }