diff --git a/orte/mca/oob/tcp/oob_tcp_component.c b/orte/mca/oob/tcp/oob_tcp_component.c index 7e8e01ae24..e0f070c9bf 100644 --- a/orte/mca/oob/tcp/oob_tcp_component.c +++ b/orte/mca/oob/tcp/oob_tcp_component.c @@ -433,6 +433,23 @@ static int tcp_component_register(void) OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_oob_tcp_component.keepalive_probes); + + mca_oob_tcp_component.retry_delay = 0; + (void)mca_base_component_var_register(component, "retry_delay", + "Time (in sec) to wait before trying to connect to peer again", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_oob_tcp_component.retry_delay); + + mca_oob_tcp_component.max_recon_attempts = 10; + (void)mca_base_component_var_register(component, "max_recon_attempts", + "Max number of times to attempt connection before giving up (-1 -> never give up)", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_oob_tcp_component.max_retries); + return ORTE_SUCCESS; } @@ -1204,6 +1221,7 @@ static void peer_cons(mca_oob_tcp_peer_t *peer) OBJ_CONSTRUCT(&peer->addrs, opal_list_t); peer->active_addr = NULL; peer->state = MCA_OOB_TCP_UNCONNECTED; + peer->num_retries = 0; OBJ_CONSTRUCT(&peer->send_queue, opal_list_t); peer->send_msg = NULL; peer->recv_msg = NULL; diff --git a/orte/mca/oob/tcp/oob_tcp_component.h b/orte/mca/oob/tcp/oob_tcp_component.h index cd48d2e639..120dac3d61 100644 --- a/orte/mca/oob/tcp/oob_tcp_component.h +++ b/orte/mca/oob/tcp/oob_tcp_component.h @@ -80,6 +80,8 @@ typedef struct { int keepalive_probes; /**< number of keepalives that can be missed before declaring error */ int keepalive_time; /**< idle time in seconds before starting to send keepalives */ int keepalive_intvl; /**< time between keepalives, in seconds */ + int retry_delay; /**< time to wait before retrying connection */ + int max_recon_attempts; /**< maximum number of times to attempt connect before giving up (-1 for never) */ } mca_oob_tcp_component_t; ORTE_MODULE_DECLSPEC extern mca_oob_tcp_component_t mca_oob_tcp_component; diff --git a/orte/mca/oob/tcp/oob_tcp_connection.c b/orte/mca/oob/tcp/oob_tcp_connection.c index 4a5bca883c..fd0b53bc1d 100644 --- a/orte/mca/oob/tcp/oob_tcp_connection.c +++ b/orte/mca/oob/tcp/oob_tcp_connection.c @@ -258,11 +258,32 @@ void mca_oob_tcp_peer_try_connect(int fd, short args, void *cbdata) /* connection succeeded */ addr->retries = 0; connected = true; + peer->num_retries = 0; break; } } if (!connected) { + /* it could be that the intended recipient just hasn't + * started yet. if requested, wait awhile and try again + * unless/until we hit the maximum number of retries */ + if (0 < mca_oob_tcp_component.retry_delay) { + if (mca_oob_tcp_component.max_recon_attempts < 0 || + peer->num_retries < mca_oob_tcp_component.max_recon_attempts) { + struct timeval tv; + /* reset the addr states */ + OPAL_LIST_FOREACH(addr, &peer->addrs, mca_oob_tcp_addr_t) { + addr->state = MCA_OOB_TCP_UNCONNECTED; + addr->retries = 0; + } + /* give it awhile and try again */ + tv.tv_sec = mca_oob_tcp_component.retry_delay; + tv.tv_usec = 0; + ++peer->num_retries; + ORTE_RETRY_TCP_CONN_STATE(peer, mca_oob_tcp_peer_try_connect, &tv); + goto cleanup; + } + } /* no address succeeded, so we cannot reach this peer */ peer->state = MCA_OOB_TCP_FAILED; host = orte_get_proc_hostname(&(peer->name)); diff --git a/orte/mca/oob/tcp/oob_tcp_peer.h b/orte/mca/oob/tcp/oob_tcp_peer.h index a0386a9b77..c4c1397755 100644 --- a/orte/mca/oob/tcp/oob_tcp_peer.h +++ b/orte/mca/oob/tcp/oob_tcp_peer.h @@ -48,6 +48,7 @@ typedef struct { opal_list_t addrs; mca_oob_tcp_addr_t *active_addr; mca_oob_tcp_state_t state; + int num_retries; opal_event_t send_event; /**< registration with event thread for send events */ bool send_ev_active; opal_event_t recv_event; /**< registration with event thread for recv events */