diff --git a/oshmem/mca/spml/ikrit/spml_ikrit.c b/oshmem/mca/spml/ikrit/spml_ikrit.c index 2c14a83df0..565b835403 100644 --- a/oshmem/mca/spml/ikrit/spml_ikrit.c +++ b/oshmem/mca/spml/ikrit/spml_ikrit.c @@ -1055,6 +1055,7 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr, } #else if (mca_spml_ikrit.free_list_max - mca_spml_ikrit.n_active_puts <= SPML_IKRIT_PUT_LOW_WATER || + opal_list_get_size(&mca_spml_ikrit.active_peers) > mca_spml_ikrit.unsync_conn_max || (mca_spml_ikrit.mxm_peers[dst]->n_active_puts + 1) % SPML_IKRIT_PACKETS_PER_SYNC == 0) { put_req->mxm_req.flags = 0; need_progress = 1; diff --git a/oshmem/mca/spml/ikrit/spml_ikrit.h b/oshmem/mca/spml/ikrit/spml_ikrit.h index e59630bd10..fc572a44dd 100644 --- a/oshmem/mca/spml/ikrit/spml_ikrit.h +++ b/oshmem/mca/spml/ikrit/spml_ikrit.h @@ -94,6 +94,9 @@ struct mca_spml_ikrit_t { int n_relays; /* number of procs/node serving as relays */ char *mxm_tls; +#if MXM_API >= MXM_VERSION(2,0) + int unsync_conn_max; +#endif }; typedef struct mca_spml_ikrit_t mca_spml_ikrit_t; diff --git a/oshmem/mca/spml/ikrit/spml_ikrit_component.c b/oshmem/mca/spml/ikrit/spml_ikrit_component.c index 383e0ef3fd..233cef0780 100644 --- a/oshmem/mca/spml/ikrit/spml_ikrit_component.c +++ b/oshmem/mca/spml/ikrit/spml_ikrit_component.c @@ -125,6 +125,13 @@ static int mca_spml_ikrit_component_register(void) 0, #endif "[integer] Minimal allowed job's NP to activate ikrit"); +#if MXM_API >= MXM_VERSION(2,0) + mca_spml_ikrit.unsync_conn_max = + mca_spml_ikrit_param_register_int("unsync_conn_max", + 8, + "[integer] Max number of connections that do not require notification of PUT operation remote completion. Increasing this number improves efficiency of p2p communication but increases overhead of shmem_fence/shmem_quiet/shmem_barrier"); +#endif + if (oshmem_num_procs() < np) { SPML_VERBOSE(1, "Not enough ranks (%d<%d), disqualifying spml/ikrit",