diff --git a/oshmem/mca/spml/ikrit/help-shmem-spml-ikrit.txt b/oshmem/mca/spml/ikrit/help-shmem-spml-ikrit.txt index edffaa846c..19019f6082 100644 --- a/oshmem/mca/spml/ikrit/help-shmem-spml-ikrit.txt +++ b/oshmem/mca/spml/ikrit/help-shmem-spml-ikrit.txt @@ -70,4 +70,10 @@ Unable to post application send buffer Unable while waiting in send Error: %s - \ No newline at end of file + +[mxm tls] +ERROR: MXM shared memory transport can not be used +bacause it is not fully compliant with OSHMEM spec + +MXM transport setting: %s + diff --git a/oshmem/mca/spml/ikrit/spml_ikrit.c b/oshmem/mca/spml/ikrit/spml_ikrit.c index 35a65093c9..65174732f1 100644 --- a/oshmem/mca/spml/ikrit/spml_ikrit.c +++ b/oshmem/mca/spml/ikrit/spml_ikrit.c @@ -76,6 +76,9 @@ static int spml_ikrit_get_ep_address(spml_ikrit_mxm_ep_conn_info_t *ep_info, #else static inline mxm_mem_key_t *to_mxm_mkey(mca_spml_mkey_t *mkey) { + if (0 == mkey->len) { + return &mxm_empty_mem_key; + } return (mxm_mem_key_t *)mkey->u.data; } #endif @@ -559,6 +562,11 @@ mca_spml_mkey_t *mca_spml_ikrit_register(void* addr, #if MXM_API < MXM_VERSION(2,0) mkeys[i].len = 0; #else + if (mca_spml_ikrit.ud_only) { + mkeys[i].len = 0; + break; + } + err = mxm_mem_map(mca_spml_ikrit.mxm_context, &addr, &size, 0, 0, 0); if (MXM_OK != err) { SPML_ERROR("Failed to register memory: %s", mxm_error_string(err)); @@ -652,7 +660,6 @@ static inline int get_ptl_id(int dst) int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys) { -#if MXM_API < MXM_VERSION(2,0) int ptl; ptl = get_ptl_id(pe); if (ptl < 0) @@ -661,6 +668,7 @@ int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys) if (ptl != MXM_PTL_RDMA) return OSHMEM_ERROR; +#if MXM_API < MXM_VERSION(2,0) if (seg > 1) return OSHMEM_ERROR; @@ -669,7 +677,12 @@ int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys) /* we are actually registering memory in 2.0 and later. * So can not really skip mkey exchange */ - return OSHMEM_ERROR; + if (mca_spml_ikrit.ud_only) { + mkeys[ptl].len = 0; + mkeys[ptl].u.data = &mxm_empty_mem_key; + } else { + return OSHMEM_SUCCESS; + } #endif } diff --git a/oshmem/mca/spml/ikrit/spml_ikrit.h b/oshmem/mca/spml/ikrit/spml_ikrit.h index 13dd2dbb15..7d11e9e2f2 100644 --- a/oshmem/mca/spml/ikrit/spml_ikrit.h +++ b/oshmem/mca/spml/ikrit/spml_ikrit.h @@ -87,6 +87,9 @@ struct mca_spml_ikrit_t { int n_relays; /* number of procs/node serving as relays */ char *mxm_tls; + int ud_only; /* only ud transport is used. In this case + it is possible to speedup mkey exchange + and not to register memheap */ #if MXM_API >= MXM_VERSION(2,0) int unsync_conn_max; #endif diff --git a/oshmem/mca/spml/ikrit/spml_ikrit_component.c b/oshmem/mca/spml/ikrit/spml_ikrit_component.c index 84ad6b6375..94fa697128 100644 --- a/oshmem/mca/spml/ikrit/spml_ikrit_component.c +++ b/oshmem/mca/spml/ikrit/spml_ikrit_component.c @@ -59,6 +59,53 @@ mca_spml_base_component_2_0_0_t mca_spml_ikrit_component = { }; +#if MXM_API >= MXM_VERSION(2,1) +static int check_mxm_tls(char *var) +{ + char *str; + + str = getenv(var); + if (NULL == str) { + return OSHMEM_SUCCESS; + } + + if (NULL != strstr(str, "shm")) { + if (0 < asprintf(&str, + "%s=%s", + var, getenv(var) + )) { + orte_show_help("help-shmem-spml-ikrit.txt", "mxm tls", true, + str); + free(str); + } + return OSHMEM_ERROR; + } + if (NULL == strstr(str, "rc")) { + mca_spml_ikrit.ud_only = 1; + } else { + mca_spml_ikrit.ud_only = 0; + } + return OSHMEM_SUCCESS; +} + +static int set_mxm_tls() +{ + char *tls; + + tls = getenv("MXM_OSHMEM_TLS"); + if (NULL != tls) { + return check_mxm_tls("MXM_OSHMEM_TLS"); + } + + tls = getenv("MXM_TLS"); + if (NULL == tls) { + setenv("MXM_OSHMEM_TLS", mca_spml_ikrit.mxm_tls, 1); + return OSHMEM_SUCCESS; + } + return check_mxm_tls("MXM_TLS"); +} +#endif + static inline int mca_spml_ikrit_param_register_int(const char* param_name, int default_value, const char *help_msg) @@ -165,11 +212,14 @@ static int mca_spml_ikrit_component_open(void) (cur_ver >> MXM_MINOR_BIT) & 0xff); } + mca_spml_ikrit.ud_only = 0; #if MXM_API < MXM_VERSION(2,1) if ((MXM_OK != mxm_config_read_context_opts(&mca_spml_ikrit.mxm_ctx_opts)) || (MXM_OK != mxm_config_read_ep_opts(&mca_spml_ikrit.mxm_ep_opts))) #else - setenv("MXM_OSHMEM_TLS", mca_spml_ikrit.mxm_tls, 0); + if (OSHMEM_SUCCESS != set_mxm_tls()) { + return OSHMEM_ERROR; + } if (MXM_OK != mxm_config_read_opts(&mca_spml_ikrit.mxm_ctx_opts, &mca_spml_ikrit.mxm_ep_opts, "OSHMEM", NULL, 0)) @@ -180,6 +230,7 @@ static int mca_spml_ikrit_component_open(void) } #if MXM_API < MXM_VERSION(2,0) + mca_spml_ikrit.ud_only = 1; mca_spml_ikrit.mxm_ctx_opts->ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA)); #endif