OSHMEM: misc ikrit/mxm enhancements

- fix mxm/tl selection logic
- do not require memory registration if mxm/ud was selected

fixed by Alex, reviewed by Miked

cmr=v1.7.5:reviewer=ompi-rm1.7

This commit was SVN r30802.

oshmem/mca/spml/ikrit/help-shmem-spml-ikrit.txt

@@ -70,4 +70,10 @@ Unable to post application send buffer
 Unable while waiting in send
 
   Error:      %s
-  
+  
+[mxm tls]
+ERROR: MXM shared memory transport can not be used
+bacause it is not fully compliant with OSHMEM spec
+
+MXM transport setting: %s
+

oshmem/mca/spml/ikrit/spml_ikrit.c

@@ -76,6 +76,9 @@ static int spml_ikrit_get_ep_address(spml_ikrit_mxm_ep_conn_info_t *ep_info,
 #else
 static inline mxm_mem_key_t *to_mxm_mkey(mca_spml_mkey_t *mkey) {
 
+    if (0 == mkey->len) {
+        return &mxm_empty_mem_key;
+    }
     return (mxm_mem_key_t *)mkey->u.data;
 }
 #endif
@@ -559,6 +562,11 @@ mca_spml_mkey_t *mca_spml_ikrit_register(void* addr,
 #if MXM_API < MXM_VERSION(2,0)
             mkeys[i].len = 0;
 #else
+            if (mca_spml_ikrit.ud_only) {
+                mkeys[i].len = 0;
+                break;
+            }
+
             err = mxm_mem_map(mca_spml_ikrit.mxm_context, &addr, &size, 0, 0, 0);
             if (MXM_OK != err) {
                 SPML_ERROR("Failed to register memory: %s", mxm_error_string(err));
@@ -652,7 +660,6 @@ static inline int get_ptl_id(int dst)
 
 int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys)
 {
-#if MXM_API < MXM_VERSION(2,0)
     int ptl;
     ptl = get_ptl_id(pe);
     if (ptl < 0)
@@ -661,6 +668,7 @@ int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys)
     if (ptl != MXM_PTL_RDMA)
         return OSHMEM_ERROR;
 
+#if MXM_API < MXM_VERSION(2,0)
     if (seg > 1)
         return OSHMEM_ERROR;
 
@@ -669,7 +677,12 @@ int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys)
     /* we are actually registering memory in 2.0 and later.
      * So can not really skip mkey exchange
      */
-    return OSHMEM_ERROR;
+    if (mca_spml_ikrit.ud_only) {
+        mkeys[ptl].len = 0;
+        mkeys[ptl].u.data = &mxm_empty_mem_key;
+    } else {
+        return OSHMEM_SUCCESS;
+    }
 #endif
 }
 

oshmem/mca/spml/ikrit/spml_ikrit.h

@@ -87,6 +87,9 @@ struct mca_spml_ikrit_t {
     int n_relays; /* number of procs/node serving as relays */
 
     char *mxm_tls;
+    int   ud_only;  /* only ud transport is used. In this case 
+                       it is possible to speedup mkey exchange 
+                       and not to register memheap */
 #if MXM_API >= MXM_VERSION(2,0)
     int unsync_conn_max;
 #endif

oshmem/mca/spml/ikrit/spml_ikrit_component.c

@@ -59,6 +59,53 @@ mca_spml_base_component_2_0_0_t mca_spml_ikrit_component = {
     
 };
 
+#if MXM_API >= MXM_VERSION(2,1)
+static int check_mxm_tls(char *var)
+{
+    char *str;
+
+    str = getenv(var);
+    if (NULL == str) {
+        return OSHMEM_SUCCESS;
+    }
+
+    if (NULL != strstr(str, "shm")) {
+        if (0 < asprintf(&str,
+                    "%s=%s",
+                    var, getenv(var)
+                    )) {
+            orte_show_help("help-shmem-spml-ikrit.txt", "mxm tls", true,
+                    str);
+            free(str);
+        }
+        return OSHMEM_ERROR;
+    }
+    if (NULL == strstr(str, "rc")) {
+        mca_spml_ikrit.ud_only = 1;
+    } else {
+        mca_spml_ikrit.ud_only = 0;
+    }
+    return OSHMEM_SUCCESS;
+}
+
+static int set_mxm_tls()
+{
+    char *tls;
+
+    tls = getenv("MXM_OSHMEM_TLS");
+    if (NULL != tls) {
+        return check_mxm_tls("MXM_OSHMEM_TLS");
+    }
+
+    tls = getenv("MXM_TLS");
+    if (NULL == tls) {
+        setenv("MXM_OSHMEM_TLS", mca_spml_ikrit.mxm_tls, 1);
+        return OSHMEM_SUCCESS;
+    }
+    return check_mxm_tls("MXM_TLS");
+}
+#endif
+
 static inline int mca_spml_ikrit_param_register_int(const char* param_name,
                                                     int default_value,
                                                     const char *help_msg)
@@ -165,11 +212,14 @@ static int mca_spml_ikrit_component_open(void)
                     (cur_ver >> MXM_MINOR_BIT) & 0xff);
     }
 
+    mca_spml_ikrit.ud_only = 0;
 #if MXM_API < MXM_VERSION(2,1)
     if ((MXM_OK != mxm_config_read_context_opts(&mca_spml_ikrit.mxm_ctx_opts)) ||
         (MXM_OK != mxm_config_read_ep_opts(&mca_spml_ikrit.mxm_ep_opts)))
 #else
-    setenv("MXM_OSHMEM_TLS", mca_spml_ikrit.mxm_tls, 0);
+    if (OSHMEM_SUCCESS != set_mxm_tls()) {
+        return OSHMEM_ERROR;
+    }
     if (MXM_OK != mxm_config_read_opts(&mca_spml_ikrit.mxm_ctx_opts,
                                        &mca_spml_ikrit.mxm_ep_opts,
                                        "OSHMEM", NULL, 0))
@@ -180,6 +230,7 @@ static int mca_spml_ikrit_component_open(void)
     }
 
 #if MXM_API < MXM_VERSION(2,0)
+    mca_spml_ikrit.ud_only = 1;
     mca_spml_ikrit.mxm_ctx_opts->ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA));
 #endif