OSHMEM: sshmem verbs: allocate memory at fixed address

Use experimental verbs to allocate memory at fixed base
virtual address.

verbs will disqualify itself if shared_mr is disabled
or not supported and it is impossible to allocate memory
starting at fixed base virtual address.

verbs contig pages allocator did not guarantee fixed va, now it does.

oshmem/mca/sshmem/verbs/configure.m4

@@ -80,6 +80,13 @@ AC_DEFUN([MCA_oshmem_sshmem_verbs_CONFIG],[
                     )],
                 [],
                 [#include <infiniband/verbs_exp.h>])
+
+              AC_CHECK_MEMBER([struct ibv_exp_reg_mr_in.create_flags],
+                [AC_DEFINE_UNQUOTED(MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS, 1,
+                    [create_flags field is part of ibv_exp_reg_mr_in]
+                    )],
+                [],
+                [#include <infiniband/verbs_exp.h>])
          ])
 
     AS_IF([test "$enable_verbs_sshmem" = "yes" -a "$oshmem_verbs_sm_build_verbs" = "0"],

oshmem/mca/sshmem/verbs/sshmem_verbs_component.c

@@ -190,6 +190,7 @@ verbs_runtime_query(mca_base_module_t **module,
             if (NULL == ib_mr) {
                 if (mca_sshmem_verbs_component.has_shared_mr == 1)
                     rc = OSHMEM_ERR_OUT_OF_RESOURCE;
+
                 mca_sshmem_verbs_component.has_shared_mr = 0;
             } else {
                 opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
@@ -204,6 +205,14 @@ verbs_runtime_query(mca_base_module_t **module,
 #endif /* MPAGE_ENABLE */
     }
 
+#ifndef MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS
+    /* disqualify ourselves if we can not alloc contig
+     * pages at fixed address
+     */
+    if (mca_sshmem_verbs_component.has_shared_mr == 0)
+        rc = OSHMEM_ERR_OUT_OF_RESOURCE;
+#endif
+
     /* all is well - rainbows and butterflies */
     if (!rc) {
         *priority = mca_sshmem_verbs_component.priority;

oshmem/mca/sshmem/verbs/sshmem_verbs_module.c

@@ -260,6 +260,15 @@ segment_create(map_segment_t *ds_buf,
 #endif /* MPAGE_ENABLE */
 
         struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0};
+
+#ifdef MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS
+        if (0 == mca_sshmem_verbs_component.has_shared_mr) {
+            in.addr = (void *)mca_sshmem_base_start_address;
+            in.comp_mask    = IBV_EXP_REG_MR_CREATE_FLAGS;
+            in.create_flags = IBV_EXP_REG_MR_CREATE_CONTIG;
+            in.exp_access   = access_flag;
+        }
+#endif
         ib_mr = ibv_exp_reg_mr(&in);
         if (NULL == ib_mr) {
             OPAL_OUTPUT_VERBOSE(