OSHMEM: sshmem verbs: allocate memory at fixed address

Use experimental verbs to allocate memory at fixed base
virtual address.

verbs will disqualify itself if shared_mr is disabled
or not supported and it is impossible to allocate memory
starting at fixed base virtual address.

verbs contig pages allocator did not guarantee fixed va, now it does.
(cherry picked from commit fd77ebd4525e9e0c1a3ab1c4966bf31aa45251b4)

Apply Jeff`s comments

Update with Jeff commits
(cherry picked from commit open-mpi/ompi-release@4dc487fc3d)

oshmem/mca/sshmem/verbs/configure.m4

@@ -72,6 +72,8 @@ AC_DEFUN([MCA_oshmem_sshmem_verbs_CONFIG],[
 		        oshmem_verbs_sm_build_verbs=0
 		    fi
         ])
+
+    exp_reg_mr_happy=0
     AS_IF([test "$oshmem_have_mpage" = "3"],
             [
               AC_CHECK_MEMBER([struct ibv_exp_reg_shared_mr_in.exp_access],
@@ -81,13 +83,12 @@ AC_DEFUN([MCA_oshmem_sshmem_verbs_CONFIG],[
                 [],
                 [#include <infiniband/verbs_exp.h>])
 
-              AC_CHECK_MEMBER([struct ibv_exp_reg_mr_in.create_flags],
-                [AC_DEFINE_UNQUOTED(MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS, 1,
-                    [create_flags field is part of ibv_exp_reg_mr_in]
-                    )],
+                AC_CHECK_MEMBER([struct ibv_exp_reg_mr_in.create_flags],
+                [exp_reg_mr_happy=1],
                 [],
                 [#include <infiniband/verbs_exp.h>])
          ])
+    AC_DEFINE_UNQUOTED(MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS, $exp_reg_mr_happy, [create_flags field is part of ibv_exp_reg_mr_in]),
 
     AS_IF([test "$enable_verbs_sshmem" = "yes" -a "$oshmem_verbs_sm_build_verbs" = "0"],
           [AC_MSG_WARN([VERBS shared memory support requested but not found])

oshmem/mca/sshmem/verbs/sshmem_verbs_component.c

@@ -205,7 +205,7 @@ verbs_runtime_query(mca_base_module_t **module,
 #endif /* MPAGE_ENABLE */
     }
 
-#ifndef MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS
+#if !MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS
     /* disqualify ourselves if we can not alloc contig
      * pages at fixed address
      */

oshmem/mca/sshmem/verbs/sshmem_verbs_module.c

@@ -261,7 +261,7 @@ segment_create(map_segment_t *ds_buf,
 
         struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0};
 
-#ifdef MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS
+#if MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS
         if (0 == mca_sshmem_verbs_component.has_shared_mr) {
             in.addr = (void *)mca_sshmem_base_start_address;
             in.comp_mask    = IBV_EXP_REG_MR_CREATE_FLAGS;