OSHMEM: sshmem verbs: ofed api changes

Use only contig allocator if shared_mr() is not supported
Fixes ibv_exp_reg_mr() parameters passing.
Fixes incorrect handling of sshmem_verbs_hca_name option.

fixed by AlexM, reviewed by Mike

cmr=v1.8.1:reviewer=ompi-rm1.8

This commit was SVN r31379.

oshmem/mca/sshmem/sshmem_types.h

@@ -71,6 +71,7 @@ typedef enum {
     MAP_SEGMENT_ALLOC_MMAP,
     MAP_SEGMENT_ALLOC_SHM,
     MAP_SEGMENT_ALLOC_IBV,
+    MAP_SEGMENT_ALLOC_IBV_NOSHMR, 
     MAP_SEGMENT_UNKNOWN
 } segment_type_t;
 

oshmem/mca/sshmem/verbs/sshmem_verbs.h

@@ -45,14 +45,13 @@ struct ibv_exp_reg_mr_in {
     struct ibv_pd *pd;
     void *addr;
     size_t length;
-    int access;
+    uint64_t access;
-    int exp_access;
     uint32_t comp_mask;
 };
 
 static inline struct ibv_mr *ibv_exp_reg_mr(struct ibv_exp_reg_mr_in *in)
 {
-    return ibv_reg_mr(in->pd, in->addr, in->length, in->access | in->exp_access);
+    return ibv_reg_mr(in->pd, in->addr, in->length, in->access);
 }
 
 
@@ -70,6 +69,7 @@ typedef struct mca_sshmem_verbs_component_t {
     int priority;
     char* hca_name;
     int mr_interleave_factor;
+    int has_shared_mr;
 } mca_sshmem_verbs_component_t;
 
 OSHMEM_MODULE_DECLSPEC extern mca_sshmem_verbs_component_t

oshmem/mca/sshmem/verbs/sshmem_verbs_component.c

@@ -148,21 +148,21 @@ verbs_runtime_query(mca_base_module_t **module,
         void *addr = NULL;
         size_t size = getpagesize();
         struct ibv_mr *ib_mr = NULL;
-        int access_flag = IBV_ACCESS_LOCAL_WRITE |
+        uint64_t access_flag = IBV_ACCESS_LOCAL_WRITE |
                           IBV_ACCESS_REMOTE_WRITE |
-                          IBV_ACCESS_REMOTE_READ;
+                          IBV_ACCESS_REMOTE_READ; 
-        int exp_access_flag = 0;
+        uint64_t exp_access_flag = 0;
 
         OBJ_CONSTRUCT(&device->ib_mr_array, opal_value_array_t);
         opal_value_array_init(&device->ib_mr_array, sizeof(struct ibv_mr *));
 
 #if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
-        exp_access_flag = IBV_EXP_ACCESS_ALLOCATE_MR |
+        exp_access_flag = IBV_EXP_ACCESS_ALLOCATE_MR  |
                           IBV_EXP_ACCESS_SHARED_MR_USER_READ |
-                          IBV_EXP_ACCESS_SHARED_MR_USER_WRITE;
+                          IBV_EXP_ACCESS_SHARED_MR_USER_WRITE; 
 #endif /* MPAGE_ENABLE */
 
-        struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag, exp_access_flag, 0};
+        struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0};
         ib_mr = ibv_exp_reg_mr(&in);
         if (NULL == ib_mr) {
             rc = OSHMEM_ERR_OUT_OF_RESOURCE;
@@ -182,9 +182,10 @@ verbs_runtime_query(mca_base_module_t **module,
             struct ibv_exp_reg_shared_mr_in in = {0, device->ib_mr_shared->handle, device->ib_pd, addr, access_flag};
             ib_mr = ibv_exp_reg_shared_mr(&in);
             if (NULL == ib_mr) {
-                rc = OSHMEM_ERR_OUT_OF_RESOURCE;
+                mca_sshmem_verbs_component.has_shared_mr = 0;
             } else {
                 opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
+                mca_sshmem_verbs_component.has_shared_mr = 1;
             }
         }
 #endif /* MPAGE_ENABLE */

oshmem/mca/sshmem/verbs/sshmem_verbs_module.c

@@ -193,7 +193,7 @@ segment_create(map_segment_t *ds_buf,
     /* Open device */
     if (NULL != mca_sshmem_verbs_component.hca_name) {
         for (i = 0; i < num_devs; i++) {
-            if (0 == strcmp(mca_sshmem_verbs_component.hca_name, ibv_get_device_name(device->ib_devs[0]))) {
+            if (0 == strcmp(mca_sshmem_verbs_component.hca_name, ibv_get_device_name(device->ib_devs[i]))) {
                 device->ib_dev = device->ib_devs[i];
                 break;
             }
@@ -245,10 +245,10 @@ segment_create(map_segment_t *ds_buf,
     if (!rc) {
         void *addr = NULL;
         struct ibv_mr *ib_mr = NULL;
-        int access_flag = IBV_ACCESS_LOCAL_WRITE |
+        uint64_t access_flag = IBV_ACCESS_LOCAL_WRITE |
                           IBV_ACCESS_REMOTE_WRITE |
                           IBV_ACCESS_REMOTE_READ;
-        int exp_access_flag = 0;
+        uint64_t exp_access_flag = 0;
 
         OBJ_CONSTRUCT(&device->ib_mr_array, opal_value_array_t);
         opal_value_array_init(&device->ib_mr_array, sizeof(struct ibv_mr *));
@@ -259,7 +259,7 @@ segment_create(map_segment_t *ds_buf,
                           IBV_EXP_ACCESS_SHARED_MR_USER_WRITE;
 #endif /* MPAGE_ENABLE */
 
-        struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag, exp_access_flag, 0};
+        struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0};
         ib_mr = ibv_exp_reg_mr(&in);
         if (NULL == ib_mr) {
             OPAL_OUTPUT_VERBOSE(
@@ -274,7 +274,7 @@ segment_create(map_segment_t *ds_buf,
         }
 
 #if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
-        if (!rc) {
+        if (!rc && mca_sshmem_verbs_component.has_shared_mr) {
             access_flag = IBV_ACCESS_LOCAL_WRITE |
                           IBV_ACCESS_REMOTE_WRITE |
                           IBV_ACCESS_REMOTE_READ|
@@ -286,8 +286,10 @@ segment_create(map_segment_t *ds_buf,
             if (NULL == ib_mr) {
                 OPAL_OUTPUT_VERBOSE(
                     (5, oshmem_sshmem_base_framework.framework_output,
-                        "error to ibv_reg_shared_mr() %llu bytes errno says %d: %s",
+                        "error to ibv_reg_shared_mr() %llu bytes errno says %d: %s has_shared_mr: %d",
-                        (unsigned long long)size, errno, strerror(errno))
+                        (unsigned long long)size, errno, strerror(errno),
+                        mca_sshmem_verbs_component.has_shared_mr
+                        )
                     );
                 rc = OSHMEM_ERR_OUT_OF_RESOURCE;
             } else {
@@ -297,16 +299,21 @@ segment_create(map_segment_t *ds_buf,
 #endif /* MPAGE_ENABLE */
 
         if (!rc) {
-            assert(size == device->ib_mr_shared->length);
-
             OPAL_OUTPUT_VERBOSE(
                 (70, oshmem_sshmem_base_framework.framework_output,
-                "ibv device %s",
+                "ibv device %s shared_mr: %d",
-                ibv_get_device_name(device->ib_dev))
+                ibv_get_device_name(device->ib_dev),
+                mca_sshmem_verbs_component.has_shared_mr)
                 );
 
-            ds_buf->type = MAP_SEGMENT_ALLOC_IBV;
+            if (mca_sshmem_verbs_component.has_shared_mr) {
-            ds_buf->seg_id = device->ib_mr_shared->handle;
+                assert(size == device->ib_mr_shared->length);
+                ds_buf->type = MAP_SEGMENT_ALLOC_IBV;
+                ds_buf->seg_id = device->ib_mr_shared->handle;
+            } else {
+                ds_buf->type = MAP_SEGMENT_ALLOC_IBV_NOSHMR;
+                ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
+            }
             ds_buf->seg_base_addr = ib_mr->addr;
             ds_buf->seg_size = size;
             ds_buf->end = (void*)((uintptr_t)ds_buf->seg_base_addr + ds_buf->seg_size);
@@ -350,7 +357,7 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
         mca_sshmem_verbs_component.mr_interleave_factor * 1024ULL * 1024ULL * 1024ULL * mr_count);
     {
         struct ibv_mr *ib_mr = NULL;
-        int access_flag = IBV_ACCESS_LOCAL_WRITE |
+        uint64_t access_flag = IBV_ACCESS_LOCAL_WRITE |
             IBV_ACCESS_REMOTE_WRITE |
             IBV_ACCESS_REMOTE_READ |
             IBV_EXP_ACCESS_NO_RDMA;