OSHMEM: fixes problem with local heap2heap copy

check for possibility of heap2heap copy was incorrect
in case when shared heaps have different virtual
addresses on same host.

It seems that ibv_exp_reg_mr() on CIB cards may return
different VAs for heap on same node. On CX3 addresses are
the same.

reviewed by miked

cmr=v1.8.2:reviewer=ompi-rm1.8

This commit was SVN r31969.

oshmem/mca/memheap/memheap.h

@@ -153,6 +153,16 @@ typedef struct mca_memheap_base_module_t mca_memheap_base_module_t;
 
 OSHMEM_DECLSPEC extern mca_memheap_base_module_t mca_memheap;
 
+/**
+ * check if memcpy() can be used to copy data to dst_addr
+ * must be memheap address and segment must be mapped
+ */
+static inline int mca_memheap_base_can_local_copy(sshmem_mkey_t *mkey, void *dst_addr) {
+    return mca_memheap.memheap_is_symmetric_addr(dst_addr) &&
+        0 == mkey->len && mkey->u.key;
+}
+
+
 END_C_DECLS
 
 #endif /* MCA_MEMHEAP_H */

oshmem/mca/spml/ikrit/spml_ikrit.c

@@ -782,7 +782,7 @@ static inline int mca_spml_ikrit_get_shm(void *src_addr,
         return OSHMEM_ERROR;
     }
 
-    if (OPAL_UNLIKELY(!mca_memheap.memheap_is_symmetric_addr(src_addr) || src_addr == rva))
+    if (!mca_memheap_base_can_local_copy(r_mkey, src_addr))
         return OSHMEM_ERROR;
 
     SPML_VERBOSE(100,
@@ -1027,7 +1027,7 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
 #endif
     if (ptl_id == MXM_PTL_SHM) {
 
-        if (OPAL_LIKELY(mca_memheap.memheap_is_symmetric_addr(dst_addr) && dst_addr != rva)) {
+        if (mca_memheap_base_can_local_copy(r_mkey, dst_addr)) {
             memcpy((void *) (unsigned long) rva, src_addr, size);
             /* call progress as often as we would have with regular put */
             if (++count % SPML_IKRIT_PACKETS_PER_SYNC == 0)
@@ -1169,7 +1169,7 @@ int mca_spml_ikrit_put_simple(void* dst_addr,
 #endif
     if (ptl_id == MXM_PTL_SHM) {
 
-        if (OPAL_LIKELY(mca_memheap.memheap_is_symmetric_addr(dst_addr) && dst_addr != rva)) {
+        if (mca_memheap_base_can_local_copy(r_mkey, dst_addr)) {
             memcpy((void *) (unsigned long) rva, src_addr, size);
             /* call progress as often as we would have with regular put */
             if (++count % SPML_IKRIT_PACKETS_PER_SYNC == 0)

oshmem/mca/spml/yoda/spml_yoda.c

@@ -797,7 +797,7 @@ static inline int mca_spml_yoda_put_internal(void *dst_addr,
      * just do memcpy
      */
     if ((YODA_BTL_SM == ybtl->btl_type)
-            && OPAL_LIKELY(mca_memheap.memheap_is_symmetric_addr(dst_addr) && dst_addr != rva)) {
+            && mca_memheap_base_can_local_copy(r_mkey, dst_addr)) {
         memcpy((void *) (unsigned long) rva, src_addr, size);
         return OSHMEM_SUCCESS;
     }
@@ -1061,7 +1061,7 @@ int mca_spml_yoda_get(void* src_addr, size_t size, void* dst_addr, int src)
      * just do memcpy
      */
     if ((YODA_BTL_SM == ybtl->btl_type)
-            && OPAL_LIKELY(mca_memheap.memheap_is_symmetric_addr(src_addr) && src_addr != rva)) {
+            && mca_memheap_base_can_local_copy(r_mkey, dst_addr)) {
         memcpy(dst_addr, (void *) rva, size);
         /* must call progress here to avoid deadlock. Scenarion:
          * pe1 pols pe2 via shm get. pe2 tries to get static variable from node one, which goes to sm btl

oshmem/mca/sshmem/verbs/sshmem_verbs_component.c

@@ -185,8 +185,6 @@ verbs_runtime_query(mca_base_module_t **module,
             ib_mr = ibv_exp_reg_shared_mr(&in_smr);
             if (NULL == ib_mr) {
                 mca_sshmem_verbs_component.has_shared_mr = 0;
-                /* device does not support shared memory allocation, select another component */
-                rc = OSHMEM_ERR_OUT_OF_RESOURCE;
             } else {
                 opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
                 mca_sshmem_verbs_component.has_shared_mr = 1;