btl/vader: don't assume the address in the put/get segment is unmodified when

using knem

It is valid to modify the remote segment that will be used with the
btl put/get operations as long as the resulting address range falls in
the originally prepared segment. Vader should have been calculating the
offset of the remote address in the registered region. This commit
fixes this issue.

opal/mca/btl/vader/btl_vader_frag.h

@@ -61,6 +61,7 @@ struct mca_btl_vader_segment_t {
     mca_btl_base_segment_t base;
 #if OPAL_BTL_VADER_HAVE_KNEM
     uint64_t cookie;
+    intptr_t registered_base;
 #endif
 };
 typedef struct mca_btl_vader_segment_t mca_btl_vader_segment_t;

opal/mca/btl/vader/btl_vader_get.c

@@ -101,6 +101,7 @@ int mca_btl_vader_get_knem (struct mca_btl_base_module_t *btl,
     mca_btl_vader_segment_t *src = (mca_btl_vader_segment_t *) des->des_remote;
     mca_btl_vader_segment_t *dst = (mca_btl_vader_segment_t *) des->des_local;
     const size_t size = min(dst->base.seg_len, src->base.seg_len);
+    intptr_t offset = src->base.seg_addr.lval - src->registered_base;
     struct knem_cmd_param_iovec recv_iovec;
     struct knem_cmd_inline_copy icopy;
 
@@ -111,7 +112,7 @@ int mca_btl_vader_get_knem (struct mca_btl_base_module_t *btl,
     icopy.local_iovec_array = (uintptr_t) &recv_iovec;
     icopy.local_iovec_nr    = 1;
     icopy.remote_cookie     = src->cookie;
-    icopy.remote_offset     = 0;
+    icopy.remote_offset     = offset;
     icopy.write             = 0;
     icopy.flags             = 0;
 

opal/mca/btl/vader/btl_vader_module.c

@@ -483,6 +483,7 @@ struct mca_btl_base_descriptor_t *vader_prepare_dst(struct mca_btl_base_module_t
         }
 
         frag->segments[0].cookie = knem_cr.cookie;
+        frag->segments[0].registered_base = (intptr_t) data_ptr;
         frag->cookie = knem_cr.cookie;
     }
 #endif /* OPAL_BTL_SM_HAVE_KNEM */
@@ -625,6 +626,7 @@ static struct mca_btl_base_descriptor_t *vader_prepare_src (struct mca_btl_base_
             }
 
             frag->segments[0].cookie = knem_cr.cookie;
+            frag->segments[0].registered_base = (intptr_t) data_ptr;
             frag->cookie = knem_cr.cookie;
         }
 #endif /* OPAL_BTL_SM_HAVE_KNEM */

opal/mca/btl/vader/btl_vader_put.c

@@ -103,6 +103,7 @@ int mca_btl_vader_put_knem (struct mca_btl_base_module_t *btl,
     mca_btl_vader_segment_t *src = (mca_btl_vader_segment_t *) des->des_local;
     mca_btl_vader_segment_t *dst = (mca_btl_vader_segment_t *) des->des_remote;
     const size_t size = min(dst->base.seg_len, src->base.seg_len);
+    intptr_t offset = dst->base.seg_addr.lval - dst->registered_base;
     struct knem_cmd_param_iovec send_iovec;
     struct knem_cmd_inline_copy icopy;
 
@@ -113,7 +114,7 @@ int mca_btl_vader_put_knem (struct mca_btl_base_module_t *btl,
     icopy.local_iovec_array = (uintptr_t) &send_iovec;
     icopy.local_iovec_nr    = 1;
     icopy.remote_cookie     = dst->cookie;
-    icopy.remote_offset     = 0;
+    icopy.remote_offset     = offset;
     icopy.write             = 1;
     icopy.flags             = 0;