* Provide flag if the BTL can do RDMA, but requires a prepare_{src,dst}

that exactly describes the buffer to be used as the target of the
    operation
  * Use the above flag to disable components setting the flag from being
    used for real RDMA operations for the one-sided component (the
    BTLs will still be used for RDMA transfers for the PML and for
    send/receive communication for the OSC component) 

This commit was SVN r15375.

ompi/mca/btl/btl.h

@@ -145,23 +145,27 @@ typedef uint8_t mca_btl_base_tag_t;
 #define MCA_BTL_TAG_MAX      255 /* 1 + highest allowed tag num */
 
 /* prefered protocol */
-#define MCA_BTL_FLAGS_SEND  0x1
-#define MCA_BTL_FLAGS_PUT   0x2
-#define MCA_BTL_FLAGS_GET   0x4
+#define MCA_BTL_FLAGS_SEND            0x0001
+#define MCA_BTL_FLAGS_PUT             0x0002
+#define MCA_BTL_FLAGS_GET             0x0004
 #define MCA_BTL_FLAGS_RDMA (MCA_BTL_FLAGS_GET|MCA_BTL_FLAGS_PUT)
 
 /* btl can send directly from user buffer w/out registration */
-#define MCA_BTL_FLAGS_SEND_INPLACE  0x8
+#define MCA_BTL_FLAGS_SEND_INPLACE    0x0008
 
 /* btl transport is reliable */
-#define MCA_BTL_FLAGS_NEED_ACK 0x10
-#define MCA_BTL_FLAGS_NEED_CSUM 0x20
+#define MCA_BTL_FLAGS_NEED_ACK        0x0010
+#define MCA_BTL_FLAGS_NEED_CSUM       0x0020
+
+/** RDMA put/get calls must have a matching prepare_{src,dst} call
+    on the target with the same base (and possibly bound). */
+#define MCA_BTL_FLAGS_RDMA_MATCHED    0x0040
 
 /* btl needs local rdma completion */
-#define MCA_BTL_FLAGS_RDMA_COMPLETION 0x80
+#define MCA_BTL_FLAGS_RDMA_COMPLETION 0x0080
 
 /* Default exclusivity levels */
-#define MCA_BTL_EXCLUSIVITY_HIGH     64*1024   /* internal loopback */
+#define MCA_BTL_EXCLUSIVITY_HIGH     (64*1024) /* internal loopback */
 #define MCA_BTL_EXCLUSIVITY_DEFAULT  1024      /* GM/IB/etc. */
 #define MCA_BTL_EXCLUSIVITY_LOW      0         /* TCP used as a last resort */
 

ompi/mca/btl/mx/btl_mx_component.c

@@ -149,7 +149,11 @@ int mca_btl_mx_component_open(void)
     mca_btl_mx_module.super.btl_rdma_pipeline_send_length = 256*1024;
     mca_btl_mx_module.super.btl_rdma_pipeline_frag_size = 8*1024*1024;
     mca_btl_mx_module.super.btl_min_rdma_pipeline_size = 0;
-    mca_btl_mx_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND;
+    mca_btl_mx_module.super.btl_flags = 
+        MCA_BTL_FLAGS_SEND_INPLACE | 
+        MCA_BTL_FLAGS_PUT | 
+        MCA_BTL_FLAGS_SEND |
+        MCA_BTL_FLAGS_RDMA_MATCHED;
     mca_btl_mx_module.super.btl_bandwidth = 2000;
     mca_btl_mx_module.super.btl_latency = 5;
     mca_btl_base_param_register(&mca_btl_mx_component.super.btl_version,

ompi/mca/btl/portals/btl_portals_component.c

@@ -135,7 +135,9 @@ mca_btl_portals_component_open(void)
     mca_btl_portals_module.super.btl_rdma_pipeline_send_length = 64 * 1024;
     mca_btl_portals_module.super.btl_rdma_pipeline_frag_size = INT_MAX;
     mca_btl_portals_module.super.btl_min_rdma_pipeline_size = 0;
-    mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_RDMA;
+    mca_btl_portals_module.super.btl_flags = 
+        MCA_BTL_FLAGS_RDMA |
+        MCA_BTL_FLAGS_RDMA_MATCHED;
     mca_btl_portals_module.super.btl_bandwidth = 1000;
     mca_btl_portals_module.super.btl_latency = 0;
 

ompi/mca/osc/rdma/osc_rdma_component.c

@@ -876,6 +876,13 @@ component_fragment_cb(struct mca_btl_base_module_t *btl,
                                 "received rdma info for unknown btl from rank %d",
                                 origin);
                     return;
+                } else {
+                    OPAL_OUTPUT_VERBOSE((1, ompi_osc_base_output,
+                                         "received rdma info from rank %d for BTL %s",
+                                         origin,
+                                         bml_btl->btl->
+                                         btl_component->btl_version.
+                                         mca_component_name));
                 }
 
                 OPAL_THREAD_LOCK(&module->m_lock);
@@ -1089,8 +1096,9 @@ rdma_send_info_send(ompi_osc_rdma_module_t *module,
 static bool
 is_valid_rdma(mca_bml_base_btl_t *bml_btl)
 {
-    if (((bml_btl->btl_flags & MCA_BTL_FLAGS_PUT) != 0) &&
-        ((bml_btl->btl_flags & MCA_BTL_FLAGS_GET) != 0)) {
+    if ((bml_btl->btl->btl_put != NULL) &&
+        (bml_btl->btl->btl_get != NULL) &&
+        ((bml_btl->btl_flags & MCA_BTL_FLAGS_RDMA_MATCHED) == 0)) {
         return true;
     }