Fixes trac:366

Add ability for ini files to recognize "use_eager_rdma" flag.  Set the
default to "no" (because we should assume that HCAs cannot support the
property necessary for using RDMA for eager messages -- that the last
byte of the message is guaranteed to be written to memory last --
unless proven otherwise.  For example, iWARP cards apparently do not
provide this guarantee), and then set all Mellanox and IBM HCAs to
override the default to enable this behavior on these cards.

This commit was SVN r12851.

The following Trac tickets were found above:
  Ticket 366 --> https://svn.open-mpi.org/trac/ompi/ticket/366

ompi/mca/btl/openib/btl_openib.h

@@ -166,6 +166,8 @@ struct mca_btl_openib_hca_t {
     mca_mpool_base_module_t *mpool;
     /* MTU for this HCA */
     uint32_t mtu;
+    /* Whether this HCA supports eager RDMA */
+    uint8_t use_eager_rdma;
     uint8_t btls;              /** < number of btls using this HCA */
 };
 typedef struct mca_btl_openib_hca_t mca_btl_openib_hca_t;

ompi/mca/btl/openib/btl_openib_component.c

@@ -387,6 +387,11 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
         hca->mtu = mca_btl_openib_component.ib_mtu;
     }
 
+    /* If "use eager rdma" was set, then enable it on this HCA */
+    if (values.use_eager_rdma_set) {
+        hca->use_eager_rdma = values.use_eager_rdma;
+    }
+
     hca->ib_pd = ibv_alloc_pd(hca->ib_dev_context);
     if(NULL == hca->ib_pd){
         BTL_ERROR(("error allocating pd for %s errno says %s\n",
@@ -489,7 +494,6 @@ btl_openib_component_init(int *num_btl_modules,
     struct ibv_device* ib_dev; 
 #endif
     unsigned short seedv[3];
-    
 
     /* initialization */
     *num_btl_modules = 0;
@@ -597,6 +601,13 @@ btl_openib_component_init(int *num_btl_modules,
         free(openib_btl); 
         OBJ_RELEASE(ib_selected); 
 
+        /* Setup the "use eager rdma" flag -- look at the "use eager
+           rdma" flag on all the hcas, and if they're *all* true, then
+           set the component-wide flag to be true.  If not, set the
+           component-wide flag to be false. */
+        mca_btl_openib_component.use_eager_rdma &= 
+            openib_btl->hca->use_eager_rdma;
+
         openib_btl = &mca_btl_openib_component.openib_btls[i];
         openib_btl->rd_num = mca_btl_openib_component.rd_num +
             mca_btl_openib_component.rd_rsv;
@@ -731,6 +742,11 @@ static void merge_values(ompi_btl_openib_ini_values_t *target,
         target->mtu = src->mtu;
         target->mtu_set = true;
     }
+
+    if (!target->use_eager_rdma_set && src->use_eager_rdma_set) {
+        target->use_eager_rdma = src->use_eager_rdma;
+        target->use_eager_rdma_set = true;
+    }
 }
 
 

ompi/mca/btl/openib/btl_openib_ini.c

@@ -380,6 +380,12 @@ static int parse_line(parsed_section_values_t *sv)
         sv->values.mtu_set = true;
     }
 
+    else if (0 == strcasecmp(key_buffer, "use_eager_rdma")) {
+        /* Single value */
+        sv->values.use_eager_rdma = (uint32_t) intify(value);
+        sv->values.use_eager_rdma_set = true;
+    }
+
     else {
         /* Have no idea what this parameter is.  Not an error -- just
            ignore it */
@@ -391,8 +397,9 @@ static int parse_line(parsed_section_values_t *sv)
 
     /* All done */
 
-    if(NULL != value)
+    if (NULL != value) {
         free(value);
+    }
     return ret;
 }
 
@@ -454,6 +461,9 @@ static void reset_values(ompi_btl_openib_ini_values_t *v)
 {
     v->mtu = 0;
     v->mtu_set = false;
+
+    v->use_eager_rdma = 0;
+    v->use_eager_rdma_set = false;
 }
 
 
@@ -493,9 +503,15 @@ static int save_section(parsed_section_values_t *s)
                     if (s->values.mtu_set) {
                         h->values.mtu = s->values.mtu;
                         h->values.mtu_set = true;
-                        found = true;
-                        break;
                     }
+
+                    if (s->values.use_eager_rdma_set) {
+                        h->values.use_eager_rdma = s->values.use_eager_rdma;
+                        h->values.use_eager_rdma_set = true;
+                    }
+                    
+                    found = true;
+                    break;
                 }
             }
 

ompi/mca/btl/openib/btl_openib_ini.h

@@ -22,6 +22,9 @@
 typedef struct ompi_btl_openib_ini_values_t {
     uint32_t mtu;
     bool mtu_set;
+
+    uint32_t use_eager_rdma;
+    bool use_eager_rdma_set;
 } ompi_btl_openib_ini_values_t;
 
 

ompi/mca/btl/openib/mca-btl-openib-hca-params.ini

@@ -42,6 +42,16 @@
 #
 #    mtu = 1024
 
+# use_eager_rdma: an integer indicating whether RDMA should be used
+# for eager messages.  0 values indicate "no" (false); non-zero values
+# indicate "yes" (true).  This flag should only be enabled for
+# HCAs that can provide guarantees about ordering of data in memory --
+# that the last byte of an incoming packet will always be written
+# last.  Certain cards cannot provide this guarantee (such as iWARP
+# cards), while others can (such as many InfiniBand cards).  
+
+#    use_eager_rdma = 1
+
 ############################################################################
 
 [default]
@@ -52,6 +62,7 @@
 # decimal.
 vendor_id = 0
 vendor_part_id = 0
+use_eager_rdma = 0
 mtu = 1024
 
 ############################################################################
@@ -67,6 +78,7 @@ mtu = 1024
 #
 vendor_id = 0x2c9,0x5ad,0x66a,0x8f1
 vendor_part_id = 23108
+use_eager_rdma = 1
 mtu = 1024
 
 ############################################################################
@@ -74,6 +86,7 @@ mtu = 1024
 [Mellanox Arbel InfiniHost III MemFree/Tavor]
 vendor_id = 0x2c9,0x5ad,0x66a,0x8f1
 vendor_part_id = 25208,25218
+use_eager_rdma = 1
 mtu = 1024
 
 ############################################################################
@@ -81,6 +94,7 @@ mtu = 1024
 [Mellanox Sinai Infinihost III]
 vendor_id = 0x2c9,0x5ad,0x66a,0x8f1
 vendor_part_id = 25204
+use_eager_rdma = 1
 mtu = 2048
 
 ############################################################################
@@ -88,4 +102,5 @@ mtu = 2048
 [IBM eHCA 4x and 12x ]
 vendor_id = 0x5076
 vendor_part_id = 0
+use_eager_rdma = 1
 mtu = 2048