Introduce a new BTL parameter btl_rndv_eager_limit which determines size of a

first fragment of rendezvous protocol. Remove no longer used btl_min_send_size
parameter.

This commit was SVN r16969.

ompi/mca/bml/base/bml_base_init.c

@@ -26,7 +26,7 @@ int mca_bml_base_output = -1;
 mca_bml_base_module_t mca_bml = {
     NULL,                    /* bml_component */
     0,                       /* bml_eager_limit */ 
-    0,                       /* bml_min_send_size */ 
+    0,                       /* bml_rndv_eager_limit */ 
     0,                       /* bml_max_send_size */ 
     0,                       /* bml_min_rdma_size */ 
     0,                       /* bml_max_rdma_size */ 

ompi/mca/bml/bml.h

@@ -57,8 +57,8 @@ struct mca_bml_base_btl_t {
     int    btl_flags;                             /**< support for put/get? */
     double btl_weight;                            /**< BTL weight for scheduling */
     size_t btl_eager_limit;                       /**< BTL eager limit */
-    size_t btl_min_send_size;                     /**< BTL min send size */
-    size_t btl_max_send_size;                     /**< BTL max send size */
+    size_t btl_rndv_eager_limit;                  /**< BTL rndv eager limit */
+    size_t btl_max_send_size;                     /**< BTL min send size */
     size_t btl_rdma_pipeline_send_length;         /**< BTL send length */
     size_t btl_rdma_pipeline_frag_size;           /**< BTL rdma frag size */
     size_t btl_min_rdma_pipeline_size;            /**< BTL min rdma size */
@@ -592,7 +592,7 @@ struct mca_bml_base_module_t {
     /* BML common attributes */
     mca_bml_base_component_t* bml_component; /**< pointer back to the BML component structure */
     size_t      bml_eager_limit;      /**< maximum size of first fragment -- eager send */
-    size_t      bml_min_send_size;    /**< threshold below which the BML should not fragment */
+    size_t      bml_rndv_eager_limit;    /**< size of a first fragment of rndv protocol */
     size_t      bml_max_send_size;    /**< maximum send fragment size supported by the BML */
     size_t      bml_min_rdma_size;    /**< threshold below which the BML should not fragment */
     size_t      bml_max_rdma_size;    /**< maximum rdma fragment size supported by the BML */

ompi/mca/bml/r2/bml_r2.c

@@ -301,7 +301,7 @@ int mca_bml_r2_add_procs(
                 bml_btl = mca_bml_base_btl_array_insert(&bml_endpoint->btl_send);
                 bml_btl->btl = btl;
                 bml_btl->btl_eager_limit = btl->btl_eager_limit;
-                bml_btl->btl_min_send_size = btl->btl_min_send_size;
+                bml_btl->btl_rndv_eager_limit = btl->btl_rndv_eager_limit;
                 bml_btl->btl_max_send_size = btl->btl_max_send_size;
                 bml_btl->btl_rdma_pipeline_send_length =
                     btl->btl_rdma_pipeline_send_length;

ompi/mca/btl/base/btl_base_mca.c

@@ -57,10 +57,10 @@ int mca_btl_base_param_register(mca_base_component_t *version,
             0, uint32_t);
     free(msg);
 
-    REG_INT("min_send_size", "Minimum message size (in bytes) that will be striped across multiple network devices when using send/receive semantics.  Messages shorter than this size will be sent across a single network (must be >= 1)",
-            module->btl_min_send_size, 1, size_t);
+    REG_INT("rndv_eager_limit", "Size (in bytes) of \"phase 1\" fragment sent for all large messages (must be >= 0 and <= eager_limit)",
+            module->btl_rndv_eager_limit, 0, size_t);
 
-    REG_INT("eager_limit", "Size (in bytes) of the first fragment sent of any message.  It is the maximum size of \"short\" messages and the maximum size of the \"phase 1\" fragment sent for all large messages (must be >= 1).",
+    REG_INT("eager_limit", "Maximum size (in bytes) of \"short\" messages (must be >= 1).",
             module->btl_eager_limit, 1, size_t);
 
     REG_INT("max_send_size", "Maximum size (in bytes) of a single \"phase 2\" fragment of a long message when using the pipeline protocol (must be >= 1)",

ompi/mca/btl/btl.h

@@ -685,7 +685,7 @@ struct mca_btl_base_module_t {
     /* BTL common attributes */
     mca_btl_base_component_t* btl_component; /**< pointer back to the BTL component structure */
     size_t      btl_eager_limit;      /**< maximum size of first fragment -- eager send */
-    size_t      btl_min_send_size;    /**< threshold below which the BTL should not fragment */
+    size_t      btl_rndv_eager_limit;    /**< the size of a data sent in a first fragment of rendezvous protocol */
     size_t      btl_max_send_size;    /**< maximum send fragment size supported by the BTL */
     size_t      btl_rdma_pipeline_send_length; /**< amount of bytes that should be send by pipeline protocol */
     size_t      btl_rdma_pipeline_frag_size; /**< maximum rdma fragment size supported by the BTL */

ompi/mca/btl/elan/btl_elan_component.c

@@ -112,7 +112,7 @@ int mca_btl_elan_component_open(void)
         mca_btl_elan_param_register_string("mpool", "elan"); 
     mca_btl_elan_module.super.btl_exclusivity = 0;
     mca_btl_elan_module.super.btl_eager_limit =  32*1024;
-    mca_btl_elan_module.super.btl_min_send_size = 32*1024;
+    mca_btl_elan_module.super.btl_rndv_eager_limit = 32*1024;
     mca_btl_elan_module.super.btl_max_send_size = 64*1024; /*64*1024;*/
     mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 * 1024;
     mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;

ompi/mca/btl/gm/btl_gm_component.c

@@ -153,7 +153,7 @@ int mca_btl_gm_component_open(void)
     /* register gm module parameters */
     mca_btl_gm_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
     mca_btl_gm_module.super.btl_eager_limit = 32*1024;
-    mca_btl_gm_module.super.btl_min_send_size = 32*1024;
+    mca_btl_gm_module.super.btl_rndv_eager_limit = 32*1024;
     mca_btl_gm_module.super.btl_max_send_size = 64*1024;
     mca_btl_gm_module.super.btl_rdma_pipeline_send_length = 512*1024;
     mca_btl_gm_module.super.btl_rdma_pipeline_frag_size = 128*1024;

ompi/mca/btl/mx/btl_mx_component.c

@@ -144,7 +144,7 @@ int mca_btl_mx_component_open(void)
 
     mca_btl_mx_module.super.btl_exclusivity = 50;
     mca_btl_mx_module.super.btl_eager_limit = 4096;
-    mca_btl_mx_module.super.btl_min_send_size = 4096;
+    mca_btl_mx_module.super.btl_rndv_eager_limit = 4096;
     mca_btl_mx_module.super.btl_max_send_size = 64*1024;
     mca_btl_mx_module.super.btl_rdma_pipeline_send_length = 256*1024;
     mca_btl_mx_module.super.btl_rdma_pipeline_frag_size = 8*1024*1024;

ompi/mca/btl/ofud/btl_ofud_component.c

@@ -156,7 +156,7 @@ int mca_btl_ud_component_open(void)
                              2048, &val); */
     mca_btl_ud_param_reg_int("min_send_size", "minimum send size",
                              2048, &val);
-    mca_btl_ofud_module.super.btl_min_send_size = val;
+    mca_btl_ofud_module.super.btl_rndv_eager_limit = val;
     mca_btl_ud_param_reg_int("max_send_size", "maximum send size",
                              2048, &val);
     mca_btl_ofud_module.super.btl_eager_limit = val;

ompi/mca/btl/openib/btl_openib_mca.c

@@ -436,7 +436,7 @@ int btl_openib_register_mca_params(void)
     mca_btl_openib_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
     
     mca_btl_openib_module.super.btl_eager_limit = 12 * 1024;
-    mca_btl_openib_module.super.btl_min_send_size = 32 * 1024;
+    mca_btl_openib_module.super.btl_rndv_eager_limit = 12 * 1024;
     mca_btl_openib_module.super.btl_max_send_size = 64 * 1024;
     mca_btl_openib_module.super.btl_rdma_pipeline_send_length = 1024 * 1024;
     mca_btl_openib_module.super.btl_rdma_pipeline_frag_size = 1024 * 1024;    

ompi/mca/btl/portals/btl_portals_component.c

@@ -130,7 +130,7 @@ mca_btl_portals_component_open(void)
      */
     mca_btl_portals_module.super.btl_exclusivity = 60;
     mca_btl_portals_module.super.btl_eager_limit = 32 * 1024;
-    mca_btl_portals_module.super.btl_min_send_size = 32 * 1024;
+    mca_btl_portals_module.super.btl_rndv_eager_limit = 32 * 1024;
     mca_btl_portals_module.super.btl_max_send_size = 64 * 1024;
     mca_btl_portals_module.super.btl_rdma_pipeline_send_length = 64 * 1024;
     mca_btl_portals_module.super.btl_rdma_pipeline_frag_size = INT_MAX;

ompi/mca/btl/sctp/btl_sctp_component.c

@@ -246,7 +246,7 @@ int mca_btl_sctp_component_open(void)
     /* have lower exclusivity than tcp */
     mca_btl_sctp_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_LOW;
     mca_btl_sctp_module.super.btl_eager_limit = 64*1024;
-    mca_btl_sctp_module.super.btl_min_send_size = 64*1024;
+    mca_btl_sctp_module.super.btl_rndv_eager_limit = 64*1024;
     mca_btl_sctp_module.super.btl_max_send_size = 128*1024;
     mca_btl_sctp_module.super.btl_rdma_pipeline_send_length = 128*1024;
     mca_btl_sctp_module.super.btl_rdma_pipeline_frag_size = INT_MAX;

ompi/mca/btl/self/btl_self.c

@@ -46,7 +46,7 @@
 mca_btl_base_module_t mca_btl_self = {
     &mca_btl_self_component.super,
     0, /* btl_eager_limit */
-    0, /* btl_min_send_size */
+    0, /* btl_rndv_eager_limit */
     0, /* btl_max_send_size */
     0, /* btl_rdma_pipeline_send_length */
     0, /* btl_rdma_pipeline_frag_size */

ompi/mca/btl/self/btl_self_component.c

@@ -92,7 +92,7 @@ int mca_btl_self_component_open(void)
 
     mca_btl_self.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
     mca_btl_self.btl_eager_limit = 128 * 1024;
-    mca_btl_self.btl_min_send_size = 256 * 1024;
+    mca_btl_self.btl_rndv_eager_limit = 128 * 1024;
     mca_btl_self.btl_max_send_size = 256 * 1024;
     mca_btl_self.btl_rdma_pipeline_send_length = INT_MAX;
     mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX;

ompi/mca/btl/sm/btl_sm.c

@@ -87,7 +87,7 @@ mca_btl_sm_t mca_btl_sm = {
     {
         &mca_btl_sm_component.super,
         0, /* btl_eager_limit */
-        0, /* btl_min_send_size */
+        0, /* btl_rndv_eager_limit */
         0, /* btl_max_send_size */
         0, /* btl_rdma_pipeline_send_length */
         0, /* btl_rdma_pipeline_frag_size */

ompi/mca/btl/sm/btl_sm_component.c

@@ -160,7 +160,7 @@ int mca_btl_sm_component_open(void)
 
     mca_btl_sm.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH-1;
     mca_btl_sm.super.btl_eager_limit = 4*1024;
-    mca_btl_sm.super.btl_min_send_size = 32*1024;
+    mca_btl_sm.super.btl_rndv_eager_limit = 4*1024;
     mca_btl_sm.super.btl_max_send_size = 32*1024;
     mca_btl_sm.super.btl_rdma_pipeline_send_length = 32*1024;
     mca_btl_sm.super.btl_rdma_pipeline_frag_size = 32*1024;

ompi/mca/btl/tcp/btl_tcp_component.c

@@ -246,7 +246,7 @@ int mca_btl_tcp_component_open(void)
 #endif
     mca_btl_tcp_module.super.btl_exclusivity =  MCA_BTL_EXCLUSIVITY_LOW + 100;
     mca_btl_tcp_module.super.btl_eager_limit = 64*1024;
-    mca_btl_tcp_module.super.btl_min_send_size = 64*1024;
+    mca_btl_tcp_module.super.btl_rndv_eager_limit = 64*1024;
     mca_btl_tcp_module.super.btl_max_send_size = 128*1024;
     mca_btl_tcp_module.super.btl_rdma_pipeline_send_length = 128*1024;
     mca_btl_tcp_module.super.btl_rdma_pipeline_frag_size = INT_MAX;

ompi/mca/btl/template/btl_template_component.c

@@ -120,7 +120,7 @@ int mca_btl_template_component_open(void)
         mca_btl_template_param_register_int ("exclusivity", 0);
     mca_btl_template_module.super.btl_eager_limit = 
         mca_btl_template_param_register_int ("first_frag_size", 64*1024) - sizeof(mca_btl_base_header_t);
-    mca_btl_template_module.super.btl_min_send_size =
+    mca_btl_template_module.super.btl_rndv_eager_limit =
         mca_btl_template_param_register_int ("min_send_size", 64*1024) - sizeof(mca_btl_base_header_t);
     mca_btl_template_module.super.btl_max_send_size =
         mca_btl_template_param_register_int ("max_send_size", 128*1024) - sizeof(mca_btl_base_header_t);

ompi/mca/btl/udapl/btl_udapl_mca.c

@@ -283,7 +283,7 @@ int mca_btl_udapl_register_mca_params(void)
     mca_btl_udapl_module.super.btl_exclusivity =
         MCA_BTL_EXCLUSIVITY_DEFAULT - 10;
     mca_btl_udapl_module.super.btl_eager_limit = 8*1024;
-    mca_btl_udapl_module.super.btl_min_send_size = 16*1024;
+    mca_btl_udapl_module.super.btl_rndv_eager_limit = 8*1024;
     mca_btl_udapl_module.super.btl_max_send_size = 64*1024;
     mca_btl_udapl_module.super.btl_rdma_pipeline_send_length = 512*1024;
     mca_btl_udapl_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -363,7 +363,10 @@ mca_pml_ob1_send_request_start_btl( mca_pml_ob1_send_request_t* sendreq,
             break;
         }
     } else {
-        size = eager_limit;
+        if(bml_btl->btl_rndv_eager_limit < eager_limit)
+            size = bml_btl->btl_rndv_eager_limit;
+        else
+            size = eager_limit;
         if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
             rc = mca_pml_ob1_send_request_start_buffered(sendreq, bml_btl, size);
         } else if