Allow maximum send size to be less than the eager limit.

Instead of figuring out which free list the fragment belongs to based on size
we simply store a pointer to the list which it belongs in the fragment.

This was reviewed by Brian and should hit all the branches.

This commit was SVN r10072.

ompi/mca/btl/tcp/btl_tcp.c

@@ -215,15 +215,7 @@ int mca_btl_tcp_free(
     mca_btl_base_descriptor_t* des) 
 {
     mca_btl_tcp_frag_t* frag = (mca_btl_tcp_frag_t*)des; 
-    if(frag->size == 0) {
-        MCA_BTL_TCP_FRAG_RETURN_USER(frag); 
-    } else if(frag->size == btl->btl_eager_limit){ 
-        MCA_BTL_TCP_FRAG_RETURN_EAGER(frag); 
-    } else if(frag->size == btl->btl_max_send_size) {
-        MCA_BTL_TCP_FRAG_RETURN_MAX(frag); 
-    }  else {
-        return OMPI_ERR_BAD_PARAM;
-    }
+    MCA_BTL_TCP_FRAG_RETURN(frag); 
     return OMPI_SUCCESS; 
 }
 

ompi/mca/btl/tcp/btl_tcp_component.c

@@ -98,7 +98,6 @@ mca_btl_tcp_component_t mca_btl_tcp_component = {
     }
 };
 
-
 /*
  * utility routines for parameter registration
  */
@@ -216,6 +215,7 @@ int mca_btl_tcp_component_open(void)
         mca_btl_tcp_param_register_int ("min_send_size", 64*1024);
     mca_btl_tcp_module.super.btl_max_send_size =
         mca_btl_tcp_param_register_int ("max_send_size", 128*1024);
+    
     mca_btl_tcp_module.super.btl_min_rdma_size =
         mca_btl_tcp_param_register_int("min_rdma_size", 128*1024);
     mca_btl_tcp_module.super.btl_max_rdma_size =

ompi/mca/btl/tcp/btl_tcp_endpoint.c

@@ -606,7 +606,13 @@ static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void* user)
             frag = btl_endpoint->endpoint_recv_frag;
             if(NULL == frag) {
                 int rc;
-                MCA_BTL_TCP_FRAG_ALLOC_MAX(frag, rc);
+                if(mca_btl_tcp_module.super.btl_max_send_size > 
+                   mca_btl_tcp_module.super.btl_eager_limit) { 
+                    MCA_BTL_TCP_FRAG_ALLOC_MAX(frag, rc);
+                } else { 
+                    MCA_BTL_TCP_FRAG_ALLOC_EAGER(frag, rc);
+                }
+                
                 if(NULL == frag) {
                     OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
                     return;
@@ -642,7 +648,7 @@ static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void* user)
                     goto data_still_pending_on_endpoint;
                 }
 #endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
-                MCA_BTL_TCP_FRAG_RETURN_MAX(frag);
+                MCA_BTL_TCP_FRAG_RETURN(frag);
             }
             OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
 #if MCA_BTL_TCP_ENDPOINT_CACHE

ompi/mca/btl/tcp/btl_tcp_frag.c

@@ -50,18 +50,21 @@ static void mca_btl_tcp_frag_common_constructor(mca_btl_tcp_frag_t* frag)
 static void mca_btl_tcp_frag_eager_constructor(mca_btl_tcp_frag_t* frag) 
 { 
     frag->size = mca_btl_tcp_module.super.btl_eager_limit;   
+    frag->my_list = &mca_btl_tcp_component.tcp_frag_eager;
     mca_btl_tcp_frag_common_constructor(frag); 
 }
 
 static void mca_btl_tcp_frag_max_constructor(mca_btl_tcp_frag_t* frag) 
 { 
     frag->size = mca_btl_tcp_module.super.btl_max_send_size; 
+    frag->my_list = &mca_btl_tcp_component.tcp_frag_max;
     mca_btl_tcp_frag_common_constructor(frag); 
 }
 
 static void mca_btl_tcp_frag_user_constructor(mca_btl_tcp_frag_t* frag) 
 { 
     frag->size = 0; 
+    frag->my_list = &mca_btl_tcp_component.tcp_frag_user;
     mca_btl_tcp_frag_common_constructor(frag); 
 }
 

ompi/mca/btl/tcp/btl_tcp_frag.h

@@ -55,6 +55,7 @@ struct mca_btl_tcp_frag_t {
     size_t iov_idx;
     size_t size; 
     int rc;
+    ompi_free_list_t* my_list;
 }; 
 typedef struct mca_btl_tcp_frag_t mca_btl_tcp_frag_t; 
 OBJ_CLASS_DECLARATION(mca_btl_tcp_frag_t); 
@@ -86,12 +87,6 @@ OBJ_CLASS_DECLARATION(mca_btl_tcp_frag_user_t);
     frag = (mca_btl_tcp_frag_t*) item;                                     \
 }
 
-#define MCA_BTL_TCP_FRAG_RETURN_EAGER(frag)                                \
-{                                                                          \
-    OMPI_FREE_LIST_RETURN(&mca_btl_tcp_component.tcp_frag_eager,           \
-        (opal_list_item_t*)(frag));                                        \
-}
-
 #define MCA_BTL_TCP_FRAG_ALLOC_MAX(frag, rc)                               \
 {                                                                          \
                                                                            \
@@ -100,13 +95,6 @@ OBJ_CLASS_DECLARATION(mca_btl_tcp_frag_user_t);
     frag = (mca_btl_tcp_frag_t*) item;                                     \
 }
 
-#define MCA_BTL_TCP_FRAG_RETURN_MAX(frag)                                  \
-{                                                                          \
-    OMPI_FREE_LIST_RETURN(&mca_btl_tcp_component.tcp_frag_max,             \
-        (opal_list_item_t*)(frag));                                        \
-}
-
-
 #define MCA_BTL_TCP_FRAG_ALLOC_USER(frag, rc)                              \
 {                                                                          \
     opal_list_item_t *item;                                                \
@@ -114,9 +102,9 @@ OBJ_CLASS_DECLARATION(mca_btl_tcp_frag_user_t);
     frag = (mca_btl_tcp_frag_t*) item;                                     \
 }
 
-#define MCA_BTL_TCP_FRAG_RETURN_USER(frag)                                 \
+#define MCA_BTL_TCP_FRAG_RETURN(frag)                                      \
 {                                                                          \
-    OMPI_FREE_LIST_RETURN(&mca_btl_tcp_component.tcp_frag_user,            \
+    OMPI_FREE_LIST_RETURN(frag->my_list,                                   \
         (opal_list_item_t*)(frag));                                        \
 }