tuning

This commit was SVN r6370.

ompi/mca/btl/gm/btl_gm.c

@@ -507,6 +507,7 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_dst(
              * insert the registration into the tree and bump the reference
              * count so that it doesn't go away on completion.
             */
+            OBJ_RETAIN(registration);
             rc = mca_mpool_base_insert(
                 frag->segment.seg_addr.pval,
                 frag->segment.seg_len,
@@ -515,10 +516,11 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_dst(
                 registration);
             if(rc != OMPI_SUCCESS) {
                 MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
+                /* release twice */
+                OBJ_RELEASE(registration); 
                 OBJ_RELEASE(registration);
                 return NULL;
             }
-            OBJ_RETAIN(registration);
         }
         frag->registration = registration;
     }

ompi/mca/btl/gm/btl_gm.h

@@ -68,9 +68,7 @@ struct mca_btl_gm_component_t {
     opal_list_t gm_procs;   /**< list of gm proc structures */
     opal_mutex_t gm_lock;   /**< lock for accessing module state */
     char* gm_mpool_name;    /**< name of memory pool */ 
-
+    bool leave_pinned;      /**< pin memory on first use and leave pinned */
-    bool leave_pinned;
-    /**< pin memory on first use and leave pinned */
 }; 
 typedef struct mca_btl_gm_component_t mca_btl_gm_component_t;
 

ompi/mca/btl/gm/btl_gm_component.c

@@ -101,6 +101,8 @@ static inline int mca_btl_gm_param_register_int(
 
 int mca_btl_gm_component_open(void)
 {  
+    int param, value;
+
     /* initialize state */
     mca_btl_gm_component.gm_num_btls=0;
     mca_btl_gm_component.gm_btls=NULL;
@@ -141,9 +143,9 @@ int mca_btl_gm_component_open(void)
     mca_btl_gm_module.super.btl_max_send_size =
         mca_btl_gm_param_register_int ("max_send_size", 64*1024);
     mca_btl_gm_module.super.btl_min_rdma_size = 
-        mca_btl_gm_param_register_int("min_rdma_size", 1024*1024); 
+        mca_btl_gm_param_register_int("min_rdma_size", 256*1024); 
     mca_btl_gm_module.super.btl_max_rdma_size = 
-        mca_btl_gm_param_register_int("max_rdma_size", 1024*1024); 
+        mca_btl_gm_param_register_int("max_rdma_size", 256*1024); 
 #if OMPI_MCA_BTL_GM_SUPPORT_REGISTERING && OMPI_MCA_BTL_GM_HAVE_RDMA_PUT 
     mca_btl_gm_module.super.btl_flags  = 
         mca_btl_gm_param_register_int("flags", MCA_BTL_FLAGS_RDMA); 
@@ -164,6 +166,12 @@ int mca_btl_gm_component_open(void)
     mca_btl_gm_module.super.btl_max_send_size = 
         gm_max_length_for_size(mca_btl_gm_component.gm_max_frag_size) -
         sizeof(mca_btl_base_header_t);
+
+    /* leave pinned option */
+    value = 0;
+    param = mca_base_param_find("mpi", NULL, "leave_pinned");
+    mca_base_param_lookup_int(param, &value);
+    mca_btl_gm_component.leave_pinned = value;
     return OMPI_SUCCESS;
 }
 

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -110,13 +110,13 @@ int mca_pml_ob1_component_open(void)
     mca_pml_ob1.priority =
         mca_pml_ob1_param_register_int("priority", 0);
     mca_pml_ob1.eager_limit =
-        mca_pml_ob1_param_register_int("eager_limit", 256 * 1024);
+        mca_pml_ob1_param_register_int("eager_limit", 128 * 1024);
     mca_pml_ob1.send_pipeline_depth =
         mca_pml_ob1_param_register_int("send_pipeline_depth", 3);
     mca_pml_ob1.recv_pipeline_depth =
-        mca_pml_ob1_param_register_int("recv_pipeline_depth", 3);
+        mca_pml_ob1_param_register_int("recv_pipeline_depth", 5);
     mca_pml_ob1.rdma_offset = 
-        mca_pml_ob1_param_register_int("rdma_offset", 1024*1024); 
+        mca_pml_ob1_param_register_int("rdma_offset", 512*1024); 
     
     mca_base_param_register_int("mpi", NULL, "leave_pinned", "leave_pinned", 0); 
     param = mca_base_param_find("mpi", NULL, "leave_pinned"); 

ompi/mca/pml/ob1/pml_ob1_recvreq.h

@@ -124,6 +124,7 @@ do {
         mca_mpool_base_reg_mpool_t* reg = (recvreq)->req_chunk->mpools;              \
         while(NULL != reg->mpool) {                                                  \
             OBJ_RELEASE(reg->mpool_registration);                                    \
+            reg++;                                                                   \
         }                                                                            \
         OBJ_RELEASE((recvreq)->req_chunk);                                           \
     }                                                                                \

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -383,7 +383,9 @@ int mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq)
         do {
             /* allocate remaining bytes to BTLs */
             size_t bytes_remaining = sendreq->req_rdma_offset - sendreq->req_send_offset;
-            while(bytes_remaining > 0 && sendreq->req_pipeline_depth < mca_pml_ob1.send_pipeline_depth) {
+            while(bytes_remaining > 0 && 
+                  (sendreq->req_pipeline_depth < mca_pml_ob1.send_pipeline_depth ||
+                   sendreq->req_rdma_offset < sendreq->req_send.req_bytes_packed)) {
                 mca_pml_ob1_endpoint_t* ep = mca_pml_ob1_ep_array_get_next(&proc->btl_send);
                 mca_pml_ob1_frag_hdr_t* hdr;
                 mca_btl_base_descriptor_t* des;

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -231,6 +231,7 @@ OBJ_CLASS_DECLARATION(mca_pml_ob1_send_request_t);
         mca_mpool_base_reg_mpool_t* reg = (sendreq)->req_chunk->mpools;     \
         while(NULL != reg->mpool) {                                         \
             OBJ_RELEASE(reg->mpool_registration);                           \
+            reg++;                                                          \
         }                                                                   \
         OBJ_RELEASE((sendreq)->req_chunk);                                  \
     }                                                                       \