diff --git a/ompi/mca/btl/openib/btl_openib.c b/ompi/mca/btl/openib/btl_openib.c index 98ec20254e..c83ddea8f0 100644 --- a/ompi/mca/btl/openib/btl_openib.c +++ b/ompi/mca/btl/openib/btl_openib.c @@ -223,17 +223,9 @@ int mca_btl_openib_free( btl->btl_mpool->mpool_release(btl->btl_mpool, (mca_mpool_base_registration_t*) frag->openib_reg); - MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag); - - } - else if(frag->size == mca_btl_openib_component.max_send_size){ - MCA_BTL_IB_FRAG_RETURN_MAX(btl, frag); - } else if(frag->size == mca_btl_openib_component.eager_limit){ - MCA_BTL_IB_FRAG_RETURN_EAGER(btl, frag); - } else { - BTL_ERROR(("invalid descriptor")); - } - + } + MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*) btl), frag); + return OMPI_SUCCESS; } @@ -326,7 +318,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( reserve == 0) { /* The user buffer is contigous and we are asked to send more than the max send size. */ - MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc); + MCA_BTL_IB_FRAG_ALLOC_FRAG(openib_btl, frag, rc); if(NULL == frag){ return NULL; } @@ -348,7 +340,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( (mca_mpool_base_registration_t**) &openib_reg); if(OMPI_SUCCESS != rc || NULL == openib_reg) { BTL_ERROR(("mpool_register(%p,%lu) failed", iov.iov_base, max_data)); - MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag); + MCA_BTL_IB_FRAG_RETURN(openib_btl, frag); return NULL; } @@ -386,7 +378,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); *size = max_data; if( rc < 0 ) { - MCA_BTL_IB_FRAG_RETURN_EAGER(btl, frag); + MCA_BTL_IB_FRAG_RETURN(openib_btl, frag); return NULL; } @@ -416,7 +408,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( *size = max_data; if( rc < 0 ) { - MCA_BTL_IB_FRAG_RETURN_MAX(btl, frag); + MCA_BTL_IB_FRAG_RETURN(openib_btl, frag); return NULL; } @@ -493,7 +485,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst( if(OMPI_SUCCESS != rc || NULL == openib_reg) { BTL_ERROR(("mpool_register(%p,%lu) failed: base %p lb %lu offset %lu", frag->segment.seg_addr.pval, *size, convertor->pBaseBuf, lb, convertor->bConverted)); - MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag); + MCA_BTL_IB_FRAG_RETURN(openib_btl, frag); return NULL; } } diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index 4d8bc61be1..d4fffa0c2d 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -57,7 +57,7 @@ struct mca_btl_openib_component_t { int ib_max_btls; /**< maximum number of hcas available to the IB component */ - uint32_t ib_num_btls; + int ib_num_btls; /**< number of hcas available to the IB component */ struct mca_btl_openib_module_t *openib_btls; diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index afef8ae65e..a934474e7e 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -237,8 +237,7 @@ int mca_btl_openib_component_close(void) static int mca_btl_openib_modex_send(void) { - int rc; - size_t i; + int rc, i; size_t size; mca_btl_openib_port_info_t *ports = NULL; @@ -322,15 +321,16 @@ mca_btl_base_module_t** mca_btl_openib_component_init(int *num_btl_modules, bool enable_mpi_threads) { struct ibv_device **ib_devs; - uint32_t num_devs; mca_btl_base_module_t** btls; - uint32_t i,j, length; + int i,j, length, num_devs; struct mca_mpool_base_resources_t mpool_resources; opal_list_t btl_list; mca_btl_openib_module_t * openib_btl; mca_btl_base_selected_module_t* ib_selected; opal_list_item_t* item; +#if OMPI_MCA_BTL_OPENIB_HAVE_DEVICE_LIST == 0 struct dlist *dev_list; +#endif struct ibv_device* ib_dev; unsigned short seedv[3]; @@ -718,7 +718,7 @@ int mca_btl_openib_handle_incoming_hp( int mca_btl_openib_component_progress() { - uint32_t i, j, c; + int i, j, c; int count = 0,ne = 0, ret; int32_t credits; mca_btl_openib_frag_t* frag; diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.c b/ompi/mca/btl/openib/btl_openib_endpoint.c index 335fd44d5e..d8c0ae46c9 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.c +++ b/ompi/mca/btl/openib/btl_openib_endpoint.c @@ -1049,7 +1049,8 @@ static void mca_btl_openib_endpoint_credits_lp( mca_btl_openib_endpoint_send_credits_lp(endpoint); } } - MCA_BTL_IB_FRAG_RETURN_EAGER((mca_btl_openib_module_t*)btl, (mca_btl_openib_frag_t*)descriptor); + MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*)btl), + ((mca_btl_openib_frag_t*)descriptor)); } /** @@ -1095,7 +1096,7 @@ void mca_btl_openib_endpoint_send_credits_lp( &bad_wr)) { OPAL_THREAD_ADD32(&endpoint->sd_credits_lp, -1); OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, frag->hdr->credits); - MCA_BTL_IB_FRAG_RETURN_EAGER(openib_btl, frag); + MCA_BTL_IB_FRAG_RETURN(openib_btl, frag); BTL_ERROR(("error posting send request errno %d says %s", strerror(errno))); return; } @@ -1126,7 +1127,8 @@ static void mca_btl_openib_endpoint_credits_hp( mca_btl_openib_endpoint_send_credits_hp(endpoint); } } - MCA_BTL_IB_FRAG_RETURN_EAGER((mca_btl_openib_module_t*)btl, (mca_btl_openib_frag_t*)descriptor); + MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*)btl), + ((mca_btl_openib_frag_t*)descriptor)); } /** @@ -1176,7 +1178,7 @@ void mca_btl_openib_endpoint_send_credits_hp( &bad_wr)) { OPAL_THREAD_ADD32(&endpoint->sd_credits_hp, -1); OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, frag->hdr->credits); - MCA_BTL_IB_FRAG_RETURN_EAGER(openib_btl, frag); + MCA_BTL_IB_FRAG_RETURN(openib_btl, frag); BTL_ERROR(("error posting send request errno %d says %s", errno, strerror(errno))); return; @@ -1189,8 +1191,8 @@ static void mca_btl_openib_endpoint_eager_rdma( struct mca_btl_base_descriptor_t* descriptor, int status) { - MCA_BTL_IB_FRAG_RETURN_EAGER((mca_btl_openib_module_t*)btl, - (mca_btl_openib_frag_t*)descriptor); + MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*)btl), + ((mca_btl_openib_frag_t*)descriptor)); } static int mca_btl_openib_endpoint_send_eager_rdma( @@ -1220,7 +1222,7 @@ static int mca_btl_openib_endpoint_send_eager_rdma( frag->segment.seg_len = sizeof(mca_btl_openib_eager_rdma_header_t); if (mca_btl_openib_endpoint_post_send(openib_btl, endpoint, frag) != OMPI_SUCCESS) { - MCA_BTL_IB_FRAG_RETURN_EAGER(openib_btl, frag); + MCA_BTL_IB_FRAG_RETURN(openib_btl, frag); BTL_ERROR(("Error sending RDMA buffer", strerror(errno))); return -1; } diff --git a/ompi/mca/btl/openib/btl_openib_frag.h b/ompi/mca/btl/openib/btl_openib_frag.h index 4d4e5e251b..d1ad0272ec 100644 --- a/ompi/mca/btl/openib/btl_openib_frag.h +++ b/ompi/mca/btl/openib/btl_openib_frag.h @@ -135,12 +135,6 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_recv_frag_max_t); frag = (mca_btl_openib_frag_t*) item; \ } -#define MCA_BTL_IB_FRAG_RETURN_EAGER(btl, frag) \ -{ \ - OMPI_FREE_LIST_RETURN(&((mca_btl_openib_module_t*)btl)->send_free_eager, (opal_list_item_t*)(frag)); \ -} - - #define MCA_BTL_IB_FRAG_ALLOC_MAX(btl, frag, rc) \ { \ \ @@ -149,12 +143,6 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_recv_frag_max_t); frag = (mca_btl_openib_frag_t*) item; \ } -#define MCA_BTL_IB_FRAG_RETURN_MAX(btl, frag) \ -{ \ - OMPI_FREE_LIST_RETURN(&((mca_btl_openib_module_t*)btl)->send_free_max, (opal_list_item_t*)(frag)); \ -} - - #define MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc) \ { \ \ @@ -163,9 +151,22 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_recv_frag_max_t); frag = (mca_btl_openib_frag_t*) item; \ } -#define MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag) \ -{ \ - OMPI_FREE_LIST_RETURN(&((mca_btl_openib_module_t*)btl)->send_free_frag, (opal_list_item_t*)(frag)); \ +#define MCA_BTL_IB_FRAG_RETURN(btl, frag) \ +{ do { \ + ompi_free_list_t* my_list; \ + switch(frag->type) { \ + case MCA_BTL_OPENIB_FRAG_EAGER_RDMA: \ + case MCA_BTL_OPENIB_FRAG_EAGER: \ + my_list = &btl->send_free_eager; \ + break; \ + case MCA_BTL_OPENIB_FRAG_MAX: \ + my_list = &btl->send_free_max; \ + break; \ + case MCA_BTL_OPENIB_FRAG_FRAG: \ + my_list = &btl->send_free_frag; \ + } \ + OMPI_FREE_LIST_RETURN(my_list, (opal_list_item_t*)(frag)); \ + } while(0); \ }