From 1f3ac2d7ae927741fea0e11f3426ba06c1d19cb0 Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Wed, 14 Mar 2007 14:36:03 +0000 Subject: [PATCH] Hold pointers to free_max/free_eager lists in array indexed by priority. This eliminates couple of ifs from fast path. This commit was SVN r14031. --- ompi/mca/btl/openib/btl_openib.h | 11 ++----- ompi/mca/btl/openib/btl_openib_component.c | 22 +++++++------- ompi/mca/btl/openib/btl_openib_endpoint.h | 5 +--- ompi/mca/btl/openib/btl_openib_frag.h | 35 +++++++++++----------- 4 files changed, 31 insertions(+), 42 deletions(-) diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index 9b34318d8f..ff4009007f 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -198,12 +198,10 @@ struct mca_btl_openib_module_t { uint16_t lid; /**< lid that is actually used (for LMC) */ uint8_t src_path_bits; /**< offset from base lid (for LMC) */ - ompi_free_list_t send_free_eager; /**< free list of eager buffer descriptors */ - ompi_free_list_t send_free_max; /**< free list of max buffer descriptors */ + ompi_free_list_t send_free[2]; /**< free lists of send buffer descriptors */ ompi_free_list_t send_free_frag; /**< free list of frags only... used for pining memory */ - ompi_free_list_t recv_free_eager; /**< High priority free list of buffer descriptors */ - ompi_free_list_t recv_free_max; /**< Low priority free list of buffer descriptors */ + ompi_free_list_t recv_free[2]; /**< free lists of receive buffer descriptors */ ompi_free_list_t recv_free_frag; /**< free list of frags only... used for pining memory */ ompi_free_list_t send_free_control; /**< frags for control massages */ @@ -458,10 +456,7 @@ static inline int mca_btl_openib_post_srr(mca_btl_openib_module_t* openib_btl, struct ibv_recv_wr *bad_wr; ompi_free_list_t *free_list; - if(BTL_OPENIB_HP_QP == prio) - free_list = &openib_btl->recv_free_eager; - else - free_list = &openib_btl->recv_free_max; + free_list = &openib_btl->recv_free[prio]; for(i = 0; i < num_post; i++) { OMPI_FREE_LIST_WAIT(free_list, item, rc); diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index c354bcad57..c6bf6ce9f5 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -683,13 +683,13 @@ btl_openib_component_init(int *num_btl_modules, OBJ_CONSTRUCT(&openib_btl->pending_frags[BTL_OPENIB_LP_QP], opal_list_t); OBJ_CONSTRUCT(&openib_btl->ib_lock, opal_mutex_t); - OBJ_CONSTRUCT(&openib_btl->send_free_eager, ompi_free_list_t); - OBJ_CONSTRUCT(&openib_btl->send_free_max, ompi_free_list_t); + OBJ_CONSTRUCT(&openib_btl->send_free[BTL_OPENIB_HP_QP], ompi_free_list_t); + OBJ_CONSTRUCT(&openib_btl->send_free[BTL_OPENIB_LP_QP], ompi_free_list_t); OBJ_CONSTRUCT(&openib_btl->send_free_frag, ompi_free_list_t); OBJ_CONSTRUCT(&openib_btl->send_free_control, ompi_free_list_t); - OBJ_CONSTRUCT(&openib_btl->recv_free_eager, ompi_free_list_t); - OBJ_CONSTRUCT(&openib_btl->recv_free_max, ompi_free_list_t); + OBJ_CONSTRUCT(&openib_btl->recv_free[BTL_OPENIB_HP_QP], ompi_free_list_t); + OBJ_CONSTRUCT(&openib_btl->recv_free[BTL_OPENIB_LP_QP], ompi_free_list_t); OBJ_CONSTRUCT(&openib_btl->recv_free_frag, ompi_free_list_t); /* initialize the memory pool using the hca */ @@ -707,7 +707,7 @@ btl_openib_component_init(int *num_btl_modules, openib_btl->super.btl_eager_limit, mca_btl_openib_component.buffer_alignment, size_t); - ompi_free_list_init_ex(&openib_btl->send_free_eager, + ompi_free_list_init_ex(&openib_btl->send_free[BTL_OPENIB_HP_QP], length, mca_btl_openib_component.buffer_alignment, OBJ_CLASS(mca_btl_openib_send_frag_eager_t), @@ -721,7 +721,7 @@ btl_openib_component_init(int *num_btl_modules, sizeof(mca_btl_openib_footer_t) + openib_btl->super.btl_eager_limit; - ompi_free_list_init_ex(&openib_btl->recv_free_eager, + ompi_free_list_init_ex(&openib_btl->recv_free[BTL_OPENIB_HP_QP], length, mca_btl_openib_component.buffer_alignment, OBJ_CLASS(mca_btl_openib_recv_frag_eager_t), @@ -734,7 +734,7 @@ btl_openib_component_init(int *num_btl_modules, sizeof(mca_btl_openib_header_t) + openib_btl->super.btl_max_send_size; - ompi_free_list_init_ex(&openib_btl->send_free_max, + ompi_free_list_init_ex(&openib_btl->send_free[BTL_OPENIB_LP_QP], length, mca_btl_openib_component.buffer_alignment, OBJ_CLASS(mca_btl_openib_send_frag_max_t), @@ -748,7 +748,7 @@ btl_openib_component_init(int *num_btl_modules, openib_btl->super.btl_max_send_size; /* Initialize pool of receive fragments */ - ompi_free_list_init_ex(&openib_btl->recv_free_max, + ompi_free_list_init_ex(&openib_btl->recv_free[BTL_OPENIB_LP_QP], length, mca_btl_openib_component.buffer_alignment, OBJ_CLASS(mca_btl_openib_recv_frag_max_t), @@ -837,10 +837,8 @@ static int btl_openib_handle_incoming(mca_btl_openib_module_t *openib_btl, if(endpoint->nbo) { BTL_OPENIB_HEADER_NTOH((*(frag->hdr))); } - if(BTL_OPENIB_HP_QP == prio) - free_list = &openib_btl->recv_free_eager; - else - free_list = &openib_btl->recv_free_max; + + free_list = &openib_btl->recv_free[prio]; /* advance the segment address past the header and subtract from the * length..*/ diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.h b/ompi/mca/btl/openib/btl_openib_endpoint.h index 5b9a53e5ab..2678960f4e 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.h +++ b/ompi/mca/btl/openib/btl_openib_endpoint.h @@ -180,10 +180,7 @@ static inline int btl_openib_endpoint_post_rr(mca_btl_base_endpoint_t *endpoint, struct ibv_recv_wr* bad_wr; ompi_free_list_t *free_list; - if(BTL_OPENIB_HP_QP == prio) - free_list = &openib_btl->recv_free_eager; - else - free_list = &openib_btl->recv_free_max; + free_list = &openib_btl->recv_free[prio]; for(i = 0; i < num_post; i++) { ompi_free_list_item_t* item; diff --git a/ompi/mca/btl/openib/btl_openib_frag.h b/ompi/mca/btl/openib/btl_openib_frag.h index d5f3152376..da1db6de3b 100644 --- a/ompi/mca/btl/openib/btl_openib_frag.h +++ b/ompi/mca/btl/openib/btl_openib_frag.h @@ -219,21 +219,20 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_send_frag_control_t); frag = (mca_btl_openib_frag_t*) item; \ } -#define MCA_BTL_IB_FRAG_ALLOC_EAGER(btl, frag, rc) \ -{ \ - \ - ompi_free_list_item_t *item; \ - OMPI_FREE_LIST_GET(&((mca_btl_openib_module_t*)btl)->send_free_eager, item, rc); \ - frag = (mca_btl_openib_frag_t*) item; \ -} +#define MCA_BTL_IB_FRAG_ALLOC(btl, frag, rc, prio) \ + do { \ + ompi_free_list_item_t *item; \ + OMPI_FREE_LIST_GET( \ + &((mca_btl_openib_module_t*)btl)->send_free[prio], \ + item, rc); \ + frag = (mca_btl_openib_frag_t*)item; \ + } while (0) -#define MCA_BTL_IB_FRAG_ALLOC_MAX(btl, frag, rc) \ -{ \ - \ - ompi_free_list_item_t *item; \ - OMPI_FREE_LIST_GET(&((mca_btl_openib_module_t*)btl)->send_free_max, item, rc); \ - frag = (mca_btl_openib_frag_t*) item; \ -} +#define MCA_BTL_IB_FRAG_ALLOC_EAGER(btl, frag, rc) \ + MCA_BTL_IB_FRAG_ALLOC(btl, frag, rc, BTL_OPENIB_HP_QP) + +#define MCA_BTL_IB_FRAG_ALLOC_MAX(btl, frag, rc) \ + MCA_BTL_IB_FRAG_ALLOC(btl, frag, rc, BTL_OPENIB_LP_QP) #define MCA_BTL_IB_FRAG_ALLOC_SEND_FRAG(btl, frag, rc) \ { \ @@ -257,13 +256,13 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_send_frag_control_t); switch(frag->type) { \ case MCA_BTL_OPENIB_FRAG_EAGER_RDMA: \ case MCA_BTL_OPENIB_FRAG_EAGER: \ - my_list = &btl->send_free_eager; \ + my_list = &btl->send_free[BTL_OPENIB_HP_QP]; \ break; \ case MCA_BTL_OPENIB_FRAG_MAX: \ - my_list = &btl->send_free_max; \ + my_list = &btl->send_free[BTL_OPENIB_LP_QP]; \ break; \ - case MCA_BTL_OPENIB_FRAG_CONTROL: \ - my_list = &btl->send_free_control; \ + case MCA_BTL_OPENIB_FRAG_CONTROL: \ + my_list = &btl->send_free_control; \ break; \ case MCA_BTL_OPENIB_RECV_FRAG_FRAG: \ my_list = &btl->recv_free_frag; \