From c13240a1d1dcf2d60cd0b8a2807608c0b4e0035c Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Tue, 5 Sep 2006 16:02:09 +0000 Subject: [PATCH] remove rdma_credits from openib BTL header. Use one field for regular and rdma credits. This commit was SVN r11529. --- ompi/mca/btl/openib/btl_openib_component.c | 21 +++++-- ompi/mca/btl/openib/btl_openib_endpoint.c | 65 +++++++++++++++------- ompi/mca/btl/openib/btl_openib_frag.h | 14 ++++- 3 files changed, 70 insertions(+), 30 deletions(-) diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index 84aff826e1..ff9460d9be 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -197,11 +197,12 @@ static void btl_openib_control(struct mca_btl_base_module_t* btl, mca_btl_openib_endpoint_t* endpoint = frag->endpoint; mca_btl_openib_control_header_t *ctl_hdr = frag->segment.seg_addr.pval; mca_btl_openib_eager_rdma_header_t *rdma_hdr; + mca_btl_openib_rdma_credits_header_t *credits_hdr; if(frag->size == mca_btl_openib_component.eager_limit) { /* if not sent via rdma */ if(!MCA_BTL_OPENIB_RDMA_FRAG(frag) && - ctl_hdr->type == MCA_BTL_OPENIB_CONTROL_NOOP) { + ctl_hdr->type == MCA_BTL_OPENIB_CONTROL_CREDITS) { OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, -1); } } else { @@ -209,7 +210,11 @@ static void btl_openib_control(struct mca_btl_base_module_t* btl, } switch (ctl_hdr->type) { - case MCA_BTL_OPENIB_CONTROL_NOOP: + case MCA_BTL_OPENIB_CONTROL_CREDITS: + credits_hdr = (mca_btl_openib_rdma_credits_header_t*)ctl_hdr; + if(credits_hdr->rdma_credits) + OPAL_THREAD_ADD32(&endpoint->eager_rdma_remote.tokens, + credits_hdr->rdma_credits); break; case MCA_BTL_OPENIB_CONTROL_RDMA: rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)ctl_hdr; @@ -720,10 +725,14 @@ static int btl_openib_handle_incoming_hp(mca_btl_openib_module_t *openib_btl, } if (!mca_btl_openib_component.use_srq) { - OPAL_THREAD_ADD32(&endpoint->sd_tokens[BTL_OPENIB_HP_QP], - frag->hdr->credits); - OPAL_THREAD_ADD32(&endpoint->eager_rdma_remote.tokens, - frag->hdr->rdma_credits); + if(BTL_OPENIB_IS_RDMA_CREDITS(frag->hdr->credits) && + BTL_OPENIB_CREDITS(frag->hdr->credits) > 0) + OPAL_THREAD_ADD32(&endpoint->eager_rdma_remote.tokens, + BTL_OPENIB_CREDITS(frag->hdr->credits)); + else + if(frag->hdr->credits > 0) + OPAL_THREAD_ADD32(&endpoint->sd_tokens[BTL_OPENIB_HP_QP], + frag->hdr->credits); } if (mca_btl_openib_component.use_eager_rdma && diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.c b/ompi/mca/btl/openib/btl_openib_endpoint.c index 7ee1107f41..3e8d76f39d 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.c +++ b/ompi/mca/btl/openib/btl_openib_endpoint.c @@ -127,21 +127,29 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope BTL_OPENIB_HP_QP, &do_rdma) == OMPI_ERR_OUT_OF_RESOURCE) return MPI_SUCCESS; - frag->hdr->credits = - (endpoint->rd_credits_hp > 0) ? endpoint->rd_credits_hp : 0; - OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, -frag->hdr->credits); - frag->hdr->rdma_credits = endpoint->eager_rdma_local.credits; - OPAL_THREAD_ADD32(&endpoint->eager_rdma_local.credits, - -frag->hdr->rdma_credits); + if(endpoint->eager_rdma_local.credits > 0) { + frag->hdr->credits = endpoint->eager_rdma_local.credits; + OPAL_THREAD_ADD32(&endpoint->eager_rdma_local.credits, + -frag->hdr->credits); + frag->hdr->credits |= BTL_OPENIB_RDMA_CREDITS_FLAG; + } else if(endpoint->rd_credits_hp > 0) { + frag->hdr->credits = endpoint->rd_credits_hp; + OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, -frag->hdr->credits); + } else { + frag->hdr->credits = 0; + } ib_qp = endpoint->lcl_qp_hp; } else { if(btl_openib_acquire_send_resources(openib_btl, endpoint, frag, BTL_OPENIB_LP_QP, NULL) == OMPI_ERR_OUT_OF_RESOURCE) return MPI_SUCCESS; - frag->hdr->credits = (endpoint->rd_credits_lp > 0) ? - endpoint->rd_credits_lp : 0; - OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, -frag->hdr->credits); + if(endpoint->rd_credits_lp > 0) { + frag->hdr->credits = endpoint->rd_credits_lp; + OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, -frag->hdr->credits); + } else { + frag->hdr->credits = 0; + } ib_qp = endpoint->lcl_qp_lp; } @@ -1079,21 +1087,28 @@ void mca_btl_openib_endpoint_send_credits_lp( mca_btl_openib_module_t* openib_btl = endpoint->endpoint_btl; mca_btl_openib_frag_t* frag; struct ibv_send_wr* bad_wr; + mca_btl_openib_rdma_credits_header_t *credits_hdr; frag = endpoint->lp_credit_frag; + credits_hdr = (mca_btl_openib_rdma_credits_header_t*)frag->segment.seg_addr.pval; frag->base.des_cbfunc = mca_btl_openib_endpoint_credits_lp; frag->base.des_cbdata = NULL; frag->endpoint = endpoint; frag->hdr->tag = MCA_BTL_TAG_BTL; - frag->hdr->credits = endpoint->rd_credits_lp; - OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, -frag->hdr->credits); - ((mca_btl_openib_control_header_t *)frag->segment.seg_addr.pval)->type = MCA_BTL_OPENIB_CONTROL_NOOP; + if(endpoint->rd_credits_lp > 0) { + frag->hdr->credits = endpoint->rd_credits_lp; + OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, -frag->hdr->credits); + } else { + frag->hdr->credits = 0; + } + credits_hdr->control.type = MCA_BTL_OPENIB_CONTROL_CREDITS; + credits_hdr->rdma_credits = 0; frag->wr_desc.sr_desc.opcode = IBV_WR_SEND; frag->sg_entry.length = sizeof(mca_btl_openib_header_t) + - sizeof(mca_btl_openib_control_header_t); + sizeof(mca_btl_openib_rdma_credits_header_t); frag->sg_entry.addr = (unsigned long) frag->hdr; if(frag->sg_entry.length <= openib_btl->ib_inline_max) { @@ -1150,25 +1165,33 @@ void mca_btl_openib_endpoint_send_credits_hp( mca_btl_openib_module_t* openib_btl = endpoint->endpoint_btl; mca_btl_openib_frag_t* frag; struct ibv_send_wr* bad_wr; + mca_btl_openib_rdma_credits_header_t *credits_hdr; frag = endpoint->hp_credit_frag; + credits_hdr = (mca_btl_openib_rdma_credits_header_t*)frag->segment.seg_addr.pval; frag->base.des_cbfunc = mca_btl_openib_endpoint_credits_hp; frag->base.des_cbdata = NULL; frag->endpoint = endpoint; frag->hdr->tag = MCA_BTL_TAG_BTL; - frag->hdr->credits = - (endpoint->rd_credits_hp > 0) ? endpoint->rd_credits_hp : 0; - OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, -frag->hdr->credits); - frag->hdr->rdma_credits = endpoint->eager_rdma_local.credits; - OPAL_THREAD_ADD32(&endpoint->eager_rdma_local.credits, - -frag->hdr->rdma_credits); - ((mca_btl_openib_control_header_t *)frag->segment.seg_addr.pval)->type = MCA_BTL_OPENIB_CONTROL_NOOP; + if(endpoint->rd_credits_hp > 0) { + frag->hdr->credits = endpoint->rd_credits_hp; + OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, -frag->hdr->credits); + } else + frag->hdr->credits = 0; + if(endpoint->eager_rdma_local.credits > 0) { + credits_hdr->rdma_credits = endpoint->eager_rdma_local.credits; + OPAL_THREAD_ADD32(&endpoint->eager_rdma_local.credits, + -credits_hdr->rdma_credits); + } else + credits_hdr->rdma_credits = 0; + + credits_hdr->control.type = MCA_BTL_OPENIB_CONTROL_CREDITS; frag->wr_desc.sr_desc.opcode = IBV_WR_SEND; frag->sg_entry.length = sizeof(mca_btl_openib_header_t) + - sizeof(mca_btl_openib_control_header_t); + sizeof(mca_btl_openib_rdma_credits_header_t); frag->sg_entry.addr = (unsigned long) frag->hdr; if(frag->sg_entry.length <= openib_btl->ib_inline_max) { diff --git a/ompi/mca/btl/openib/btl_openib_frag.h b/ompi/mca/btl/openib/btl_openib_frag.h index cecfd894ae..ecf6cfe399 100644 --- a/ompi/mca/btl/openib/btl_openib_frag.h +++ b/ompi/mca/btl/openib/btl_openib_frag.h @@ -31,10 +31,12 @@ extern "C" { struct mca_btl_openib_header_t { mca_btl_base_tag_t tag; - int16_t credits; - int16_t rdma_credits; + uint16_t credits; }; typedef struct mca_btl_openib_header_t mca_btl_openib_header_t; +#define BTL_OPENIB_RDMA_CREDITS_FLAG (1<<15) +#define BTL_OPENIB_IS_RDMA_CREDITS(I) ((I)&BTL_OPENIB_RDMA_CREDITS_FLAG) +#define BTL_OPENIB_CREDITS(I) ((I)&~BTL_OPENIB_RDMA_CREDITS_FLAG) struct mca_btl_openib_footer_t { #if OMPI_ENABLE_DEBUG @@ -48,7 +50,7 @@ struct mca_btl_openib_footer_t { typedef struct mca_btl_openib_footer_t mca_btl_openib_footer_t; typedef enum { - MCA_BTL_OPENIB_CONTROL_NOOP, + MCA_BTL_OPENIB_CONTROL_CREDITS, MCA_BTL_OPENIB_CONTROL_RDMA } mca_btl_openib_control_t; @@ -64,6 +66,12 @@ struct mca_btl_openib_eager_rdma_header_t { }; typedef struct mca_btl_openib_eager_rdma_header_t mca_btl_openib_eager_rdma_header_t; +struct mca_btl_openib_rdma_credits_header_t { + mca_btl_openib_control_header_t control; + uint16_t rdma_credits; +}; +typedef struct mca_btl_openib_rdma_credits_header_t mca_btl_openib_rdma_credits_header_t; + enum mca_btl_openib_frag_type_t { MCA_BTL_OPENIB_FRAG_EAGER, MCA_BTL_OPENIB_FRAG_MAX,