ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

We have QP description in component structure, module structure and endpoint.

Просмотр исходного кода 
Each one of them has a field to store QP type, but this is redundant.
Store qp type only in one structure (the component one).

This commit was SVN r16272.
Этот коммит содержится в:
Gleb Natapov 2007-09-30 16:14:17 +00:00
родитель 9c04b127f5
Коммит 60af46d541
5 изменённых файлов: 27 добавлений и 33 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
ompi/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -250,7 +250,7 @@ static int create_srq(mca_btl_openib_module_t *openib_btl)
/* create the SRQ's */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
struct ibv_srq_init_attr attr;
if(MCA_BTL_OPENIB_SRQ_QP == openib_btl->qps[qp].type) {
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
attr.attr.max_wr = mca_btl_openib_component.qp_infos[qp].rd_num +
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max;
attr.attr.max_sge = mca_btl_openib_component.ib_sg_list_size;
@ -328,7 +328,7 @@ static int mca_btl_openib_size_queues( struct mca_btl_openib_module_t* openib_bt
/* figure out reasonable sizes for completion queues */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
if(MCA_BTL_OPENIB_SRQ_QP == mca_btl_openib_component.qp_infos[qp].type) {
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
cq_size = mca_btl_openib_component.qp_infos[qp].rd_num +
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max;
if(mca_btl_openib_component.qp_infos[qp].size <=
@ -825,7 +825,7 @@ int mca_btl_openib_finalize(struct mca_btl_base_module_t* btl)
}
/* Release SRQ resources */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
if(MCA_BTL_OPENIB_SRQ_QP == mca_btl_openib_component.qp_infos[qp].type){
if(BTL_OPENIB_QP_TYPE_SRQ(qp)){
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(openib_btl,
&openib_btl->qps[qp].u.srq_qp.pending_frags);

12
ompi/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -72,16 +72,21 @@ struct mca_btl_openib_srq_qp_info_t {
}; typedef struct mca_btl_openib_srq_qp_info_t mca_btl_openib_srq_qp_info_t;
struct mca_btl_openib_qp_info_t {
mca_btl_openib_qp_type_t type;
size_t size;
int32_t rd_num;
int32_t rd_low;
mca_btl_openib_qp_type_t type;
union {
mca_btl_openib_pp_qp_info_t pp_qp;
mca_btl_openib_srq_qp_info_t srq_qp;
} u;
}; typedef struct mca_btl_openib_qp_info_t mca_btl_openib_qp_info_t;
#define BTL_OPENIB_QP_TYPE(Q) (mca_btl_openib_component.qp_infos[(Q)].type)
#define BTL_OPENIB_QP_TYPE_PP(Q) \
(BTL_OPENIB_QP_TYPE(Q) == MCA_BTL_OPENIB_PP_QP)
#define BTL_OPENIB_QP_TYPE_SRQ(Q) \
(BTL_OPENIB_QP_TYPE(Q) == MCA_BTL_OPENIB_SRQ_QP)
struct mca_btl_openib_component_t {
mca_btl_base_component_1_0_1_t super; /**< base BTL component */
@ -262,7 +267,6 @@ struct mca_btl_openib_module_srq_qp_t {
struct mca_btl_openib_module_qp_t {
ompi_free_list_t send_free; /**< free lists of send buffer descriptors */
ompi_free_list_t recv_free; /**< free lists of receive buffer descriptors */
mca_btl_openib_qp_type_t type;
union {
mca_btl_openib_module_pp_qp_t pp_qp;
mca_btl_openib_module_srq_qp_t srq_qp;
@ -536,7 +540,7 @@ static inline int mca_btl_openib_post_srr(mca_btl_openib_module_t* openib_btl,
const int additional,
const int qp)
{
assert(MCA_BTL_OPENIB_SRQ_QP == openib_btl->qps[qp].type);
assert(BTL_OPENIB_QP_TYPE_SRQ(qp));
OPAL_THREAD_LOCK(&openib_btl->ib_lock);
if(openib_btl->qps[qp].u.srq_qp.rd_posted <=
mca_btl_openib_component.qp_infos[qp].rd_low + additional &&
@ -579,7 +583,7 @@ static inline int mca_btl_openib_post_srr_all(mca_btl_openib_module_t *openib_bt
{
int qp;
for(qp = 0; qp < mca_btl_openib_component.num_srq_qps; qp++){
if(MCA_BTL_OPENIB_SRQ_QP == openib_btl->qps[qp].type) {
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
mca_btl_openib_post_srr(openib_btl, additional, qp);
}
}

20
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -1042,10 +1042,8 @@ btl_openib_component_init(int *num_btl_modules,
OBJ_CONSTRUCT(&openib_btl->qps[qp].send_free, ompi_free_list_t);
OBJ_CONSTRUCT(&openib_btl->qps[qp].recv_free, ompi_free_list_t);
openib_btl->qps[qp].type = mca_btl_openib_component.qp_infos[qp].type;
if(MCA_BTL_OPENIB_SRQ_QP == openib_btl->qps[qp].type) {
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
OBJ_CONSTRUCT(&openib_btl->qps[qp].u.srq_qp.pending_frags, opal_list_t);
openib_btl->qps[qp].u.srq_qp.sd_credits =
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max;
@ -1178,8 +1176,7 @@ static int btl_openib_handle_incoming(mca_btl_openib_module_t *openib_btl,
OPAL_THREAD_ADD32(&endpoint->eager_rdma_remote.tokens,
BTL_OPENIB_CREDITS(frag->hdr->credits));
} else {
if(MCA_BTL_OPENIB_PP_QP == endpoint->qps[qp].qp_type &&
frag->hdr->credits > 0) {
if(BTL_OPENIB_QP_TYPE_PP(qp) && frag->hdr->credits > 0) {
OPAL_THREAD_ADD32(&endpoint->qps[qp].u.pp_qp.sd_credits,
frag->hdr->credits);
}
@ -1192,8 +1189,7 @@ static int btl_openib_handle_incoming(mca_btl_openib_module_t *openib_btl,
/* We may receive credits here so try to progress only things that
* may be pending because of credit shortage */
if(MCA_BTL_OPENIB_PP_QP == endpoint->qps[qp].qp_type ||
BTL_OPENIB_EAGER_RDMA_QP(qp)) {
if(BTL_OPENIB_QP_TYPE_PP(qp) || BTL_OPENIB_EAGER_RDMA_QP(qp)) {
btl_openib_frag_progress_pending_pp(endpoint, qp);
}
@ -1276,8 +1272,7 @@ static char* btl_openib_component_status_to_string(enum ibv_wc_status status)
}
#define BTL_OPENIB_TOKENS(E, P) \
(((E)->qps[qp].qp_type == MCA_BTL_OPENIB_SRQ_QP) ? 1 : \
((E)->qps[(P)].u.pp_qp.sd_credits + \
(BTL_OPENIB_QP_TYPE_SRQ(P) ? 1 : ((E)->qps[(P)].u.pp_qp.sd_credits + \
((BTL_OPENIB_EAGER_RDMA_QP(P))?(E)->eager_rdma_remote.tokens:0)))
static void btl_openib_frag_progress_pending_pp(
mca_btl_base_endpoint_t *endpoint, const int qp)
@ -1344,7 +1339,7 @@ static void btl_openib_frag_progress_pending_srq(
mca_btl_openib_frag_t* frag;
size_t i, len;
assert(MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type);
assert(BTL_OPENIB_QP_TYPE_SRQ(qp));
len = opal_list_get_size(&openib_btl->qps[qp].u.srq_qp.pending_frags);
for(i = 0; i < len && openib_btl->qps[qp].u.srq_qp.sd_credits > 0; i++) {
@ -1553,8 +1548,7 @@ static int btl_openib_module_progress(mca_btl_openib_hca_t* hca)
/* return send wqe */
OPAL_THREAD_ADD32(&endpoint->qps[qp].sd_wqe, 1);
if(IBV_WC_SEND == wc.opcode &&
MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type) {
if(IBV_WC_SEND == wc.opcode && BTL_OPENIB_QP_TYPE_SRQ(qp)) {
OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.srq_qp.sd_credits, 1);
/* new SRQ credit available. Try to progress pending frags*/
btl_openib_frag_progress_pending_srq(openib_btl, endpoint, qp);
@ -1581,7 +1575,7 @@ static int btl_openib_module_progress(mca_btl_openib_hca_t* hca)
}
OMPI_FREE_LIST_RETURN(frag->list, (ompi_free_list_item_t*) frag);
if(MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type) {
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
OPAL_THREAD_ADD32((int32_t*)
&openib_btl->qps[qp].u.srq_qp.rd_posted, -1);
mca_btl_openib_post_srr(openib_btl, 0, qp);

15
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -94,7 +94,7 @@ static int post_send(mca_btl_openib_module_t *openib_btl,
sizeof(mca_btl_openib_footer_t);
frag->wr_desc.sr_desc.wr.rdma.remote_addr -= frag->sg_entry.length;
} else {
if(MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type) {
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
frag->wr_desc.sr_desc.opcode = IBV_WR_SEND_WITH_IMM;
frag->wr_desc.sr_desc.imm_data = endpoint->rem_info.rem_index;
} else {
@ -135,7 +135,7 @@ static int btl_openib_acquire_send_resources(
if(*do_rdma)
return OMPI_SUCCESS;
if(MCA_BTL_OPENIB_PP_QP == endpoint->qps[*qp].qp_type) {
if(BTL_OPENIB_QP_TYPE_PP(*qp)) {
if(OPAL_THREAD_ADD32(&endpoint->qps[*qp].u.pp_qp.sd_credits, -1) < 0) {
OPAL_THREAD_ADD32(&endpoint->qps[*qp].u.pp_qp.sd_credits, 1);
OPAL_THREAD_ADD32(&endpoint->qps[*qp].sd_wqe, 1);
@ -193,7 +193,7 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
if(frag->hdr->credits)
frag->hdr->credits |= BTL_OPENIB_RDMA_CREDITS_FLAG;
}
if(MCA_BTL_OPENIB_PP_QP == endpoint->qps[qp].qp_type &&
if(BTL_OPENIB_QP_TYPE_PP(qp) &&
0 == frag->hdr->credits) {
GET_CREDITS(endpoint->qps[qp].u.pp_qp.rd_credits, frag->hdr->credits);
}
@ -221,7 +221,7 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
if(do_rdma) {
OPAL_THREAD_ADD32(&endpoint->eager_rdma_remote.tokens, 1);
} else {
if(MCA_BTL_OPENIB_PP_QP == endpoint->qps[qp].qp_type) {
if(BTL_OPENIB_QP_TYPE_PP(qp)) {
OPAL_THREAD_ADD32(&endpoint->qps[qp].u.pp_qp.rd_credits, frag->hdr->credits);
OPAL_THREAD_ADD32(&endpoint->qps[qp].u.pp_qp.sd_credits, 1);
} else {
@ -249,9 +249,6 @@ OBJ_CLASS_INSTANCE(mca_btl_openib_endpoint_t,
static void mca_btl_openib_endpoint_construct_qp(mca_btl_base_endpoint_t *endpoint, int qp)
{
bool pp = (MCA_BTL_OPENIB_PP_QP ==
mca_btl_openib_component.qp_infos[qp].type);
endpoint->qps[qp].qp_type = mca_btl_openib_component.qp_infos[qp].type;
endpoint->qps[qp].lcl_qp = NULL;
endpoint->qps[qp].rd_credit_send_lock = 0;
/* setup rem_info */
@ -259,7 +256,7 @@ static void mca_btl_openib_endpoint_construct_qp(mca_btl_base_endpoint_t *endpoi
endpoint->rem_info.rem_qps[qp].rem_psn = 0;
OBJ_CONSTRUCT(&endpoint->qps[qp].pending_frags, opal_list_t);
if(pp) {
if(BTL_OPENIB_QP_TYPE_PP(qp)) {
/* local credits are set here such that on initial posting
* of the receive buffers we end up with zero credits to return
* to our peer. The peer initializes his sd_credits to reflect this
@ -408,7 +405,7 @@ int mca_btl_openib_endpoint_post_recvs(mca_btl_openib_endpoint_t *endpoint)
int qp;
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) {
if (MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type) {
if (BTL_OPENIB_QP_TYPE_SRQ(qp)) {
mca_btl_openib_post_srr(endpoint->endpoint_btl, 1, qp);
} else {
mca_btl_openib_endpoint_post_rr(endpoint, 1, qp);

7
ompi/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -116,7 +116,6 @@ struct mca_btl_openib_endpoint_qp_t {
struct ibv_qp* lcl_qp; /* Local QP (Low and High) */
uint32_t lcl_psn;
int32_t sd_wqe; /**< number of available send wqe entries */
int qp_type;
opal_list_t pending_frags; /**< put fragments here if there
is no wqe available or, in
case of PP QP, if there is
@ -218,7 +217,7 @@ static inline int mca_btl_openib_endpoint_post_rr(mca_btl_base_endpoint_t *endpo
int cm_received, rd_posted, rd_low;
assert(MCA_BTL_OPENIB_PP_QP == endpoint->qps[qp].qp_type);
assert(BTL_OPENIB_QP_TYPE_PP(qp));
OPAL_THREAD_LOCK(&openib_btl->ib_lock);
cm_received = endpoint->qps[qp].u.pp_qp.cm_received;
rd_posted = endpoint->qps[qp].u.pp_qp.rd_posted;
@ -270,7 +269,7 @@ static inline int mca_btl_openib_endpoint_post_rr_all(mca_btl_base_endpoint_t *e
{
int qp;
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++){
if(MCA_BTL_OPENIB_PP_QP == mca_btl_openib_component.qp_infos[qp].type) {
if(BTL_OPENIB_QP_TYPE_PP(qp)) {
mca_btl_openib_endpoint_post_rr(endpoint, additional, qp);
}
}
@ -291,7 +290,7 @@ static inline bool check_send_credits(mca_btl_openib_endpoint_t *endpoint,
}
}
if(MCA_BTL_OPENIB_PP_QP == mca_btl_openib_component.qp_infos[qp].type) {
if(BTL_OPENIB_QP_TYPE_PP(qp)) {
if(endpoint->qps[qp].u.pp_qp.rd_credits >=
mca_btl_openib_component.qp_infos[qp].u.pp_qp.rd_win) {
return true;