ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Changed send tokens to both send/rdma tokens for both low and high priority

Просмотр исходного кода 
queue pairs. Tested on intel p2p with 16 procs - Passed. 

This commit was SVN r7119.
Этот коммит содержится в:
Galen Shipman 2005-09-01 02:41:44 +00:00
родитель 96f4bb7a63
Коммит 29f7b4deda
4 изменённых файлов: 76 добавлений и 40 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
ompi/mca/btl/mvapi/btl_mvapi.h
Просмотреть файл

@ -118,7 +118,7 @@ struct mca_btl_mvapi_component_t {
uint32_t ib_static_rate; uint32_t ib_static_rate;
uint32_t ib_src_path_bits; uint32_t ib_src_path_bits;
/* number of send tokes available */ /* number of send tokes available */
uint32_t max_send_tokens; uint32_t max_wr_sq_tokens;
}; typedef struct mca_btl_mvapi_component_t mca_btl_mvapi_component_t; }; typedef struct mca_btl_mvapi_component_t mca_btl_mvapi_component_t;

29
ompi/mca/btl/mvapi/btl_mvapi_component.c
Просмотреть файл

@ -208,12 +208,12 @@ int mca_btl_mvapi_component_open(void)
mca_base_param_lookup_int(param, &value); mca_base_param_lookup_int(param, &value);
mca_btl_mvapi_component.leave_pinned = value; mca_btl_mvapi_component.leave_pinned = value;
mca_base_param_reg_int(&mca_btl_mvapi_component.super.btl_version, mca_base_param_reg_int(&mca_btl_mvapi_component.super.btl_version,
"max_send_tokens", "max_wr_sq_tokens",
"Maximum number of send tokens", "Maximum number of send/rdma work request tokens",
false, false,
false, false,
16, 16,
&(mca_btl_mvapi_component.max_send_tokens)); &(mca_btl_mvapi_component.max_wr_sq_tokens));
mca_btl_mvapi_component.max_send_size = mca_btl_mvapi_module.super.btl_max_send_size; mca_btl_mvapi_component.max_send_size = mca_btl_mvapi_module.super.btl_max_send_size;
mca_btl_mvapi_component.eager_limit = mca_btl_mvapi_module.super.btl_eager_limit; mca_btl_mvapi_component.eager_limit = mca_btl_mvapi_module.super.btl_eager_limit;
@ -551,21 +551,18 @@ int mca_btl_mvapi_component_progress()
return OMPI_ERROR; return OMPI_ERROR;
case VAPI_CQE_SQ_SEND_DATA : case VAPI_CQE_SQ_SEND_DATA :
frag = (mca_btl_mvapi_frag_t*) comp.id;
frag->endpoint->send_tokens++;
/* fall through */
case VAPI_CQE_SQ_RDMA_READ: case VAPI_CQE_SQ_RDMA_READ:
case VAPI_CQE_SQ_RDMA_WRITE: case VAPI_CQE_SQ_RDMA_WRITE:
frag = (mca_btl_mvapi_frag_t*) comp.id;
OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_hp, 1);
/* Process a completed send or an rdma write */ /* Process a completed send or an rdma write */
frag->rc = OMPI_SUCCESS; frag->rc = OMPI_SUCCESS;
frag->base.des_cbfunc(&mvapi_btl->super, frag->endpoint, &frag->base, frag->rc); frag->base.des_cbfunc(&mvapi_btl->super, frag->endpoint, &frag->base, frag->rc);
count++; count++;
/* check and see if we need to progress pending sends */ /* check and see if we need to progress pending sends */
if(frag->endpoint->send_tokens && !opal_list_is_empty(&(frag->endpoint->pending_send_frags))) { if(frag->endpoint->wr_sq_tokens_hp && !opal_list_is_empty(&(frag->endpoint->pending_frags_hp))) {
opal_list_item_t *frag_item; opal_list_item_t *frag_item;
frag_item = opal_list_remove_first(&(frag->endpoint->pending_send_frags)); frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_hp));
frag = (mca_btl_mvapi_frag_t *) frag_item; frag = (mca_btl_mvapi_frag_t *) frag_item;
if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_send(frag->endpoint, frag)) { if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_send(frag->endpoint, frag)) {
@ -627,9 +624,21 @@ int mca_btl_mvapi_component_progress()
/* Process a completed send */ /* Process a completed send */
frag = (mca_btl_mvapi_frag_t*) comp.id; frag = (mca_btl_mvapi_frag_t*) comp.id;
OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_lp, 1);
frag->rc = OMPI_SUCCESS; frag->rc = OMPI_SUCCESS;
frag->base.des_cbfunc(&mvapi_btl->super, frag->endpoint, &frag->base, frag->rc); frag->base.des_cbfunc(&mvapi_btl->super, frag->endpoint, &frag->base, frag->rc);
count++; count++;
/* check and see if we need to progress pending sends */
if(frag->endpoint->wr_sq_tokens_lp && !opal_list_is_empty(&(frag->endpoint->pending_frags_lp))) {
opal_list_item_t *frag_item;
frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_lp));
frag = (mca_btl_mvapi_frag_t *) frag_item;
if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_send(frag->endpoint, frag)) {
BTL_ERROR(("error in posting pending send\n"));
}
}
break; break;
case VAPI_CQE_RQ_SEND_DATA: case VAPI_CQE_RQ_SEND_DATA:

69
ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
Просмотреть файл

@ -64,11 +64,35 @@ static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi
VAPI_qp_hndl_t qp_hndl; VAPI_qp_hndl_t qp_hndl;
if(frag->base.des_flags & MCA_BTL_DES_FLAGS_PRIORITY && frag->size <= mvapi_btl->super.btl_eager_limit){ if(frag->base.des_flags & MCA_BTL_DES_FLAGS_PRIORITY && frag->size <= mvapi_btl->super.btl_eager_limit){
frag->sr_desc.remote_qp = endpoint->rem_qp_num_high;
qp_hndl = endpoint->lcl_qp_hndl_high; if(0 == endpoint->wr_sq_tokens_hp) {
BTL_VERBOSE(("Queing because no send tokens \n"));
opal_list_append(&endpoint->pending_frags_hp,
(opal_list_item_t *)frag);
return OMPI_SUCCESS;
} else {
frag->sr_desc.remote_qp = endpoint->rem_qp_num_high;
qp_hndl = endpoint->lcl_qp_hndl_high;
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_hp, -1);
}
} else { } else {
frag->sr_desc.remote_qp = endpoint->rem_qp_num_low; if(0 == endpoint->wr_sq_tokens_lp) {
qp_hndl = endpoint->lcl_qp_hndl_low; BTL_VERBOSE(("Queing because no send tokens \n"));
opal_list_append(&endpoint->pending_frags_lp,
(opal_list_item_t *)frag);
return OMPI_SUCCESS;
} else {
frag->sr_desc.remote_qp = endpoint->rem_qp_num_low;
qp_hndl = endpoint->lcl_qp_hndl_low;
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp, -1);
}
} }
frag->sr_desc.opcode = VAPI_SEND; frag->sr_desc.opcode = VAPI_SEND;
frag->sg_entry.len = frag->segment.seg_len + ((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr); /* sizeof(mca_btl_mvapi_header_t); */ frag->sg_entry.len = frag->segment.seg_len + ((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr); /* sizeof(mca_btl_mvapi_header_t); */
@ -120,10 +144,15 @@ static void mca_btl_mvapi_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
OBJ_CONSTRUCT(&endpoint->endpoint_send_lock, opal_mutex_t); OBJ_CONSTRUCT(&endpoint->endpoint_send_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_recv_lock, opal_mutex_t); OBJ_CONSTRUCT(&endpoint->endpoint_recv_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->pending_send_frags, opal_list_t); OBJ_CONSTRUCT(&endpoint->pending_send_frags, opal_list_t);
OBJ_CONSTRUCT(&endpoint->pending_frags_hp, opal_list_t);
OBJ_CONSTRUCT(&endpoint->pending_frags_lp, opal_list_t);
endpoint->rr_posted_high = 0; endpoint->rr_posted_high = 0;
endpoint->rr_posted_low = 0; endpoint->rr_posted_low = 0;
endpoint->send_tokens = mca_btl_mvapi_component.max_send_tokens; /* initialize the high and low priority tokens */
endpoint->wr_sq_tokens_hp = mca_btl_mvapi_component.max_wr_sq_tokens;
endpoint->wr_sq_tokens_lp = mca_btl_mvapi_component.max_wr_sq_tokens;
} }
@ -569,26 +598,14 @@ int mca_btl_mvapi_endpoint_send(
mvapi_btl = endpoint->endpoint_btl; mvapi_btl = endpoint->endpoint_btl;
if(0 == endpoint->send_tokens) { BTL_VERBOSE(("Send to : %d, len : %d, frag : %p",
BTL_VERBOSE(("Queing because no send tokens \n")); endpoint->endpoint_proc->proc_guid.vpid,
frag->sg_entry.len,
opal_list_append(&endpoint->pending_send_frags, frag));
(opal_list_item_t *)frag);
rc = OMPI_SUCCESS;
} else {
endpoint->send_tokens--; rc = mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag);
}
BTL_VERBOSE(("Send to : %d, len : %d, frag : %p", break;
endpoint->endpoint_proc->proc_guid.vpid,
frag->sg_entry.len,
frag));
rc = mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag);
}
break;
}
default: default:
rc = OMPI_ERR_UNREACH; rc = OMPI_ERR_UNREACH;
@ -690,7 +707,7 @@ int mca_btl_mvapi_endpoint_create_qp(
case VAPI_TS_RC: /* Set up RC qp parameters */ case VAPI_TS_RC: /* Set up RC qp parameters */
qp_init_attr.cap.max_oust_wr_rq = mca_btl_mvapi_component.ib_rr_buf_max; qp_init_attr.cap.max_oust_wr_rq = mca_btl_mvapi_component.ib_rr_buf_max;
qp_init_attr.cap.max_oust_wr_sq = mca_btl_mvapi_component.max_send_tokens; qp_init_attr.cap.max_oust_wr_sq = mca_btl_mvapi_component.max_wr_sq_tokens;
qp_init_attr.cap.max_sg_size_rq = mca_btl_mvapi_component.ib_sg_list_size; qp_init_attr.cap.max_sg_size_rq = mca_btl_mvapi_component.ib_sg_list_size;
qp_init_attr.cap.max_sg_size_sq = mca_btl_mvapi_component.ib_sg_list_size; qp_init_attr.cap.max_sg_size_sq = mca_btl_mvapi_component.ib_sg_list_size;
qp_init_attr.pd_hndl = ptag; qp_init_attr.pd_hndl = ptag;

16
ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
Просмотреть файл

@ -105,9 +105,19 @@ struct mca_btl_base_endpoint_t {
opal_list_t pending_send_frags; opal_list_t pending_send_frags;
/**< list of pending send frags for this endpoint */ /**< list of pending send frags for this endpoint */
uint32_t send_tokens; opal_list_t pending_frags_hp;
/**< number of sends that can be outstanding (down counter) */ /**< list of pending high priority frags */
opal_list_t pending_frags_lp;
/**< list of pending low prioirty frags */
uint32_t wr_sq_tokens_hp;
/**< number of high priority frags that can be outstanding (down counter) */
uint32_t wr_sq_tokens_lp;
/**< number of low priority frags that can be outstanding (down counter) */
VAPI_qp_num_t rem_qp_num_high; VAPI_qp_num_t rem_qp_num_high;
/* High priority remote side QP number */ /* High priority remote side QP number */