add support for srq limit reached async event, even though it doesn't appear
to be supported by mellanox vapi.. perhaps this will be supported in the near future, for now it doesn't hurt to have it in the trunk Also cleanup the receive descriptor posting macro's.. This commit was SVN r7903.
Этот коммит содержится в:
Galen Shipman
родитель
3fd351117a
Коммит
4a15761732
@ -718,13 +718,19 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
|
||||
|
||||
case VAPI_SRQ_LIMIT_REACHED:
|
||||
{
|
||||
int i;
|
||||
BTL_ERROR(("SRQ limit is reached, posting more buffers %s\n", VAPI_event_record_sym(event_p->type)));
|
||||
for(i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++) {
|
||||
mca_btl_mvapi_module_t* mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i];
|
||||
|
||||
BTL_VERBOSE(("SRQ limit is reached, posting more buffers %s\n", VAPI_event_record_sym(event_p->type)));
|
||||
MCA_BTL_MVAPI_POST_SRR_HIGH(mvapi_btl, 1);
|
||||
MCA_BTL_MVAPI_POST_SRR_LOW(mvapi_btl, 1);
|
||||
}
|
||||
}
|
||||
|
||||
case VAPI_RECEIVE_QUEUE_DRAINED: {
|
||||
|
||||
|
||||
|
||||
}
|
||||
default:
|
||||
BTL_ERROR(("Warning!! Got an undefined "
|
||||
@ -744,8 +750,9 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
|
||||
/* Allocate Protection Domain */
|
||||
VAPI_ret_t ret;
|
||||
uint32_t cqe_cnt = 0;
|
||||
VAPI_srq_attr_t srq_attr, srq_attr_out;
|
||||
|
||||
VAPI_srq_attr_t srq_attr, srq_attr_out, srq_attr_mod;
|
||||
VAPI_srq_attr_mask_t srq_attr_mask;
|
||||
uint32_t max_outs_wr;
|
||||
ret = VAPI_alloc_pd(mvapi_btl->nic, &mvapi_btl->ptag);
|
||||
|
||||
if(ret != VAPI_OK) {
|
||||
@ -759,8 +766,8 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
|
||||
srq_attr.pd_hndl = mvapi_btl->ptag;
|
||||
srq_attr.max_outs_wr = mca_btl_mvapi_component.ib_wq_size;
|
||||
srq_attr.max_sentries = mca_btl_mvapi_component.ib_sg_list_size;
|
||||
srq_attr.srq_limit = mca_btl_mvapi_component.ib_wq_size;
|
||||
|
||||
srq_attr_mod.srq_limit = 16;/* mca_btl_mvapi_component.ib_wq_size; */
|
||||
ret = VAPI_create_srq(mvapi_btl->nic,
|
||||
&srq_attr,
|
||||
&mvapi_btl->srq_hndl_high,
|
||||
@ -769,6 +776,24 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
|
||||
BTL_ERROR(("error in VAPI_create_srq: %s", VAPI_strerror(ret)));
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
srq_attr_mask = 0;
|
||||
srq_attr_mask |= VAPI_SRQ_ATTR_LIMIT;
|
||||
|
||||
ret = VAPI_modify_srq
|
||||
(
|
||||
mvapi_btl->nic,
|
||||
mvapi_btl->srq_hndl_high,
|
||||
&srq_attr_mod,
|
||||
srq_attr_mask,
|
||||
&max_outs_wr
|
||||
);
|
||||
|
||||
if(ret != VAPI_OK) {
|
||||
/* BTL_ERROR(("error in VAPI_modify_srq: %s", VAPI_strerror(ret))); */
|
||||
/* return OMPI_ERROR; */
|
||||
}
|
||||
|
||||
ret = VAPI_create_srq(mvapi_btl->nic,
|
||||
&srq_attr,
|
||||
&mvapi_btl->srq_hndl_low,
|
||||
@ -778,6 +803,24 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
srq_attr_mask = 0;
|
||||
srq_attr_mask |= VAPI_SRQ_ATTR_LIMIT;
|
||||
|
||||
ret = VAPI_modify_srq
|
||||
(
|
||||
mvapi_btl->nic,
|
||||
mvapi_btl->srq_hndl_low,
|
||||
&srq_attr_mod,
|
||||
srq_attr_mask,
|
||||
&max_outs_wr
|
||||
);
|
||||
|
||||
if(ret != VAPI_OK) {
|
||||
/* BTL_ERROR(("error in VAPI_modify_srq: %s", VAPI_strerror(ret))); */
|
||||
/* return OMPI_ERROR; */
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
mvapi_btl->srq_hndl_high = VAPI_INVAL_SRQ_HNDL;
|
||||
mvapi_btl->srq_hndl_low = VAPI_INVAL_SRQ_HNDL;
|
||||
|
||||
@ -165,75 +165,81 @@ int mca_btl_mvapi_endpoint_connect(mca_btl_base_endpoint_t*);
|
||||
void mca_btl_mvapi_post_recv(void);
|
||||
|
||||
|
||||
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(post_rr_high_endpoint, \
|
||||
post_rr_high_additional) \
|
||||
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(endpoint, \
|
||||
additional) \
|
||||
{ \
|
||||
mca_btl_mvapi_module_t * post_rr_high_mvapi_btl = post_rr_high_endpoint->endpoint_btl; \
|
||||
OPAL_THREAD_LOCK(&post_rr_high_mvapi_btl->ib_lock); \
|
||||
if(post_rr_high_endpoint->rr_posted_high <= mca_btl_mvapi_component.ib_rr_buf_min+post_rr_high_additional && \
|
||||
post_rr_high_endpoint->rr_posted_high < post_rr_high_mvapi_btl->rd_buf_max){ \
|
||||
MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(post_rr_high_mvapi_btl->rd_buf_max - \
|
||||
post_rr_high_endpoint->rr_posted_high, \
|
||||
post_rr_high_endpoint, \
|
||||
&post_rr_high_mvapi_btl->recv_free_eager, \
|
||||
&post_rr_high_endpoint->rr_posted_high, \
|
||||
post_rr_high_mvapi_btl->nic, \
|
||||
post_rr_high_endpoint->lcl_qp_hndl_high); \
|
||||
do { \
|
||||
mca_btl_mvapi_module_t * mvapi_btl = endpoint->endpoint_btl; \
|
||||
OPAL_THREAD_LOCK(&mvapi_btl->ib_lock); \
|
||||
if(endpoint->rr_posted_high <= mca_btl_mvapi_component.ib_rr_buf_min+additional && \
|
||||
endpoint->rr_posted_high < mvapi_btl->rd_buf_max){ \
|
||||
MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(mvapi_btl->rd_buf_max - \
|
||||
endpoint->rr_posted_high, \
|
||||
endpoint, \
|
||||
&mvapi_btl->recv_free_eager, \
|
||||
&endpoint->rr_posted_high, \
|
||||
mvapi_btl->nic, \
|
||||
endpoint->lcl_qp_hndl_high); \
|
||||
} \
|
||||
OPAL_THREAD_UNLOCK(&post_rr_high_mvapi_btl->ib_lock); \
|
||||
OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock); \
|
||||
} while(0); \
|
||||
}
|
||||
|
||||
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(post_rr_low_endpoint, \
|
||||
post_rr_low_additional) \
|
||||
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(endpoint, \
|
||||
additional) \
|
||||
{ \
|
||||
mca_btl_mvapi_module_t * post_rr_low_mvapi_btl = post_rr_low_endpoint->endpoint_btl; \
|
||||
OPAL_THREAD_LOCK(&post_rr_low_mvapi_btl->ib_lock); \
|
||||
if(post_rr_low_endpoint->rr_posted_low <= mca_btl_mvapi_component.ib_rr_buf_min+post_rr_low_additional && \
|
||||
post_rr_low_endpoint->rr_posted_low < post_rr_low_mvapi_btl->rd_buf_max){ \
|
||||
MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(post_rr_low_mvapi_btl->rd_buf_max - \
|
||||
post_rr_low_endpoint->rr_posted_low, \
|
||||
post_rr_low_endpoint, \
|
||||
&post_rr_low_mvapi_btl->recv_free_max, \
|
||||
&post_rr_low_endpoint->rr_posted_low, \
|
||||
post_rr_low_mvapi_btl->nic, \
|
||||
post_rr_low_endpoint->lcl_qp_hndl_low); \
|
||||
do { \
|
||||
mca_btl_mvapi_module_t * mvapi_btl = endpoint->endpoint_btl; \
|
||||
OPAL_THREAD_LOCK(&mvapi_btl->ib_lock); \
|
||||
if(endpoint->rr_posted_low <= mca_btl_mvapi_component.ib_rr_buf_min+additional && \
|
||||
endpoint->rr_posted_low < mvapi_btl->rd_buf_max){ \
|
||||
MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(mvapi_btl->rd_buf_max - \
|
||||
endpoint->rr_posted_low, \
|
||||
endpoint, \
|
||||
&mvapi_btl->recv_free_max, \
|
||||
&endpoint->rr_posted_low, \
|
||||
mvapi_btl->nic, \
|
||||
endpoint->lcl_qp_hndl_low); \
|
||||
} \
|
||||
OPAL_THREAD_UNLOCK(&post_rr_low_mvapi_btl->ib_lock); \
|
||||
OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock); \
|
||||
} while(0); \
|
||||
}
|
||||
|
||||
|
||||
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(post_rr_sub_cnt, \
|
||||
post_rr_sub_endpoint, \
|
||||
post_rr_sub_frag_list, \
|
||||
post_rr_sub_rr_posted, \
|
||||
post_rr_sub_nic, \
|
||||
post_rr_sub_qp ) \
|
||||
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(cnt, \
|
||||
my_endpoint, \
|
||||
frag_list, \
|
||||
rr_posted, \
|
||||
nic, \
|
||||
qp ) \
|
||||
{\
|
||||
uint32_t post_rr_sub_i; \
|
||||
int post_rr_sub_rc; \
|
||||
opal_list_item_t* post_rr_sub_item; \
|
||||
mca_btl_mvapi_frag_t* post_rr_sub_frag = NULL; \
|
||||
mca_btl_mvapi_module_t *post_rr_sub_mvapi_btl = post_rr_sub_endpoint->endpoint_btl; \
|
||||
VAPI_rr_desc_t* post_rr_sub_desc_post = post_rr_sub_mvapi_btl->rr_desc_post; \
|
||||
for(post_rr_sub_i = 0; post_rr_sub_i < post_rr_sub_cnt; post_rr_sub_i++) { \
|
||||
OMPI_FREE_LIST_WAIT(post_rr_sub_frag_list, post_rr_sub_item, post_rr_sub_rc); \
|
||||
post_rr_sub_frag = (mca_btl_mvapi_frag_t*) post_rr_sub_item; \
|
||||
post_rr_sub_frag->endpoint = post_rr_sub_endpoint; \
|
||||
post_rr_sub_frag->sg_entry.len = post_rr_sub_frag->size + \
|
||||
((unsigned char*) post_rr_sub_frag->segment.seg_addr.pval- \
|
||||
(unsigned char*) post_rr_sub_frag->hdr); \
|
||||
post_rr_sub_desc_post[post_rr_sub_i] = post_rr_sub_frag->rr_desc; \
|
||||
do { \
|
||||
uint32_t i; \
|
||||
int rc; \
|
||||
opal_list_item_t* item; \
|
||||
mca_btl_mvapi_frag_t* frag = NULL; \
|
||||
mca_btl_mvapi_module_t *mvapi_btl = my_endpoint->endpoint_btl; \
|
||||
VAPI_rr_desc_t* desc_post = mvapi_btl->rr_desc_post; \
|
||||
for(i = 0; i < cnt; i++) { \
|
||||
OMPI_FREE_LIST_WAIT(frag_list, item, rc); \
|
||||
frag = (mca_btl_mvapi_frag_t*) item; \
|
||||
frag->endpoint = my_endpoint; \
|
||||
frag->sg_entry.len = frag->size + \
|
||||
((unsigned char*) frag->segment.seg_addr.pval- \
|
||||
(unsigned char*) frag->hdr); \
|
||||
desc_post[i] = frag->rr_desc; \
|
||||
}\
|
||||
post_rr_sub_frag->ret = EVAPI_post_rr_list( post_rr_sub_nic, \
|
||||
post_rr_sub_qp, \
|
||||
post_rr_sub_cnt, \
|
||||
post_rr_sub_desc_post); \
|
||||
if(NULL != post_rr_sub_frag && VAPI_OK != post_rr_sub_frag->ret) { \
|
||||
frag->ret = EVAPI_post_rr_list( nic, \
|
||||
qp, \
|
||||
cnt, \
|
||||
desc_post); \
|
||||
if(NULL != frag && VAPI_OK != frag->ret) { \
|
||||
BTL_ERROR(("error posting receive descriptors: %s",\
|
||||
VAPI_strerror(post_rr_sub_frag->ret))); \
|
||||
} else if (NULL != post_rr_sub_frag){\
|
||||
OPAL_THREAD_ADD32(post_rr_sub_rr_posted, post_rr_sub_cnt); \
|
||||
VAPI_strerror(frag->ret))); \
|
||||
} else if (NULL != frag){\
|
||||
OPAL_THREAD_ADD32(rr_posted, cnt); \
|
||||
}\
|
||||
} while(0); \
|
||||
}
|
||||
|
||||
#define BTL_MVAPI_INSERT_PENDING(frag, frag_list, tokens, lock, rc) \
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user