ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Make xrc use srq_qp unions instead of the xrc_qp which is exactly like srq_qp.

Просмотр исходного кода 
This commit was SVN r16789.
Этот коммит содержится в:
Gleb Natapov 2007-11-28 07:20:26 +00:00
родитель be0981fc07
Коммит b46c9cc7bc
8 изменённых файлов: 55 добавлений и 183 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

62
ompi/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -277,39 +277,32 @@ static int create_srq(mca_btl_openib_module_t *openib_btl)
/* create the SRQ's */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
struct ibv_srq_init_attr attr;
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
if(!BTL_OPENIB_QP_TYPE_PP(qp)) {
attr.attr.max_wr = mca_btl_openib_component.qp_infos[qp].rd_num +
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max;
attr.attr.max_sge = mca_btl_openib_component.ib_sg_list_size;
openib_btl->qps[qp].u.srq_qp.rd_posted = 0;
openib_btl->qps[qp].u.srq_qp.srq =
ibv_create_srq(openib_btl->hca->ib_pd, &attr);
openib_btl->qps[qp].u.srq_qp.rd_posted = 0;
#if HAVE_XRC
if(BTL_OPENIB_QP_TYPE_XRC(qp)) {
int prio = qp_cq_prio(qp);
openib_btl->qps[qp].u.srq_qp.srq =
ibv_create_xrc_srq(openib_btl->hca->ib_pd,
openib_btl->hca->xrc_domain,
openib_btl->hca->ib_cq[prio], &attr);
} else
#endif
{
openib_btl->qps[qp].u.srq_qp.srq =
ibv_create_srq(openib_btl->hca->ib_pd, &attr);
}
if (NULL == openib_btl->qps[qp].u.srq_qp.srq) {
show_init_error(__FILE__, __LINE__, "ibv_create_srq",
ibv_get_device_name(openib_btl->hca->ib_dev));
return OMPI_ERROR;
}
}
#if HAVE_XRC
if(BTL_OPENIB_QP_TYPE_XRC(qp)) {
int prio = (mca_btl_openib_component.qp_infos[qp].size <=
mca_btl_openib_component.eager_limit) ?
BTL_OPENIB_HP_CQ : BTL_OPENIB_LP_CQ;
attr.attr.max_wr = mca_btl_openib_component.qp_infos[qp].rd_num +
mca_btl_openib_component.qp_infos[qp].u.xrc_qp.sd_max;
attr.attr.max_sge = mca_btl_openib_component.ib_sg_list_size;
openib_btl->qps[qp].u.xrc_qp.rd_posted = 0;
openib_btl->qps[qp].u.xrc_qp.xrc =
ibv_create_xrc_srq(openib_btl->hca->ib_pd,
openib_btl->hca->xrc_domain,
openib_btl->hca->ib_cq[prio],&attr);
if (NULL == openib_btl->qps[qp].u.xrc_qp.xrc) {
show_init_error(__FILE__, __LINE__, "ibv_create_srq",
ibv_get_device_name(openib_btl->hca->ib_dev));
return OMPI_ERROR;
}
}
#endif
}
return OMPI_SUCCESS;
@ -880,8 +873,7 @@ int mca_btl_openib_finalize(struct mca_btl_base_module_t* btl)
}
/* Release SRQ resources */
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
switch (BTL_OPENIB_QP_TYPE(qp)) {
case MCA_BTL_OPENIB_SRQ_QP:
if(!BTL_OPENIB_QP_TYPE_PP(qp)) {
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(
&openib_btl->qps[qp].u.srq_qp.pending_frags[0]);
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(
@ -893,24 +885,6 @@ int mca_btl_openib_finalize(struct mca_btl_base_module_t* btl)
OBJ_DESTRUCT(&openib_btl->qps[qp].u.srq_qp.pending_frags[0]);
OBJ_DESTRUCT(&openib_btl->qps[qp].u.srq_qp.pending_frags[1]);
break;
case MCA_BTL_OPENIB_XRC_QP:
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(
&openib_btl->qps[qp].u.xrc_qp.pending_frags[0]);
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(
&openib_btl->qps[qp].u.xrc_qp.pending_frags[1]);
if (ibv_destroy_srq(openib_btl->qps[qp].u.xrc_qp.xrc)) {
BTL_VERBOSE(("Failed to close SRQ %d", qp));
return OMPI_ERROR;
}
OBJ_DESTRUCT(&openib_btl->qps[qp].u.xrc_qp.pending_frags[0]);
OBJ_DESTRUCT(&openib_btl->qps[qp].u.xrc_qp.pending_frags[1]);
break;
case MCA_BTL_OPENIB_PP_QP:
/* Nothing to do */
break;
default:
BTL_VERBOSE(("Unknow qp type %d", qp));
break;
}
/* Destroy free lists */
OBJ_DESTRUCT(&openib_btl->qps[qp].send_free);

69
ompi/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -74,10 +74,6 @@ struct mca_btl_openib_srq_qp_info_t {
int32_t sd_max;
}; typedef struct mca_btl_openib_srq_qp_info_t mca_btl_openib_srq_qp_info_t;
struct mca_btl_openib_xrc_qp_info_t {
int32_t sd_max;
}; typedef struct mca_btl_openib_xrc_qp_info_t mca_btl_openib_xrc_qp_info_t;
struct mca_btl_openib_qp_info_t {
mca_btl_openib_qp_type_t type;
size_t size;
@ -86,7 +82,6 @@ struct mca_btl_openib_qp_info_t {
union {
mca_btl_openib_pp_qp_info_t pp_qp;
mca_btl_openib_srq_qp_info_t srq_qp;
mca_btl_openib_xrc_qp_info_t xrc_qp;
} u;
}; typedef struct mca_btl_openib_qp_info_t mca_btl_openib_qp_info_t;
@ -291,20 +286,12 @@ struct mca_btl_openib_module_srq_qp_t {
opal_list_t pending_frags[2]; /**< list of high/low prio frags */
}; typedef struct mca_btl_openib_module_srq_qp_t mca_btl_openib_module_srq_qp_t;
struct mca_btl_openib_module_xrc_qp_t {
struct ibv_srq *xrc;
int32_t rd_posted;
int32_t sd_credits;
opal_list_t pending_frags[2];
}; typedef struct mca_btl_openib_module_xrc_qp_t mca_btl_openib_module_xrc_qp_t;
struct mca_btl_openib_module_qp_t {
ompi_free_list_t send_free; /**< free lists of send buffer descriptors */
ompi_free_list_t recv_free; /**< free lists of receive buffer descriptors */
union {
mca_btl_openib_module_pp_qp_t pp_qp;
mca_btl_openib_module_srq_qp_t srq_qp;
mca_btl_openib_module_xrc_qp_t xrc_qp;
} u;
}; typedef struct mca_btl_openib_module_qp_t mca_btl_openib_module_qp_t;
@ -568,7 +555,8 @@ static inline int mca_btl_openib_post_srr(mca_btl_openib_module_t* openib_btl,
const int additional,
const int qp)
{
assert(BTL_OPENIB_QP_TYPE_SRQ(qp));
assert(!BTL_OPENIB_QP_TYPE_PP(qp));
OPAL_THREAD_LOCK(&openib_btl->ib_lock);
if(openib_btl->qps[qp].u.srq_qp.rd_posted <=
mca_btl_openib_component.qp_infos[qp].rd_low + additional &&
@ -603,53 +591,16 @@ static inline int mca_btl_openib_post_srr(mca_btl_openib_module_t* openib_btl,
return OMPI_SUCCESS;
}
/**
* Post to XRC with certain priority
*
* @param openib_btl (IN) BTL module
* @param additional (IN) Additional Bytes to reserve
* @param prio (IN) Priority (either BTL_OPENIB_HP_QP or BTL_OPENIB_LP_QP)
* @return OMPI_SUCCESS or failure status
*/
static inline int mca_btl_openib_post_xrr(mca_btl_openib_module_t* openib_btl,
const int additional,
const int qp)
static inline int qp_cq_prio(const int qp)
{
assert(BTL_OPENIB_QP_TYPE_XRC(qp));
if(0 == qp)
return BTL_OPENIB_HP_CQ; /* smallest qp is always HP */
OPAL_THREAD_LOCK(&openib_btl->ib_lock);
if(openib_btl->qps[qp].u.xrc_qp.rd_posted <=
mca_btl_openib_component.qp_infos[qp].rd_low + additional &&
openib_btl->qps[qp].u.xrc_qp.rd_posted <
mca_btl_openib_component.qp_infos[qp].rd_num) {
int rc;
int32_t i, num_post = mca_btl_openib_component.qp_infos[qp].rd_num -
openib_btl->qps[qp].u.xrc_qp.rd_posted;
struct ibv_recv_wr *bad_wr;
ompi_free_list_t *free_list;
free_list = &openib_btl->qps[qp].recv_free;
for(i = 0; i < num_post; i++) {
ompi_free_list_item_t* item;
OMPI_FREE_LIST_WAIT(free_list, item, rc);
to_base_frag(item)->base.order = qp;
to_com_frag(item)->endpoint = NULL;
if(ibv_post_srq_recv(openib_btl->qps[qp].u.xrc_qp.xrc,
&to_recv_frag(item)->rd_desc, &bad_wr)) {
BTL_ERROR(("error posting receive descriptors to shared "
"receive queue: %s", strerror(errno)));
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
return OMPI_ERROR;
}
}
OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.xrc_qp.rd_posted, num_post);
}
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
return OMPI_SUCCESS;
/* If the size for this qp is <= the eager limit, make it a
high priority QP. Otherwise, make it a low priority QP. */
return (mca_btl_openib_component.qp_infos[qp].size <=
mca_btl_openib_component.eager_limit) ?
BTL_OPENIB_HP_CQ : BTL_OPENIB_LP_CQ;
}
#define BTL_OPENIB_RDMA_QP(QP) \

45
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -806,7 +806,7 @@ static int finish_btl_init(mca_btl_openib_module_t *openib_btl)
OBJ_CONSTRUCT(&openib_btl->qps[qp].send_free, ompi_free_list_t);
OBJ_CONSTRUCT(&openib_btl->qps[qp].recv_free, ompi_free_list_t);
if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
if(!BTL_OPENIB_QP_TYPE_PP(qp)) {
OBJ_CONSTRUCT(&openib_btl->qps[qp].u.srq_qp.pending_frags[0],
opal_list_t);
OBJ_CONSTRUCT(&openib_btl->qps[qp].u.srq_qp.pending_frags[1],
@ -815,15 +815,6 @@ static int finish_btl_init(mca_btl_openib_module_t *openib_btl)
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max;
}
if(BTL_OPENIB_QP_TYPE_XRC(qp)) {
OBJ_CONSTRUCT(&openib_btl->qps[qp].u.xrc_qp.pending_frags[0],
opal_list_t);
OBJ_CONSTRUCT(&openib_btl->qps[qp].u.xrc_qp.pending_frags[1],
opal_list_t);
openib_btl->qps[qp].u.xrc_qp.sd_credits =
mca_btl_openib_component.qp_infos[qp].u.xrc_qp.sd_max;
}
init_data = malloc(sizeof(mca_btl_openib_frag_init_data_t));
/* Initialize pool of send fragments */
length = sizeof(mca_btl_openib_send_frag_t) +
@ -1192,20 +1183,16 @@ static int btl_openib_handle_incoming(mca_btl_openib_module_t *openib_btl,
OPAL_THREAD_UNLOCK(&erl->lock);
} else {
MCA_BTL_IB_FRAG_RETURN(frag);
if(BTL_OPENIB_QP_TYPE_SRQ(rqp)) {
mca_btl_openib_module_t *btl = ep->endpoint_btl;
OPAL_THREAD_ADD32(&btl->qps[rqp].u.srq_qp.rd_posted, -1);
mca_btl_openib_post_srr(btl, 0, rqp);
} else if(BTL_OPENIB_QP_TYPE_XRC(rqp)) {
mca_btl_openib_module_t *btl = ep->endpoint_btl;
OPAL_THREAD_ADD32(&btl->qps[rqp].u.xrc_qp.rd_posted, -1);
mca_btl_openib_post_xrr(openib_btl, 0, rqp);
} else { /* PP QP */
if(BTL_OPENIB_QP_TYPE_PP(rqp)) {
if(OPAL_UNLIKELY(is_credit_msg))
OPAL_THREAD_ADD32(&ep->qps[cqp].u.pp_qp.cm_received, 1);
else
OPAL_THREAD_ADD32(&ep->qps[rqp].u.pp_qp.rd_posted, -1);
mca_btl_openib_endpoint_post_rr(ep, cqp);
} else {
mca_btl_openib_module_t *btl = ep->endpoint_btl;
OPAL_THREAD_ADD32(&btl->qps[rqp].u.srq_qp.rd_posted, -1);
mca_btl_openib_post_srr(btl, 0, rqp);
}
}
@ -1409,19 +1396,10 @@ static void progress_pending_frags_srq(mca_btl_openib_module_t* openib_btl,
assert(BTL_OPENIB_QP_TYPE_SRQ(qp) || BTL_OPENIB_QP_TYPE_XRC(qp));
for(i = 0; i < 2; i++) {
opal_list_t *pending;
int32_t *sd_credits;
if (BTL_OPENIB_QP_TYPE_SRQ(qp)) {
pending = &openib_btl->qps[qp].u.srq_qp.pending_frags[i];
sd_credits = &openib_btl->qps[qp].u.srq_qp.sd_credits;
} else {
pending = &openib_btl->qps[qp].u.xrc_qp.pending_frags[i];
sd_credits = &openib_btl->qps[qp].u.xrc_qp.sd_credits;
}
while(*sd_credits > 0) {
while(openib_btl->qps[qp].u.srq_qp.sd_credits > 0) {
OPAL_THREAD_LOCK(&openib_btl->ib_lock);
frag = opal_list_remove_first(pending);
frag = opal_list_remove_first(
&openib_btl->qps[qp].u.srq_qp.pending_frags[i]);
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
if(NULL == frag)
@ -1618,10 +1596,7 @@ static int btl_openib_module_progress(mca_btl_openib_hca_t* hca)
qp_put_wqe(endpoint, qp);
if(IBV_WC_SEND == wc.opcode && !BTL_OPENIB_QP_TYPE_PP(qp)) {
int32_t *sd_credits = BTL_OPENIB_QP_TYPE_SRQ(qp) ?
&openib_btl->qps[qp].u.srq_qp.sd_credits :
&openib_btl->qps[qp].u.xrc_qp.sd_credits;
OPAL_THREAD_ADD32(sd_credits, 1);
OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.srq_qp.sd_credits, 1);
/* new SRQ credit available. Try to progress pending frags*/
progress_pending_frags_srq(openib_btl, qp);

25
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -155,7 +155,7 @@ static int acquire_send_credit(mca_btl_openib_endpoint_t *endpoint,
(opal_list_item_t *)frag);
return OMPI_ERR_OUT_OF_RESOURCE;
}
} else if(BTL_OPENIB_QP_TYPE_SRQ(qp)) {
} else {
if(OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.srq_qp.sd_credits, -1) < 0)
{
OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.srq_qp.sd_credits, 1);
@ -165,16 +165,6 @@ static int acquire_send_credit(mca_btl_openib_endpoint_t *endpoint,
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
} else { /* XRC QP */
if(OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.xrc_qp.sd_credits, -1) < 0)
{
OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.xrc_qp.sd_credits, 1);
OPAL_THREAD_LOCK(&openib_btl->ib_lock);
opal_list_append(&openib_btl->qps[qp].u.xrc_qp.pending_frags[prio],
(opal_list_item_t *)frag);
OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
}
return OMPI_SUCCESS;
@ -261,9 +251,6 @@ int mca_btl_openib_endpoint_post_send(mca_btl_openib_endpoint_t *endpoint,
} else if BTL_OPENIB_QP_TYPE_SRQ(qp){
mca_btl_openib_module_t *openib_btl = endpoint->endpoint_btl;
OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.srq_qp.sd_credits, 1);
} else { /* XRC QP */
mca_btl_openib_module_t *openib_btl = endpoint->endpoint_btl;
OPAL_THREAD_ADD32(&openib_btl->qps[qp].u.xrc_qp.sd_credits, 1);
}
}
BTL_ERROR(("error posting send request error %d: %s\n",
@ -343,7 +330,7 @@ endpoint_init_qp_xrc(mca_btl_openib_endpoint_qp_t *ep_qp, const int qp,
ep_qp->qp = endpoint_alloc_qp();
/* number of available send WQEs */
ep_qp->qp->sd_wqe =
mca_btl_openib_component.qp_infos[qp].u.xrc_qp.sd_max;
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max;
} else {
ep_qp->qp = xrc_qp;
}
@ -524,12 +511,10 @@ int mca_btl_openib_endpoint_post_recvs(mca_btl_openib_endpoint_t *endpoint)
int qp;
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) {
if (BTL_OPENIB_QP_TYPE_SRQ(qp)) {
mca_btl_openib_post_srr(endpoint->endpoint_btl, 1, qp);
} else if(BTL_OPENIB_QP_TYPE_XRC(qp)) {
mca_btl_openib_post_xrr(endpoint->endpoint_btl, 1, qp);
} else { /* PP QP */
if (BTL_OPENIB_QP_TYPE_PP(qp)) {
mca_btl_openib_endpoint_post_rr(endpoint, qp);
} else {
mca_btl_openib_post_srr(endpoint->endpoint_btl, 1, qp);
}
}

3
ompi/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -120,8 +120,6 @@ struct mca_btl_openib_endpoint_srq_qp_t {
int32_t dummy;
}; typedef struct mca_btl_openib_endpoint_srq_qp_t mca_btl_openib_endpoint_srq_qp_t;
typedef struct mca_btl_openib_endpoint_srq_qp_t mca_btl_openib_endpoint_xrc_qp_t;
typedef struct mca_btl_openib_qp_t {
struct ibv_qp *lcl_qp;
uint32_t lcl_psn;
@ -140,7 +138,6 @@ typedef struct mca_btl_openib_endpoint_qp_t {
mca_btl_openib_send_control_frag_t *credit_frag;
union {
mca_btl_openib_endpoint_srq_qp_t srq_qp;
mca_btl_openib_endpoint_xrc_qp_t xrc_qp;
mca_btl_openib_endpoint_pp_qp_t pp_qp;
} u;
} mca_btl_openib_endpoint_qp_t;

4
ompi/mca/btl/openib/btl_openib_mca.c
Просмотреть файл

@ -625,12 +625,12 @@ static int mca_btl_openib_mca_setup_qps(void)
tmp = mca_btl_openib_component.qp_infos[qp].rd_num >> 1;
mca_btl_openib_component.qp_infos[qp].rd_low = atoi_param(P(3), tmp);
tmp = mca_btl_openib_component.qp_infos[qp].rd_low >> 2;
mca_btl_openib_component.qp_infos[qp].u.xrc_qp.sd_max =
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max =
atoi_param(P(4), tmp);
BTL_VERBOSE(("xrc: rd_num is %d\trd_low is %d\tsd_max is %d\n",
mca_btl_openib_component.qp_infos[qp].rd_num,
mca_btl_openib_component.qp_infos[qp].rd_low,
mca_btl_openib_component.qp_infos[qp].u.xrc_qp.sd_max));
mca_btl_openib_component.qp_infos[qp].u.srq_qp.sd_max));
mca_btl_openib_component.qp_infos[qp].type = MCA_BTL_OPENIB_XRC_QP;
}

17
ompi/mca/btl/openib/connect/btl_openib_connect_oob.c
Просмотреть файл

@ -49,7 +49,7 @@ static int set_remote_info(mca_btl_base_endpoint_t* endpoint,
mca_btl_openib_rem_info_t* rem_info);
static int qp_connect_all(mca_btl_base_endpoint_t* endpoint);
static int qp_create_all(mca_btl_base_endpoint_t* endpoint);
static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int prio, int qp,
static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp,
struct ibv_srq *srq, uint32_t max_recv_wr, uint32_t max_send_wr);
static int send_connect_data(mca_btl_base_endpoint_t* endpoint,
uint8_t message_type);
@ -267,7 +267,7 @@ static int qp_connect_all(mca_btl_openib_endpoint_t *endpoint)
*/
static int qp_create_all(mca_btl_base_endpoint_t* endpoint)
{
int qp, rc, prio, pp_qp_num = 0;
int qp, rc, pp_qp_num = 0;
int32_t rd_rsv_total = 0;
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp)
@ -286,14 +286,6 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint)
struct ibv_srq *srq = NULL;
uint32_t max_recv_wr, max_send_wr;
int32_t rd_rsv, rd_num_credits;
/* If the size for this qp is <= the eager limit, make it a
high priority QP. Otherwise, make it a low priority QP. */
prio = (mca_btl_openib_component.qp_infos[qp].size <=
mca_btl_openib_component.eager_limit) ?
BTL_OPENIB_HP_CQ : BTL_OPENIB_LP_CQ;
if(qp == 0)
prio = BTL_OPENIB_HP_CQ; /* smallest qp is always HP */
/* QP used for SW flow control need some additional recourses */
if(qp == mca_btl_openib_component.credits_qp) {
@ -315,7 +307,7 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint)
+ rd_num_credits;
}
rc = qp_create_one(endpoint, prio, qp, srq, max_recv_wr, max_send_wr);
rc = qp_create_one(endpoint, qp, srq, max_recv_wr, max_send_wr);
if (OMPI_SUCCESS != rc) {
return rc;
}
@ -331,13 +323,14 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint)
* Create the local side of one qp. The remote side will be connected
* later.
*/
static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int prio, int qp,
static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp,
struct ibv_srq *srq, uint32_t max_recv_wr, uint32_t max_send_wr)
{
mca_btl_openib_module_t *openib_btl = endpoint->endpoint_btl;
struct ibv_qp *my_qp;
struct ibv_qp_init_attr init_attr;
struct ibv_qp_attr attr;
int prio = qp_cq_prio(qp);
memset(&init_attr, 0, sizeof(init_attr));
memset(&attr, 0, sizeof(attr));

13
ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c
Просмотреть файл

@ -248,7 +248,7 @@ static int xoob_qp_create(mca_btl_base_endpoint_t* endpoint, xoob_qp_type type)
mca_btl_openib_component.qp_infos->rd_num;
/* reserve additional wr for eager rdma credit management */
qp_init_attr.cap.max_send_wr =
mca_btl_openib_component.qp_infos->u.xrc_qp.sd_max +
mca_btl_openib_component.qp_infos->u.srq_qp.sd_max +
(mca_btl_openib_component.use_eager_rdma ?
mca_btl_openib_component.max_eager_rdma : 0);
@ -474,7 +474,7 @@ static int xoob_send_connect_data(mca_btl_base_endpoint_t* endpoint,
uint8_t message_type)
{
orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t);
int rc, srq, prio;
int rc, srq;
if (NULL == buffer) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
@ -590,18 +590,15 @@ static int xoob_send_connect_data(mca_btl_base_endpoint_t* endpoint,
}
/* on response we add all SRQ numbers */
for (srq = 0; srq < mca_btl_openib_component.num_xrc_qps; srq++) {
BTL_VERBOSE(("XOOB Send pack srq[%d] num = %d", srq, endpoint->endpoint_btl->qps[srq].u.xrc_qp.xrc->xrc_srq_num));
BTL_VERBOSE(("XOOB Send pack srq[%d] num = %d", srq, endpoint->endpoint_btl->qps[srq].u.src_qp.xrc->xrc_srq_num));
BTL_VERBOSE(("packing %d of %d\n", 1, ORTE_UINT32));
rc = orte_dss.pack(buffer, &endpoint->endpoint_btl->qps[srq].u.xrc_qp.xrc->xrc_srq_num,
rc = orte_dss.pack(buffer, &endpoint->endpoint_btl->qps[srq].u.srq_qp.srq->xrc_srq_num,
1, ORTE_UINT32);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
prio = (mca_btl_openib_component.qp_infos[srq].size <=
mca_btl_openib_component.eager_limit) ?
BTL_OPENIB_HP_CQ : BTL_OPENIB_LP_CQ;
endpoint->endpoint_btl->hca->cq_users[prio]++;
endpoint->endpoint_btl->hca->cq_users[qp_cq_prio(srq)]++;
}
}