ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

add endpoint and srq flow-control..

Просмотр исходного кода 
Note, we are failing the ring tests in the intel p2p test suite, but we seem
to fail the same tests under the current trunk.. will look into this further. 

This commit was SVN r7823.
Этот коммит содержится в:
Galen Shipman 2005-10-21 02:21:45 +00:00
родитель 88c7fd9f8d
Коммит cb84a57c57
5 изменённых файлов: 278 добавлений и 71 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

135
ompi/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -123,6 +123,10 @@ int mca_btl_openib_add_procs(
openib_btl->num_peers += nprocs;
if(mca_btl_openib_component.use_srq) {
openib_btl->rd_buf_max = mca_btl_openib_component.ib_rr_buf_max + log2(nprocs) * mca_btl_openib_component.rd_per_peer;
free(openib_btl->rr_desc_post);
openib_btl->rr_desc_post = (struct ibv_recv_wr*) malloc((openib_btl->rd_buf_max * sizeof(struct ibv_recv_wr)));
openib_btl->rd_buf_min = openib_btl->rd_buf_max / 2;
}
}
#endif
@ -577,38 +581,52 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*) btl;
frag->endpoint = endpoint;
frag->wr_desc.sr_desc.opcode = IBV_WR_RDMA_WRITE;
frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_dst->seg_addr.pval;
frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_dst->seg_key.key32[0];
frag->sg_entry.addr = (uintptr_t) frag->base.des_src->seg_addr.pval;
frag->sg_entry.length = frag->base.des_src->seg_len;
BTL_VERBOSE(("frag->wr_desc.sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu"
, frag->wr_desc.sr_desc.wr.rdma.remote_addr
, frag->wr_desc.sr_desc.wr.rdma.rkey
, frag->sg_entry.addr
, frag->sg_entry.length));
if(!mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,-1) < 0) {
BTL_VERBOSE(("Queing because no rdma write tokens \n"));
BTL_OPENIB_INSERT_PENDING(frag, endpoint->pending_frags_lp,
endpoint->wr_sq_tokens_lp, endpoint->endpoint_lock);
return OMPI_SUCCESS;
} else if(mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_lp,-1) < 0) {
opal_list_append(&openib_btl->pending_frags_lp, (opal_list_item_t *)frag);
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_lp,1);
return OMPI_SUCCESS;
} else {
frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_dst->seg_addr.pval;
frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_dst->seg_key.key32[0];
frag->sg_entry.addr = (uintptr_t) frag->base.des_src->seg_addr.pval;
frag->sg_entry.length = frag->base.des_src->seg_len;
BTL_VERBOSE(("frag->wr_desc.sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu"
, frag->wr_desc.sr_desc.wr.rdma.remote_addr
, frag->wr_desc.sr_desc.wr.rdma.rkey
, frag->sg_entry.addr
, frag->sg_entry.length));
if(ibv_post_send(endpoint->lcl_qp_low,
&frag->wr_desc.sr_desc,
&bad_wr)){
BTL_ERROR(("error posting send request errno says %s", strerror(errno)));
return OMPI_ERROR;
}
if(ibv_post_send(endpoint->lcl_qp_low,
&frag->wr_desc.sr_desc,
&bad_wr)){
BTL_ERROR(("error posting send request errno says %s", strerror(errno)));
return OMPI_ERROR;
}
#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ
if(mca_btl_openib_component.use_srq) {
MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1);
MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1);
} else {
if(mca_btl_openib_component.use_srq) {
MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1);
MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1);
} else {
#endif
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1);
#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ
}
}
#endif
}
return OMPI_SUCCESS;
}
@ -625,38 +643,53 @@ int mca_btl_openib_get( mca_btl_base_module_t* btl,
mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*) btl;
frag->endpoint = endpoint;
frag->wr_desc.sr_desc.opcode = IBV_WR_RDMA_READ;
frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_src->seg_addr.pval;
frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_src->seg_key.key32[0];
frag->sg_entry.addr = (uintptr_t) frag->base.des_dst->seg_addr.pval;
frag->sg_entry.length = frag->base.des_dst->seg_len;
BTL_VERBOSE(("frag->wr_desc.sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu"
, frag->wr_desc.sr_desc.wr.rdma.remote_addr
, frag->wr_desc.sr_desc.wr.rdma.rkey
, frag->sg_entry.addr
, frag->sg_entry.length));
if(ibv_post_send(endpoint->lcl_qp_low,
&frag->wr_desc.sr_desc,
&bad_wr)){
BTL_ERROR(("error posting send request errno says %s", strerror(errno)));
return OMPI_ERROR;
}
/* atomically test and acquire a token */
if(!mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,-1) < 0) {
BTL_VERBOSE(("Queing because no rdma write tokens \n"));
BTL_OPENIB_INSERT_PENDING(frag, endpoint->pending_frags_lp,
endpoint->wr_sq_tokens_lp, endpoint->endpoint_lock);
return OMPI_SUCCESS;
} else if(mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_lp,-1) < 0) {
opal_list_append(&openib_btl->pending_frags_lp, (opal_list_item_t *)frag);
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_lp,1);
return OMPI_SUCCESS;
} else {
frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED;
frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_src->seg_addr.pval;
frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_src->seg_key.key32[0];
frag->sg_entry.addr = (uintptr_t) frag->base.des_dst->seg_addr.pval;
frag->sg_entry.length = frag->base.des_dst->seg_len;
BTL_VERBOSE(("frag->wr_desc.sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu"
, frag->wr_desc.sr_desc.wr.rdma.remote_addr
, frag->wr_desc.sr_desc.wr.rdma.rkey
, frag->sg_entry.addr
, frag->sg_entry.length));
if(ibv_post_send(endpoint->lcl_qp_low,
&frag->wr_desc.sr_desc,
&bad_wr)){
BTL_ERROR(("error posting send request errno says %s", strerror(errno)));
return OMPI_ERROR;
}
#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ
if(mca_btl_openib_component.use_srq) {
MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1);
MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1);
} else {
if(mca_btl_openib_component.use_srq) {
MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 1);
MCA_BTL_OPENIB_POST_SRR_LOW(openib_btl, 1);
} else {
#endif
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(endpoint, 1);
#ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ
}
}
#endif
}
return OMPI_SUCCESS;
}

27
ompi/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -110,6 +110,8 @@ struct mca_btl_openib_component_t {
uint32_t ib_src_path_bits;
uint32_t rd_per_peer; /* number of receive descriptors to post per log2(peers)
in SRQ mode */
uint32_t max_wr_sq_tokens;
uint32_t max_total_wr_sq_tokens;
}; typedef struct mca_btl_openib_component_t mca_btl_openib_component_t;
@ -165,7 +167,20 @@ struct mca_btl_openib_module_t {
#endif
uint32_t num_peers;
uint32_t rd_buf_max;
uint32_t rd_buf_min;
int32_t wr_sq_tokens_hp;
/**< number of high priority frags that can be outstanding (down counter) */
int32_t wr_sq_tokens_lp;
/**< number of low priority frags that can be outstanding (down counter) */
opal_list_t pending_frags_hp;
/**< list of pending high priority frags */
opal_list_t pending_frags_lp;
/**< list of pending low priority frags */
}; typedef struct mca_btl_openib_module_t mca_btl_openib_module_t;
@ -403,9 +418,9 @@ int mca_btl_openib_module_init(mca_btl_openib_module_t* openib_btl);
{ \
do{ \
OPAL_THREAD_LOCK(&openib_btl->ib_lock); \
if(openib_btl->srr_posted_high <= mca_btl_openib_component.ib_rr_buf_min+additional && \
openib_btl->srr_posted_high < mca_btl_openib_component.ib_rr_buf_max){ \
MCA_BTL_OPENIB_POST_SRR_SUB(mca_btl_openib_component.ib_rr_buf_max - \
if(openib_btl->srr_posted_high <= openib_btl->rd_buf_min+additional && \
openib_btl->srr_posted_high < openib_btl->rd_buf_max){ \
MCA_BTL_OPENIB_POST_SRR_SUB(openib_btl->rd_buf_max - \
openib_btl->srr_posted_high, \
openib_btl, \
&openib_btl->recv_free_eager, \
@ -420,9 +435,9 @@ int mca_btl_openib_module_init(mca_btl_openib_module_t* openib_btl);
{ \
do { \
OPAL_THREAD_LOCK(&openib_btl->ib_lock); \
if(openib_btl->srr_posted_low <= mca_btl_openib_component.ib_rr_buf_min+additional && \
openib_btl->srr_posted_low < mca_btl_openib_component.ib_rr_buf_max){ \
MCA_BTL_OPENIB_POST_SRR_SUB(mca_btl_openib_component.ib_rr_buf_max - \
if(openib_btl->srr_posted_low <= openib_btl->rd_buf_min+additional && \
openib_btl->srr_posted_low < openib_btl->rd_buf_max){ \
MCA_BTL_OPENIB_POST_SRR_SUB(openib_btl->rd_buf_max - \
openib_btl->srr_posted_low, \
openib_btl, \
&openib_btl->recv_free_max, \

103
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -196,6 +196,11 @@ int mca_btl_openib_component_open(void)
mca_btl_openib_param_register_int("bandwidth", "Approximate maximum bandwidth of interconnect",
800, (int*) &mca_btl_openib_module.super.btl_bandwidth);
mca_btl_openib_param_register_int("max_wr_sq_tokens", "Maximum number of send/rdma work request tokens",
16, (int*) &mca_btl_openib_component.max_wr_sq_tokens);
mca_btl_openib_param_register_int("max_total_wr_sq_tokens", "Maximum number of send/rdma work request tokens peer btl",
32, (int*) &mca_btl_openib_component.max_total_wr_sq_tokens);
param = mca_base_param_find("mpi", NULL, "leave_pinned");
mca_base_param_lookup_int(param, &value);
@ -402,9 +407,17 @@ mca_btl_base_module_t** mca_btl_openib_component_init(int *num_btl_modules,
free(openib_btl);
openib_btl = &mca_btl_openib_component.openib_btls[i];
openib_btl->rd_buf_max = mca_btl_openib_component.ib_rr_buf_max;
openib_btl->rd_buf_min = mca_btl_openib_component.ib_rr_buf_min;
openib_btl->num_peers = 0;
openib_btl->wr_sq_tokens_hp =
openib_btl->wr_sq_tokens_lp = mca_btl_openib_component.max_total_wr_sq_tokens;
/* Initialize module state */
OBJ_CONSTRUCT(&openib_btl->pending_frags_hp, opal_list_t);
OBJ_CONSTRUCT(&openib_btl->pending_frags_lp, opal_list_t);
OBJ_CONSTRUCT(&openib_btl->ib_lock, opal_mutex_t);
OBJ_CONSTRUCT(&openib_btl->send_free_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&openib_btl->send_free_max, ompi_free_list_t);
@ -601,6 +614,31 @@ int mca_btl_openib_component_progress()
frag->rc = OMPI_SUCCESS;
frag->base.des_cbfunc(&openib_btl->super, frag->endpoint, &frag->base, frag->rc);
count++;
/* check and see if we need to progress pending sends */
if( !mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_hp, 1) > 0
&& !opal_list_is_empty(&(frag->endpoint->pending_frags_hp))) {
opal_list_item_t *frag_item;
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_lock);
frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_hp));
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_lock);
frag = (mca_btl_openib_frag_t *) frag_item;
if(OMPI_SUCCESS != mca_btl_openib_endpoint_send(frag->endpoint, frag)) {
BTL_ERROR(("error in posting pending send\n"));
}
}
if( mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_hp, 1) > 0
&& !opal_list_is_empty(&openib_btl->pending_frags_hp)) {
opal_list_item_t *frag_item;
frag_item = opal_list_remove_first(&openib_btl->pending_frags_hp);
frag = (mca_btl_openib_frag_t *) frag_item;
if(OMPI_SUCCESS != mca_btl_openib_endpoint_send(frag->endpoint, frag)) {
BTL_ERROR(("error in posting pending send\n"));
}
}
break;
@ -670,6 +708,69 @@ int mca_btl_openib_component_progress()
frag->rc = OMPI_SUCCESS;
frag->base.des_cbfunc(&openib_btl->super, frag->endpoint, &frag->base, frag->rc);
count++;
if(!mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_lp, 1) > 0 &&
!opal_list_is_empty(&(frag->endpoint->pending_frags_lp))) {
opal_list_item_t *frag_item;
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_lock);
frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_lp));
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_lock);
frag = (mca_btl_openib_frag_t *) frag_item;
switch(frag->wr_desc.sr_desc.opcode){
case IBV_WR_SEND:
if(OMPI_SUCCESS != mca_btl_openib_endpoint_send(frag->endpoint, frag)) {
BTL_ERROR(("error in posting pending send\n"));
}
break;
case IBV_WR_RDMA_WRITE:
if(OMPI_SUCCESS != mca_btl_openib_put((mca_btl_base_module_t*) openib_btl,
frag->endpoint,
(mca_btl_base_descriptor_t*) frag)) {
BTL_ERROR(("error in posting pending rdma write\n"));
}
break;
case IBV_WR_RDMA_READ:
if(OMPI_SUCCESS != mca_btl_openib_put((mca_btl_base_module_t *) openib_btl,
frag->endpoint,
(mca_btl_base_descriptor_t*) frag)) {
BTL_ERROR(("error in posting pending rdma read\n"));
}
break;
default:
BTL_ERROR(("error in posting pending operation, invalide opcode %d\n", frag->wr_desc.sr_desc.opcode));
}
}
if(mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_lp, 1) > 0
&& !opal_list_is_empty(&openib_btl->pending_frags_lp)) {
opal_list_item_t *frag_item;
frag_item = opal_list_remove_first(&openib_btl->pending_frags_lp);
frag = (mca_btl_openib_frag_t *) frag_item;
switch(frag->wr_desc.sr_desc.opcode){
case IBV_WR_SEND:
if(OMPI_SUCCESS != mca_btl_openib_endpoint_send(frag->endpoint, frag)) {
BTL_ERROR(("error in posting pending send\n"));
}
break;
case IBV_WR_RDMA_WRITE:
if(OMPI_SUCCESS != mca_btl_openib_put((mca_btl_base_module_t*) openib_btl,
frag->endpoint,
(mca_btl_base_descriptor_t*) frag)) {
BTL_ERROR(("error in posting pending rdma write\n"));
}
break;
case IBV_WR_RDMA_READ:
if(OMPI_SUCCESS != mca_btl_openib_put((mca_btl_base_module_t *) openib_btl,
frag->endpoint,
(mca_btl_base_descriptor_t*) frag)) {
BTL_ERROR(("error in posting pending rdma read\n"));
}
break;
default:
BTL_ERROR(("error in posting pending operation, invalide opcode %d\n", frag->wr_desc.sr_desc.opcode));
}
}
break;
default:

53
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -74,9 +74,40 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
frag->sg_entry.addr = (uintptr_t) frag->hdr;
if(frag->base.des_flags & MCA_BTL_DES_FLAGS_PRIORITY && frag->size <= openib_btl->super.btl_eager_limit){
ib_qp = endpoint->lcl_qp_high;
/* atomically test and acquire a token */
if(!mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_hp,-1) < 0) {
BTL_VERBOSE(("Queing because no send tokens \n"));
opal_list_append(&endpoint->pending_frags_hp, (opal_list_item_t *)frag);
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_hp,1);
return OMPI_SUCCESS;
} else if( mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_hp,-1) < 0) {
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_hp,1);
opal_list_append(&openib_btl->pending_frags_hp, (opal_list_item_t *)frag);
return OMPI_SUCCESS;
} else {
ib_qp = endpoint->lcl_qp_high;
}
} else {
ib_qp = endpoint->lcl_qp_low;
/* atomically test and acquire a token */
if(!mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,-1) < 0 ) {
BTL_VERBOSE(("Queing because no send tokens \n"));
opal_list_append(&endpoint->pending_frags_lp, (opal_list_item_t *)frag);
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,1);
return OMPI_SUCCESS;
} else if(mca_btl_openib_component.use_srq &&
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_lp,-1) < 0) {
OPAL_THREAD_ADD32(&openib_btl->wr_sq_tokens_lp,1);
opal_list_append(&openib_btl->pending_frags_lp, (opal_list_item_t *)frag);
return OMPI_SUCCESS;
} else {
ib_qp = endpoint->lcl_qp_low;
}
}
frag->wr_desc.sr_desc.opcode = IBV_WR_SEND;
@ -132,15 +163,21 @@ static void mca_btl_openib_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
endpoint->endpoint_tstamp = 0.0;
endpoint->endpoint_state = MCA_BTL_IB_CLOSED;
endpoint->endpoint_retries = 0;
OBJ_CONSTRUCT(&endpoint->endpoint_send_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_recv_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->pending_send_frags, opal_list_t);
OBJ_CONSTRUCT(&endpoint->pending_frags_hp, opal_list_t);
OBJ_CONSTRUCT(&endpoint->pending_frags_lp, opal_list_t);
endpoint->lcl_qp_attr_high = (struct ibv_qp_attr *) malloc(sizeof(struct ibv_qp_attr));
endpoint->lcl_qp_attr_low = (struct ibv_qp_attr *) malloc(sizeof(struct ibv_qp_attr));
memset(endpoint->lcl_qp_attr_high, 0, sizeof(struct ibv_qp_attr));
memset(endpoint->lcl_qp_attr_low, 0, sizeof(struct ibv_qp_attr));
endpoint->rr_posted_high = 0;
endpoint->rr_posted_low = 0;
endpoint->wr_sq_tokens_hp = mca_btl_openib_component.max_wr_sq_tokens;
endpoint->wr_sq_tokens_lp = mca_btl_openib_component.max_wr_sq_tokens;
endpoint->rem_info.rem_qp_num_high = 0;
endpoint->rem_info.rem_qp_num_low = 0;
endpoint->rem_info.rem_lid = 0;
@ -656,7 +693,7 @@ int mca_btl_openib_endpoint_send(
int rc;
mca_btl_openib_module_t *openib_btl;
OPAL_THREAD_LOCK(&endpoint->endpoint_send_lock);
OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
switch(endpoint->endpoint_state) {
case MCA_BTL_IB_CONNECTING:
@ -707,7 +744,7 @@ int mca_btl_openib_endpoint_send(
rc = OMPI_ERR_UNREACH;
}
OPAL_THREAD_UNLOCK(&endpoint->endpoint_send_lock);
OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
return rc;
}
@ -796,8 +833,8 @@ int mca_btl_openib_endpoint_create_qp(
qp_init_attr.send_cq = cq;
qp_init_attr.recv_cq = cq;
qp_init_attr.cap.max_send_wr = mca_btl_openib_component.ib_wq_size;
qp_init_attr.cap.max_recv_wr = mca_btl_openib_component.ib_wq_size;
qp_init_attr.cap.max_send_wr = mca_btl_openib_component.max_wr_sq_tokens;
qp_init_attr.cap.max_recv_wr = mca_btl_openib_component.ib_rr_buf_max;
qp_init_attr.cap.max_send_sge = mca_btl_openib_component.ib_sg_list_size;
qp_init_attr.cap.max_recv_sge = mca_btl_openib_component.ib_sg_list_size;
qp_init_attr.qp_type = IBV_QPT_RC;

31
ompi/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -118,15 +118,26 @@ struct mca_btl_base_endpoint_t {
double endpoint_tstamp;
/**< timestamp of when the first connection was attempted */
opal_mutex_t endpoint_send_lock;
opal_mutex_t endpoint_lock;
/**< lock for concurrent access to endpoint state */
opal_mutex_t endpoint_recv_lock;
/**< lock for concurrent access to endpoint state */
opal_list_t pending_send_frags;
/**< list of pending send frags for this endpotint */
opal_list_t pending_frags_hp;
/**< list of pending high priority frags */
opal_list_t pending_frags_lp;
/**< list of pending low priority frags */
int32_t wr_sq_tokens_hp;
/**< number of high priority frags that can be outstanding (down counter) */
int32_t wr_sq_tokens_lp;
/**< number of low priority frags that can be outstanding (down counter) */
mca_btl_openib_rem_info_t rem_info;
uint32_t lcl_psn_high;
@ -223,6 +234,16 @@ void mca_btl_openib_post_recv(void);
} while(0); \
}
#define BTL_OPENIB_INSERT_PENDING(frag, frag_list, tokens, lock) \
{ \
do{ \
OPAL_THREAD_LOCK(&lock); \
opal_list_append(&frag_list, (opal_list_item_t *)frag); \
OPAL_THREAD_UNLOCK(&lock); \
OPAL_THREAD_ADD32(&tokens, 1); \
} while(0); \
}
#if defined(c_plusplus) || defined(__cplusplus)
}