ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Additional changes to add_proc and tokens

Просмотр исходного кода 
This commit was SVN r7152.
Этот коммит содержится в:
Galen Shipman 2005-09-02 15:18:36 +00:00
родитель 4fbf7965e3
Коммит 589b1b8b5a
5 изменённых файлов: 35 добавлений и 16 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

29
ompi/mca/btl/mvapi/btl_mvapi.c
Просмотреть файл

@ -117,12 +117,19 @@ int mca_btl_mvapi_add_procs(
peers[i] = ib_peer;
}
mvapi_btl->num_peers += nprocs;
if(mca_btl_mvapi_component.use_srq) {
mvapi_btl->rd_buf_max = mca_btl_mvapi_component.ib_rr_buf_max + log2(nprocs) * mca_btl_mvapi_component.rd_per_peer;
}
/* currently we only scale the srq the first time
add_procs is called, subsequent calls are ignored,
we should be able to change this to modify the SRQ but
I am unsure as to what this entails
*/
return OMPI_SUCCESS;
if( 0 == mvapi_btl->num_peers ) {
mvapi_btl->num_peers += nprocs;
if(mca_btl_mvapi_component.use_srq) {
mvapi_btl->rd_buf_max = mca_btl_mvapi_component.ib_rr_buf_max + log2(nprocs) * mca_btl_mvapi_component.rd_per_peer;
}
}
return OMPI_SUCCESS;
}
/*
@ -784,11 +791,11 @@ int mca_btl_mvapi_put( mca_btl_base_module_t* btl,
mca_btl_mvapi_module_t* mvapi_btl = (mca_btl_mvapi_module_t*) btl;
mca_btl_mvapi_frag_t* frag = (mca_btl_mvapi_frag_t*) descriptor;
frag->sr_desc.opcode = VAPI_RDMA_WRITE;
OPAL_THREAD_LOCK(&endpoint->endpoint_send_lock);
/* atomically test and acquire a token */
if(OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,-1) < 0) {
BTL_VERBOSE(("Queing because no rdma write tokens \n"));
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_lp, endpoint->wr_sq_tokens_lp, rc);
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_lp,
endpoint->wr_sq_tokens_lp, endpoint->endpoint_pending_lock, rc);
} else {
frag->endpoint = endpoint;
@ -816,7 +823,7 @@ int mca_btl_mvapi_put( mca_btl_base_module_t* btl,
MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(endpoint, 1);
}
}
OPAL_THREAD_UNLOCK(&endpoint->endpoint_send_lock);
return rc;
}
@ -834,11 +841,12 @@ int mca_btl_mvapi_get( mca_btl_base_module_t* btl,
mca_btl_mvapi_frag_t* frag = (mca_btl_mvapi_frag_t*) descriptor;
frag->sr_desc.opcode = VAPI_RDMA_READ;
OPAL_THREAD_LOCK(&endpoint->endpoint_send_lock);
/* atomically test and acquire a token */
if(OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,-1) < 0) {
BTL_VERBOSE(("Queing because no rdma write tokens \n"));
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_lp, endpoint->wr_sq_tokens_lp, rc);
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_lp,
endpoint->wr_sq_tokens_lp, endpoint->endpoint_pending_lock, rc);
} else {
frag->endpoint = endpoint;
frag->sr_desc.remote_qp = endpoint->rem_info.rem_qp_num_low;
@ -863,7 +871,6 @@ int mca_btl_mvapi_get( mca_btl_base_module_t* btl,
MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(endpoint, 1);
}
}
OPAL_THREAD_UNLOCK(&endpoint->endpoint_send_lock);
return rc;
}

4
ompi/mca/btl/mvapi/btl_mvapi_component.c
Просмотреть файл

@ -566,7 +566,9 @@ int mca_btl_mvapi_component_progress()
if(OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_hp, 1) > 0
&& !opal_list_is_empty(&(frag->endpoint->pending_frags_hp))) {
opal_list_item_t *frag_item;
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_pending_lock);
frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_hp));
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_pending_lock);
frag = (mca_btl_mvapi_frag_t *) frag_item;
if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_send(frag->endpoint, frag)) {
@ -635,7 +637,9 @@ int mca_btl_mvapi_component_progress()
if(OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_lp, 1) > 0
&& !opal_list_is_empty(&(frag->endpoint->pending_frags_lp))) {
opal_list_item_t *frag_item;
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_pending_lock);
frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_lp));
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_pending_lock);
frag = (mca_btl_mvapi_frag_t *) frag_item;
switch(frag->sr_desc.opcode){
case VAPI_SEND:

7
ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
Просмотреть файл

@ -69,7 +69,8 @@ static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi
/* atomically test and acquire a token */
if(OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_hp,-1) < 0) {
BTL_VERBOSE(("Queing because no send tokens \n"));
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_hp, endpoint->wr_sq_tokens_hp, rc);
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_hp,
endpoint->wr_sq_tokens_hp, endpoint->endpoint_pending_lock, rc);
return OMPI_SUCCESS;
} else {
frag->sr_desc.remote_qp = endpoint->rem_info.rem_qp_num_high;
@ -81,7 +82,8 @@ static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi
/* atomically test and acquire a token */
if(OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,-1) < 0) {
BTL_VERBOSE(("Queing because no send tokens \n"));
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_hp, endpoint->wr_sq_tokens_hp, rc);
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_hp,
endpoint->wr_sq_tokens_hp, endpoint->endpoint_pending_lock, rc);
return OMPI_SUCCESS;
} else {
frag->sr_desc.remote_qp = endpoint->rem_info.rem_qp_num_low;
@ -137,6 +139,7 @@ static void mca_btl_mvapi_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
endpoint->endpoint_state = MCA_BTL_IB_CLOSED;
endpoint->endpoint_retries = 0;
OBJ_CONSTRUCT(&endpoint->endpoint_send_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_pending_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_recv_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->pending_send_frags, opal_list_t);
OBJ_CONSTRUCT(&endpoint->pending_frags_hp, opal_list_t);

9
ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
Просмотреть файл

@ -117,10 +117,13 @@ struct mca_btl_base_endpoint_t {
opal_mutex_t endpoint_send_lock;
/**< lock for concurrent access to endpoint state */
opal_mutex_t endpoint_recv_lock;
/**< lock for concurrent access to endpoint state */
opal_mutex_t endpoint_pending_lock;
/**< lock for pending frags list access */
opal_list_t pending_send_frags;
/**< list of pending send frags for this endpoint */
@ -239,10 +242,12 @@ void mca_btl_mvapi_progress_send_frags(mca_btl_mvapi_endpoint_t*);
}\
}
#define BTL_MVAPI_INSERT_PENDING(frag, frag_list, tokens, rc) \
#define BTL_MVAPI_INSERT_PENDING(frag, frag_list, tokens, lock, rc) \
{ \
do{ \
OPAL_THREAD_LOCK(&lock); \
opal_list_append(&frag_list, (opal_list_item_t *)frag); \
OPAL_THREAD_UNLOCK(&lock); \
OPAL_THREAD_ADD32(&tokens, 1); \
rc = OMPI_SUCCESS; \
} while(0); \

2
ompi/mca/btl/mvapi/btl_mvapi_proc.c
Просмотреть файл

@ -154,7 +154,7 @@ mca_btl_mvapi_proc_t* mca_btl_mvapi_proc_create(ompi_proc_t* ompi_proc)
mvapi_proc->proc_port_count = size/sizeof(mca_btl_mvapi_port_info_t);
mvapi_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc(mvapi_proc->proc_port_count * sizeof(mca_btl_base_endpoint_t*));