ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

minor changes to thread locking

Просмотр исходного кода 
This commit was SVN r7154.
Этот коммит содержится в:
Tim Woodall 2005-09-02 16:27:01 +00:00
родитель a9863e5ff5
Коммит dfe52fceef
4 изменённых файлов: 55 добавлений и 114 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
ompi/mca/btl/mvapi/btl_mvapi.c
Просмотреть файл

@ -772,10 +772,8 @@ int mca_btl_mvapi_send(
mca_btl_mvapi_frag_t* frag = (mca_btl_mvapi_frag_t*)descriptor;
frag->endpoint = endpoint;
frag->hdr->tag = tag;
frag->rc = mca_btl_mvapi_endpoint_send(endpoint, frag);
return frag->rc;
}

10
ompi/mca/btl/mvapi/btl_mvapi_component.c
Просмотреть файл

@ -566,11 +566,11 @@ int mca_btl_mvapi_component_progress()
if(OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_hp, 1) > 0
&& !opal_list_is_empty(&(frag->endpoint->pending_frags_hp))) {
opal_list_item_t *frag_item;
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_pending_lock);
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_lock);
frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_hp));
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_pending_lock);
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_lock);
frag = (mca_btl_mvapi_frag_t *) frag_item;
if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_send(frag->endpoint, frag)) {
BTL_ERROR(("error in posting pending send\n"));
}
@ -637,9 +637,9 @@ int mca_btl_mvapi_component_progress()
if(OPAL_THREAD_ADD32(&frag->endpoint->wr_sq_tokens_lp, 1) > 0
&& !opal_list_is_empty(&(frag->endpoint->pending_frags_lp))) {
opal_list_item_t *frag_item;
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_pending_lock);
OPAL_THREAD_LOCK(&frag->endpoint->endpoint_lock);
frag_item = opal_list_remove_first(&(frag->endpoint->pending_frags_lp));
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_pending_lock);
OPAL_THREAD_UNLOCK(&frag->endpoint->endpoint_lock);
frag = (mca_btl_mvapi_frag_t *) frag_item;
switch(frag->sr_desc.opcode){
case VAPI_SEND:

146
ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
Просмотреть файл

@ -35,30 +35,34 @@ static void mca_btl_mvapi_endpoint_construct(mca_btl_base_endpoint_t* endpoint);
static void mca_btl_mvapi_endpoint_destruct(mca_btl_base_endpoint_t* endpoint);
int mca_btl_mvapi_endpoint_create_qp(
mca_btl_mvapi_module_t* mvapi_btl,
VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag,
VAPI_cq_hndl_t cq_hndl,
VAPI_srq_hndl_t srq_hndl,
VAPI_qp_hndl_t* qp_hndl,
VAPI_qp_prop_t* qp_prop,
int transport_type);
mca_btl_mvapi_module_t* mvapi_btl,
VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag,
VAPI_cq_hndl_t cq_hndl,
VAPI_srq_hndl_t srq_hndl,
VAPI_qp_hndl_t* qp_hndl,
VAPI_qp_prop_t* qp_prop,
int transport_type
);
int mca_btl_mvapi_endpoint_qp_init_query(
mca_btl_mvapi_module_t* mvapi_btl,
VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp_num,
IB_lid_t remote_lid,
IB_port_t port_id
);
mca_btl_mvapi_module_t* mvapi_btl,
VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp_num,
IB_lid_t remote_lid,
IB_port_t port_id
);
/*
* Note this routine must be called w/ endpoint lock held.
*/
static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi_btl, mca_btl_mvapi_endpoint_t * endpoint, mca_btl_mvapi_frag_t * frag)
static inline int mca_btl_mvapi_endpoint_post_send(
mca_btl_mvapi_module_t* mvapi_btl,
mca_btl_mvapi_endpoint_t * endpoint,
mca_btl_mvapi_frag_t * frag)
{
int rc;
VAPI_qp_hndl_t qp_hndl;
frag->sr_desc.remote_qkey = 0;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
@ -69,8 +73,8 @@ static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi
/* atomically test and acquire a token */
if(OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_hp,-1) < 0) {
BTL_VERBOSE(("Queing because no send tokens \n"));
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_hp,
endpoint->wr_sq_tokens_hp, endpoint->endpoint_pending_lock, rc);
opal_list_append(&endpoint->pending_frags_hp, (opal_list_item_t *)frag);
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_hp,1);
return OMPI_SUCCESS;
} else {
frag->sr_desc.remote_qp = endpoint->rem_info.rem_qp_num_high;
@ -82,8 +86,8 @@ static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi
/* atomically test and acquire a token */
if(OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,-1) < 0) {
BTL_VERBOSE(("Queing because no send tokens \n"));
BTL_MVAPI_INSERT_PENDING(frag, endpoint->pending_frags_hp,
endpoint->wr_sq_tokens_hp, endpoint->endpoint_pending_lock, rc);
opal_list_append(&endpoint->pending_frags_lp, (opal_list_item_t *)frag);
OPAL_THREAD_ADD32(&endpoint->wr_sq_tokens_lp,1);
return OMPI_SUCCESS;
} else {
frag->sr_desc.remote_qp = endpoint->rem_info.rem_qp_num_low;
@ -105,8 +109,6 @@ static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi
&frag->sr_desc);
}
if(VAPI_OK != frag->ret)
return OMPI_ERROR;
if(mca_btl_mvapi_component.use_srq) {
@ -138,9 +140,7 @@ static void mca_btl_mvapi_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
endpoint->endpoint_tstamp = 0.0;
endpoint->endpoint_state = MCA_BTL_IB_CLOSED;
endpoint->endpoint_retries = 0;
OBJ_CONSTRUCT(&endpoint->endpoint_send_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_pending_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_recv_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->endpoint_lock, opal_mutex_t);
OBJ_CONSTRUCT(&endpoint->pending_send_frags, opal_list_t);
OBJ_CONSTRUCT(&endpoint->pending_frags_hp, opal_list_t);
OBJ_CONSTRUCT(&endpoint->pending_frags_lp, opal_list_t);
@ -234,40 +234,6 @@ static int mca_btl_mvapi_endpoint_send_connect_data(mca_btl_base_endpoint_t* end
return OMPI_SUCCESS;
}
/*
* Send connect ACK to remote endpoint
*
*/
/* static int mca_btl_mvapi_endpoint_send_connect_data(mca_btl_base_endpoint_t* endpoint) */
/* { */
/* orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t); */
/* int rc; */
/* uint32_t zero = 0; */
/* /\* pack the info in the send buffer *\/ */
/* if(ORTE_SUCCESS != (rc = orte_dps.pack(buffer, &zero, 1, ORTE_UINT32))) { */
/* ORTE_ERROR_LOG(rc); */
/* return rc; */
/* } */
/* if(ORTE_SUCCESS != (rc = orte_dps.pack(buffer, &zero, 1, ORTE_UINT32))) { */
/* ORTE_ERROR_LOG(rc); */
/* return rc; */
/* } */
/* if(ORTE_SUCCESS != (rc = orte_dps.pack(buffer, &zero, 1, ORTE_UINT32))) { */
/* ORTE_ERROR_LOG(rc); */
/* return rc; */
/* } */
/* /\* send to endpoint *\/ */
/* rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_guid, buffer, ORTE_RML_TAG_DYNAMIC-1, 0, */
/* mca_btl_mvapi_endpoint_send_cb, NULL); */
/* if(rc < 0) { */
/* ORTE_ERROR_LOG(rc); */
/* } */
/* return rc; */
/* } */
/*
* Set remote connection info
*
@ -414,14 +380,28 @@ static int mca_btl_mvapi_endpoint_reply_start_connect(mca_btl_mvapi_endpoint_t *
static void mca_btl_mvapi_endpoint_waiting_ack(mca_btl_mvapi_endpoint_t *endpoint) {
endpoint->endpoint_state = MCA_BTL_IB_WAITING_ACK;
}
/*
*
*/
static void mca_btl_mvapi_endpoint_connected(mca_btl_mvapi_endpoint_t *endpoint)
{
opal_list_item_t *frag_item;
mca_btl_mvapi_frag_t *frag;
mca_btl_mvapi_module_t* mvapi_btl;
/* While there are frags in the list, process them */
endpoint->endpoint_state = MCA_BTL_IB_CONNECTED;
mca_btl_mvapi_progress_send_frags(endpoint);
while(!opal_list_is_empty(&(endpoint->pending_send_frags))) {
frag_item = opal_list_remove_first(&(endpoint->pending_send_frags));
frag = (mca_btl_mvapi_frag_t *) frag_item;
mvapi_btl = endpoint->endpoint_btl;
/* We need to post this one */
if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag))
BTL_ERROR(("error in mca_btl_mvapi_endpoint_send"));
}
}
/*
@ -536,6 +516,8 @@ static void mca_btl_mvapi_endpoint_recv(
BTL_ERROR(("can't find suitable endpoint for this peer\n"));
return;
}
OPAL_THREAD_LOCK(&ib_endpoint->endpoint_lock);
endpoint_state = ib_endpoint->endpoint_state;
/* Update status */
@ -586,6 +568,7 @@ static void mca_btl_mvapi_endpoint_recv(
BTL_ERROR(("Invalid endpoint state %d", endpoint_state));
}
OPAL_THREAD_UNLOCK(&ib_endpoint->endpoint_lock);
break;
}
}
@ -619,8 +602,7 @@ int mca_btl_mvapi_endpoint_send(
int rc;
mca_btl_mvapi_module_t *mvapi_btl;
OPAL_THREAD_LOCK(&endpoint->endpoint_send_lock);
OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
switch(endpoint->endpoint_state) {
case MCA_BTL_IB_CONNECTING:
@ -631,6 +613,7 @@ int mca_btl_mvapi_endpoint_send(
rc = OMPI_SUCCESS;
break;
case MCA_BTL_IB_WAITING_ACK:
case MCA_BTL_IB_CONNECT_ACK:
@ -641,9 +624,6 @@ int mca_btl_mvapi_endpoint_send(
rc = OMPI_SUCCESS;
break;
case MCA_BTL_IB_CLOSED:
@ -663,14 +643,11 @@ int mca_btl_mvapi_endpoint_send(
case MCA_BTL_IB_CONNECTED:
{
mvapi_btl = endpoint->endpoint_btl;
BTL_VERBOSE(("Send to : %d, len : %d, frag : %p",
endpoint->endpoint_proc->proc_guid.vpid,
frag->sg_entry.len,
frag));
rc = mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag);
}
break;
@ -679,35 +656,10 @@ int mca_btl_mvapi_endpoint_send(
rc = OMPI_ERR_UNREACH;
}
OPAL_THREAD_UNLOCK(&endpoint->endpoint_send_lock);
OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
return rc;
}
void mca_btl_mvapi_progress_send_frags(mca_btl_mvapi_endpoint_t* endpoint)
{
opal_list_item_t *frag_item;
mca_btl_mvapi_frag_t *frag;
mca_btl_mvapi_module_t* mvapi_btl;
/*Check if endpoint is connected */
if(endpoint->endpoint_state != MCA_BTL_IB_CONNECTED) {
return;
}
/* While there are frags in the list,
* process them */
while(!opal_list_is_empty(&(endpoint->pending_send_frags))) {
frag_item = opal_list_remove_first(&(endpoint->pending_send_frags));
frag = (mca_btl_mvapi_frag_t *) frag_item;
mvapi_btl = endpoint->endpoint_btl;
/* We need to post this one */
if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag))
BTL_ERROR(("error in mca_btl_mvapi_endpoint_send"));
}
}

11
ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
Просмотреть файл

@ -115,15 +115,9 @@ struct mca_btl_base_endpoint_t {
double endpoint_tstamp;
/**< timestamp of when the first connection was attempted */
opal_mutex_t endpoint_send_lock;
opal_mutex_t endpoint_lock;
/**< lock for concurrent access to endpoint state */
opal_mutex_t endpoint_recv_lock;
/**< lock for concurrent access to endpoint state */
opal_mutex_t endpoint_pending_lock;
/**< lock for pending frags list access */
opal_list_t pending_send_frags;
/**< list of pending send frags for this endpoint */
@ -136,7 +130,6 @@ struct mca_btl_base_endpoint_t {
uint32_t wr_sq_tokens_hp;
/**< number of high priority frags that can be outstanding (down counter) */
uint32_t wr_sq_tokens_lp;
/**< number of low priority frags that can be outstanding (down counter) */
@ -169,8 +162,6 @@ int mca_btl_mvapi_endpoint_connect(mca_btl_base_endpoint_t*);
void mca_btl_mvapi_post_recv(void);
void mca_btl_mvapi_progress_send_frags(mca_btl_mvapi_endpoint_t*);
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(post_rr_high_endpoint, \
post_rr_high_additional) \
{ \