ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Trying to make IB ptl work with modex_exchange after the

Просмотр исходного кода 
name change of module -> component.

This commit was SVN r1995.
Этот коммит содержится в:
Sayantan Sur 2004-08-10 03:10:40 +00:00
родитель 882c458534
Коммит ba7c1f377e
4 изменённых файлов: 91 добавлений и 65 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
src/mca/ptl/ib/src/ptl_ib.c
Просмотреть файл

@ -74,6 +74,7 @@ int mca_ptl_ib_add_procs(
D_PRINT("Adding %d procs\n", nprocs); D_PRINT("Adding %d procs\n", nprocs);
for(i = 0; i < nprocs; i++) { for(i = 0; i < nprocs; i++) {
ompi_proc = ompi_procs[i]; ompi_proc = ompi_procs[i];

88
src/mca/ptl/ib/src/ptl_ib_component.c
Просмотреть файл

@ -140,7 +140,7 @@ static int mca_ptl_ib_component_send(void)
ud_qp_addr = (mca_ptl_ib_ud_addr_t*) malloc(size); ud_qp_addr = (mca_ptl_ib_ud_addr_t*) malloc(size);
if(NULL == ud_qp_addr) { if(NULL == ud_qp_addr) {
return OMPI_ERROR; return OMPI_ERR_OUT_OF_RESOURCE;
} }
for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) { for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) {
@ -155,6 +155,10 @@ static int mca_ptl_ib_component_send(void)
rc = mca_base_modex_send(&mca_ptl_ib_component.super.ptlm_version, rc = mca_base_modex_send(&mca_ptl_ib_component.super.ptlm_version,
ud_qp_addr, size); ud_qp_addr, size);
if(OMPI_SUCCESS != rc) {
D_PRINT("mca_base_modex_send didn't succeed : %d\n", rc);
}
free(ud_qp_addr); free(ud_qp_addr);
return rc; return rc;
@ -171,11 +175,11 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
bool *allow_multi_user_threads, bool *allow_multi_user_threads,
bool *have_hidden_threads) bool *have_hidden_threads)
{ {
mca_ptl_base_module_t **ptls; mca_ptl_base_module_t **modules;
int i, ret; int i, ret;
uint32_t num_hcas; uint32_t num_hcas;
mca_ptl_ib_module_t* ptl_ib = NULL; mca_ptl_ib_module_t* ib_modules = NULL;
*num_ptl_modules = 0; *num_ptl_modules = 0;
*allow_multi_user_threads = true; *allow_multi_user_threads = true;
@ -225,83 +229,83 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
mca_ptl_ib_component.ib_num_ptl_modules, mca_ptl_ib_component.ib_num_ptl_modules,
mca_ptl_ib_component.ib_max_ptl_modules); mca_ptl_ib_component.ib_max_ptl_modules);
ptl_ib = (mca_ptl_ib_module_t*) malloc(sizeof(mca_ptl_ib_module_t) * ib_modules = (mca_ptl_ib_module_t*) malloc(sizeof(mca_ptl_ib_module_t) *
mca_ptl_ib_component.ib_num_ptl_modules); mca_ptl_ib_component.ib_num_ptl_modules);
if(NULL == ptl_ib) { if(NULL == ib_modules) {
return NULL; return NULL;
} }
/* Zero out the PTL struct memory region */ /* Zero out the PTL struct memory region */
memset((void*)ptl_ib, 0, sizeof(mca_ptl_ib_module_t) * memset((void*)ib_modules, 0, sizeof(mca_ptl_ib_module_t) *
mca_ptl_ib_component.ib_num_ptl_modules); mca_ptl_ib_component.ib_num_ptl_modules);
/* Copy the function pointers to the IB ptls */ /* Copy the function pointers to the IB modules */
for(i = 0; i< mca_ptl_ib_component.ib_num_ptl_modules; i++) { for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) {
memcpy((void*)&ptl_ib[i], memcpy((void*)&ib_modules[i],
&mca_ptl_ib_module, &mca_ptl_ib_module,
sizeof(mca_ptl_ib_module)); sizeof(mca_ptl_ib_module));
} }
D_PRINT("About to initialize IB ptls ...\n"); D_PRINT("About to initialize IB modules ...\n");
/* For each ptl, do this */ /* For each ptl, do this */
for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) { for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) {
if(mca_ptl_ib_get_hca_id(i, &ptl_ib[i].hca_id) if(mca_ptl_ib_get_hca_id(i, &ib_modules[i].hca_id)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
D_PRINT("hca_id: %s\n", ptl_ib[i].hca_id); D_PRINT("hca_id: %s\n", ib_modules[i].hca_id);
if(mca_ptl_ib_get_hca_hndl(ptl_ib[i].hca_id, &ptl_ib[i].nic) if(mca_ptl_ib_get_hca_hndl(ib_modules[i].hca_id, &ib_modules[i].nic)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
D_PRINT("hca_hndl: %d\n", ptl_ib[i].nic); D_PRINT("hca_hndl: %d\n", ib_modules[i].nic);
/* Each HCA uses only port 1. Need to change /* Each HCA uses only port 1. Need to change
* this so that each ptl can choose different * this so that each ptl can choose different
* ports */ * ports */
if(mca_ptl_ib_query_hca_prop(ptl_ib[i].nic, &ptl_ib[i].port) if(mca_ptl_ib_query_hca_prop(ib_modules[i].nic, &ib_modules[i].port)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
D_PRINT("LID: %d\n", ptl_ib[i].port.lid); D_PRINT("LID: %d\n", ib_modules[i].port.lid);
if(mca_ptl_ib_alloc_pd(ptl_ib[i].nic, &ptl_ib[i].ptag) if(mca_ptl_ib_alloc_pd(ib_modules[i].nic, &ib_modules[i].ptag)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
D_PRINT("Protection Domain: %d\n", (int)ptl_ib[i].ptag); D_PRINT("Protection Domain: %d\n", (int)ib_modules[i].ptag);
if(mca_ptl_ib_create_cq(ptl_ib[i].nic, &ptl_ib[i].cq_hndl) if(mca_ptl_ib_create_cq(ib_modules[i].nic, &ib_modules[i].cq_hndl)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
D_PRINT("CQ handle: %d\n", (int)ptl_ib[i].cq_hndl); D_PRINT("CQ handle: %d\n", (int)ib_modules[i].cq_hndl);
if(mca_ptl_ib_ud_cq_init(ptl_ib[i].nic, &ptl_ib[i].ud_scq_hndl, if(mca_ptl_ib_ud_cq_init(ib_modules[i].nic, &ib_modules[i].ud_scq_hndl,
&ptl_ib[i].ud_rcq_hndl) &ib_modules[i].ud_rcq_hndl)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
if(mca_ptl_ib_ud_qp_init(ptl_ib[i].nic, ptl_ib[i].ud_rcq_hndl, if(mca_ptl_ib_ud_qp_init(ib_modules[i].nic, ib_modules[i].ud_rcq_hndl,
ptl_ib[i].ud_scq_hndl, ptl_ib[i].ptag, ib_modules[i].ud_scq_hndl, ib_modules[i].ptag,
ptl_ib[i].ud_qp_hndl, ptl_ib[i].ud_qp_prop) &ib_modules[i].ud_qp_hndl, &ib_modules[i].ud_qp_prop)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
/* Attach asynchronous handler */ /* Attach asynchronous handler */
if(mca_ptl_ib_set_async_handler(ptl_ib[i].nic, if(mca_ptl_ib_set_async_handler(ib_modules[i].nic,
ptl_ib[i].async_handler) &ib_modules[i].async_handler)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
@ -311,34 +315,36 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
* 1. register * 1. register
* 2. fill up descriptors * 2. fill up descriptors
*/ */
if(mca_ptl_ib_prep_ud_bufs(ptl_ib[i].nic, ptl_ib[i].ud_buf) ib_modules[i].ud_buf = NULL;
if(mca_ptl_ib_prep_ud_bufs(ib_modules[i].nic, &ib_modules[i].ud_buf)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
/* Post the UD recv descriptors */ /* Post the UD recv descriptors */
if(mca_ptl_ib_post_ud_recv(ptl_ib[i].nic, ptl_ib[i].ud_qp_hndl, if(mca_ptl_ib_post_ud_recv(ib_modules[i].nic, ib_modules[i].ud_qp_hndl,
ptl_ib[i].ud_buf) ib_modules[i].ud_buf)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
if(mca_ptl_ib_get_comp_ev_hndl(&ptl_ib[i].ud_comp_ev_handler) if(mca_ptl_ib_get_comp_ev_hndl(&ib_modules[i].ud_comp_ev_handler)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
/* Set the completion event handler for the UD recv queue */ /* Set the completion event handler for the UD recv queue */
if(mca_ptl_ib_set_comp_ev_hndl(ptl_ib[i].nic, if(mca_ptl_ib_set_comp_ev_hndl(ib_modules[i].nic,
ptl_ib[i].ud_rcq_hndl, ib_modules[i].ud_rcq_hndl,
ptl_ib[i].ud_comp_ev_handler, ib_modules[i].ud_comp_ev_handler,
(void*)NULL, &ptl_ib[i].ud_comp_ev_hndl) (void*)NULL, &ib_modules[i].ud_comp_ev_hndl)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
/* Request for interrupts on the UD recv queue */ /* Request for interrupts on the UD recv queue */
if(mca_ptl_ib_req_comp_notif(ptl_ib[i].nic, ptl_ib[i].ud_rcq_hndl) if(mca_ptl_ib_req_comp_notif(ib_modules[i].nic, ib_modules[i].ud_rcq_hndl)
!= OMPI_SUCCESS) { != OMPI_SUCCESS) {
return NULL; return NULL;
} }
@ -354,7 +360,7 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
/* Set the pointers for all IB ptls */ /* Set the pointers for all IB ptls */
for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) { for(i = 0; i < mca_ptl_ib_component.ib_num_ptl_modules; i++) {
mca_ptl_ib_component.ib_ptl_modules[i] = &ptl_ib[i]; mca_ptl_ib_component.ib_ptl_modules[i] = &ib_modules[i];
} }
if(mca_ptl_ib_component_send() != OMPI_SUCCESS) { if(mca_ptl_ib_component_send() != OMPI_SUCCESS) {
@ -362,19 +368,19 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
} }
/* Allocate list of MCA ptl pointers */ /* Allocate list of MCA ptl pointers */
ptls = (mca_ptl_base_module_t**) malloc(mca_ptl_ib_component.ib_num_ptl_modules * modules = (mca_ptl_base_module_t**) malloc(mca_ptl_ib_component.ib_num_ptl_modules *
sizeof(mca_ptl_base_module_t*)); sizeof(mca_ptl_base_module_t*));
if(NULL == ptls) { if(NULL == modules) {
return NULL; return NULL;
} }
memcpy(ptls, mca_ptl_ib_component.ib_ptl_modules, memcpy(modules, mca_ptl_ib_component.ib_ptl_modules,
mca_ptl_ib_component.ib_num_ptl_modules * mca_ptl_ib_component.ib_num_ptl_modules *
sizeof(mca_ptl_ib_module_t*)); sizeof(mca_ptl_ib_module_t*));
*num_ptl_modules = mca_ptl_ib_component.ib_num_ptl_modules; *num_ptl_modules = mca_ptl_ib_component.ib_num_ptl_modules;
return ptls; return modules;
} }
/* /*

59
src/mca/ptl/ib/src/ptl_ib_priv.c
Просмотреть файл

@ -45,7 +45,15 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
static void ud_completion_handler(VAPI_hca_hndl_t nic, static void ud_completion_handler(VAPI_hca_hndl_t nic,
VAPI_cq_hndl_t cq_hndl, void* priv_data) VAPI_cq_hndl_t cq_hndl, void* priv_data)
{ {
VAPI_ret_t ret;
D_PRINT("Got interrupt!!\n"); D_PRINT("Got interrupt!!\n");
ret = VAPI_req_comp_notif(nic, cq_hndl, VAPI_NEXT_COMP);
if(VAPI_OK != ret) {
MCA_PTL_IB_VAPI_RET(ret, "VAPI_req_comp_notif");
}
} }
int mca_ptl_ib_ud_cq_init(VAPI_hca_hndl_t nic, int mca_ptl_ib_ud_cq_init(VAPI_hca_hndl_t nic,
@ -89,8 +97,8 @@ int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t nic,
VAPI_cq_hndl_t ud_rcq_hndl, VAPI_cq_hndl_t ud_rcq_hndl,
VAPI_cq_hndl_t ud_scq_hndl, VAPI_cq_hndl_t ud_scq_hndl,
VAPI_pd_hndl_t ptag, VAPI_pd_hndl_t ptag,
VAPI_qp_hndl_t ud_qp_hndl, VAPI_qp_hndl_t* ud_qp_hndl,
VAPI_qp_prop_t ud_qp_prop) VAPI_qp_prop_t* ud_qp_prop)
{ {
VAPI_qp_init_attr_t qp_init_attr; VAPI_qp_init_attr_t qp_init_attr;
VAPI_qp_attr_t qp_attr; VAPI_qp_attr_t qp_attr;
@ -119,7 +127,7 @@ int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t nic,
qp_init_attr.ts_type = VAPI_TS_UD; qp_init_attr.ts_type = VAPI_TS_UD;
ret = VAPI_create_qp(nic, &qp_init_attr, ret = VAPI_create_qp(nic, &qp_init_attr,
&ud_qp_hndl, &ud_qp_prop); ud_qp_hndl, ud_qp_prop);
if(VAPI_OK != ret) { if(VAPI_OK != ret) {
MCA_PTL_IB_VAPI_RET(ret, "VAPI_create_qp"); MCA_PTL_IB_VAPI_RET(ret, "VAPI_create_qp");
@ -127,8 +135,8 @@ int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t nic,
} }
D_PRINT("UD QP[%d] created ..hndl=%d\n", D_PRINT("UD QP[%d] created ..hndl=%d\n",
ud_qp_prop.qp_num, ud_qp_prop->qp_num,
(int)ud_qp_hndl); (int)*ud_qp_hndl);
/* Modifying QP to INIT */ /* Modifying QP to INIT */
QP_ATTR_MASK_CLR_ALL(qp_attr_mask); QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
@ -142,7 +150,7 @@ int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t nic,
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QKEY); QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QKEY);
ret = VAPI_modify_qp(nic, ret = VAPI_modify_qp(nic,
ud_qp_hndl, &qp_attr, (VAPI_qp_hndl_t)*ud_qp_hndl, &qp_attr,
&qp_attr_mask, &qp_cap); &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) { if(VAPI_OK != ret) {
@ -160,7 +168,7 @@ int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t nic,
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE); QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);
ret = VAPI_modify_qp(nic, ret = VAPI_modify_qp(nic,
ud_qp_hndl, &qp_attr, (VAPI_qp_hndl_t)*ud_qp_hndl, &qp_attr,
&qp_attr_mask, &qp_cap); &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) { if(VAPI_OK != ret) {
@ -180,7 +188,7 @@ int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t nic,
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_SQ_PSN); QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_SQ_PSN);
ret = VAPI_modify_qp(nic, ret = VAPI_modify_qp(nic,
ud_qp_hndl, &qp_attr, (VAPI_qp_hndl_t)*ud_qp_hndl, &qp_attr,
&qp_attr_mask, &qp_cap); &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) { if(VAPI_OK != ret) {
@ -218,13 +226,17 @@ int mca_ptl_ib_get_num_hcas(uint32_t* num_hcas)
int mca_ptl_ib_get_hca_id(int num, VAPI_hca_id_t* hca_id) int mca_ptl_ib_get_hca_id(int num, VAPI_hca_id_t* hca_id)
{ {
int num_hcas; uint32_t num_hcas;
VAPI_ret_t ret; VAPI_ret_t ret;
VAPI_hca_id_t* hca_ids = NULL; VAPI_hca_id_t* hca_ids = NULL;
hca_ids = (VAPI_hca_id_t*) malloc(mca_ptl_ib_component.ib_num_hcas * hca_ids = (VAPI_hca_id_t*) malloc(mca_ptl_ib_component.ib_num_hcas *
sizeof(VAPI_hca_id_t)); sizeof(VAPI_hca_id_t));
if(NULL == hca_ids) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Now get the hca_id from underlying VAPI layer */ /* Now get the hca_id from underlying VAPI layer */
ret = EVAPI_list_hcas(mca_ptl_ib_component.ib_num_hcas, ret = EVAPI_list_hcas(mca_ptl_ib_component.ib_num_hcas,
&num_hcas, hca_ids); &num_hcas, hca_ids);
@ -299,7 +311,7 @@ int mca_ptl_ib_alloc_pd(VAPI_hca_hndl_t nic,
int mca_ptl_ib_create_cq(VAPI_hca_hndl_t nic, int mca_ptl_ib_create_cq(VAPI_hca_hndl_t nic,
VAPI_cq_hndl_t* cq_hndl) VAPI_cq_hndl_t* cq_hndl)
{ {
int act_num_cqe = 0; uint32_t act_num_cqe = 0;
VAPI_ret_t ret; VAPI_ret_t ret;
ret = VAPI_create_cq(nic, DEFAULT_CQ_SIZE, ret = VAPI_create_cq(nic, DEFAULT_CQ_SIZE,
@ -314,12 +326,12 @@ int mca_ptl_ib_create_cq(VAPI_hca_hndl_t nic,
} }
int mca_ptl_ib_set_async_handler(VAPI_hca_hndl_t nic, int mca_ptl_ib_set_async_handler(VAPI_hca_hndl_t nic,
EVAPI_async_handler_hndl_t async_handler) EVAPI_async_handler_hndl_t *async_handler)
{ {
VAPI_ret_t ret; VAPI_ret_t ret;
ret = EVAPI_set_async_event_handler(nic, ret = EVAPI_set_async_event_handler(nic,
async_event_handler, 0, &async_handler); async_event_handler, 0, async_handler);
if(VAPI_OK != ret) { if(VAPI_OK != ret) {
MCA_PTL_IB_VAPI_RET(ret, "EVAPI_set_async_event_handler"); MCA_PTL_IB_VAPI_RET(ret, "EVAPI_set_async_event_handler");
@ -330,15 +342,14 @@ int mca_ptl_ib_set_async_handler(VAPI_hca_hndl_t nic,
} }
int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t nic, int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t nic,
mca_ptl_ib_ud_buf_t* ud_buf) mca_ptl_ib_ud_buf_t** ud_buf_ptr)
{ {
int size, len; int size, len;
int i, num_ud_bufs; int i, num_ud_bufs;
vapi_descriptor_t* desc; vapi_descriptor_t* desc;
mca_ptl_ib_ud_buf_data_t* buf_data; mca_ptl_ib_ud_buf_data_t* buf_data;
vapi_memhandle_t* memhandle; vapi_memhandle_t* memhandle;
mca_ptl_ib_ud_buf_t* ud_buf;
ud_buf = NULL;
num_ud_bufs = MAX_UD_PREPOST_DEPTH; num_ud_bufs = MAX_UD_PREPOST_DEPTH;
@ -346,11 +357,14 @@ int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t nic,
len = sizeof(mca_ptl_ib_ud_buf_data_t); len = sizeof(mca_ptl_ib_ud_buf_data_t);
ud_buf = (mca_ptl_ib_ud_buf_t*) malloc(size); (mca_ptl_ib_ud_buf_t*)*ud_buf_ptr = (mca_ptl_ib_ud_buf_t*) malloc(size);
if(NULL == ud_buf) {
return OMPI_ERROR; if(NULL == *ud_buf_ptr) {
return OMPI_ERR_OUT_OF_RESOURCE;
} }
ud_buf = (mca_ptl_ib_ud_buf_t*)*ud_buf_ptr;
/* Walk through the list of UD bufs */ /* Walk through the list of UD bufs */
for(i = 0; i < num_ud_bufs; i++) { for(i = 0; i < num_ud_bufs; i++) {
desc = &ud_buf[i].desc; desc = &ud_buf[i].desc;
@ -359,7 +373,7 @@ int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t nic,
buf_data = (mca_ptl_ib_ud_buf_data_t*) malloc(len); buf_data = (mca_ptl_ib_ud_buf_data_t*) malloc(len);
if(NULL == buf_data) { if(NULL == buf_data) {
return OMPI_ERROR; return OMPI_ERR_OUT_OF_RESOURCE;
} }
if(mca_ptl_ib_register_mem(nic, buf_data, len, memhandle) if(mca_ptl_ib_register_mem(nic, buf_data, len, memhandle)
@ -369,7 +383,7 @@ int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t nic,
desc->rr.comp_type = VAPI_SIGNALED; desc->rr.comp_type = VAPI_SIGNALED;
desc->rr.opcode = VAPI_RECEIVE; desc->rr.opcode = VAPI_RECEIVE;
desc->rr.id = (VAPI_virt_addr_t) &ud_buf[i]; desc->rr.id = (VAPI_virt_addr_t) (unsigned int) &ud_buf[i];
desc->rr.sg_lst_len = 1; desc->rr.sg_lst_len = 1;
desc->rr.sg_lst_p = &(desc->sg_entry); desc->rr.sg_lst_p = &(desc->sg_entry);
desc->sg_entry.len = len; desc->sg_entry.len = len;
@ -457,6 +471,9 @@ int mca_ptl_ib_set_comp_ev_hndl(VAPI_hca_hndl_t nic,
MCA_PTL_IB_VAPI_RET(ret, "EVAPI_set_comp_eventh"); MCA_PTL_IB_VAPI_RET(ret, "EVAPI_set_comp_eventh");
return OMPI_ERROR; return OMPI_ERROR;
} }
D_PRINT("Completion hander: %p, Handle = %d\n",
handler, (int)*handler_hndl);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -479,5 +496,7 @@ int mca_ptl_ib_get_comp_ev_hndl(VAPI_completion_event_handler_t* handler_ptr)
{ {
*handler_ptr = ud_completion_handler; *handler_ptr = ud_completion_handler;
D_PRINT("UD Completion Event Handler = %p\n", ud_completion_handler);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }

8
src/mca/ptl/ib/src/ptl_ib_priv.h
Просмотреть файл

@ -45,8 +45,8 @@ typedef struct mca_ptl_ib_ud_buf_t mca_ptl_ib_ud_buf_t;
int mca_ptl_ib_ud_cq_init(VAPI_hca_hndl_t, VAPI_cq_hndl_t*, int mca_ptl_ib_ud_cq_init(VAPI_hca_hndl_t, VAPI_cq_hndl_t*,
VAPI_cq_hndl_t*); VAPI_cq_hndl_t*);
int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t, VAPI_cq_hndl_t, int mca_ptl_ib_ud_qp_init(VAPI_hca_hndl_t, VAPI_cq_hndl_t,
VAPI_cq_hndl_t, VAPI_pd_hndl_t, VAPI_qp_hndl_t, VAPI_cq_hndl_t, VAPI_pd_hndl_t, VAPI_qp_hndl_t*,
VAPI_qp_prop_t); VAPI_qp_prop_t*);
int mca_ptl_ib_get_num_hcas(uint32_t*); int mca_ptl_ib_get_num_hcas(uint32_t*);
int mca_ptl_ib_get_hca_id(int, VAPI_hca_id_t*); int mca_ptl_ib_get_hca_id(int, VAPI_hca_id_t*);
int mca_ptl_ib_get_hca_hndl(VAPI_hca_id_t, VAPI_hca_hndl_t*); int mca_ptl_ib_get_hca_hndl(VAPI_hca_id_t, VAPI_hca_hndl_t*);
@ -54,10 +54,10 @@ int mca_ptl_ib_query_hca_prop(VAPI_hca_hndl_t, VAPI_hca_port_t*);
int mca_ptl_ib_alloc_pd(VAPI_hca_hndl_t, VAPI_pd_hndl_t*); int mca_ptl_ib_alloc_pd(VAPI_hca_hndl_t, VAPI_pd_hndl_t*);
int mca_ptl_ib_create_cq(VAPI_hca_hndl_t, VAPI_cq_hndl_t*); int mca_ptl_ib_create_cq(VAPI_hca_hndl_t, VAPI_cq_hndl_t*);
int mca_ptl_ib_set_async_handler(VAPI_hca_hndl_t, int mca_ptl_ib_set_async_handler(VAPI_hca_hndl_t,
EVAPI_async_handler_hndl_t); EVAPI_async_handler_hndl_t*);
int mca_ptl_ib_post_ud_recv(VAPI_hca_hndl_t, VAPI_qp_hndl_t, int mca_ptl_ib_post_ud_recv(VAPI_hca_hndl_t, VAPI_qp_hndl_t,
mca_ptl_ib_ud_buf_t*); mca_ptl_ib_ud_buf_t*);
int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t, mca_ptl_ib_ud_buf_t*); int mca_ptl_ib_prep_ud_bufs(VAPI_hca_hndl_t, mca_ptl_ib_ud_buf_t**);
int mca_ptl_ib_register_mem(VAPI_hca_hndl_t, void*, int, int mca_ptl_ib_register_mem(VAPI_hca_hndl_t, void*, int,
vapi_memhandle_t*); vapi_memhandle_t*);
int mca_ptl_ib_set_comp_ev_hndl(VAPI_hca_hndl_t, VAPI_cq_hndl_t, int mca_ptl_ib_set_comp_ev_hndl(VAPI_hca_hndl_t, VAPI_cq_hndl_t,