ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

- ib bmi now creates a seperate module for each ib port, instead of simply

Просмотр исходного кода 
- using port 1 while ignoring others 
- added 2 free lists for varying size frags 
- query queue pair for max inline send for small fragments 

This commit was SVN r6007.
Этот коммит содержится в:
Galen Shipman 2005-06-09 04:37:37 +00:00
родитель 98c0861d78
Коммит b0eb765f22
10 изменённых файлов: 615 добавлений и 256 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

295
src/mca/bmi/ib/bmi_ib.c
Просмотреть файл

@ -29,7 +29,8 @@
#include "bmi_ib_proc.h"
#include "bmi_ib_endpoint.h"
#include "bmi_ib_priv.h"
#include "datatype/convertor.h"
#include "mca/common/vapi/vapi_mem_reg.h"
mca_bmi_ib_module_t mca_bmi_ib_module = {
{
@ -147,14 +148,36 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_alloc(
size_t size)
{
mca_bmi_ib_frag_t* frag;
mca_bmi_ib_module_t* ib_bmi;
int rc;
/* if(size <= mca_bmi_ib_component.first_fragment_size) { */
/* MCA_BMI_IB_FRAG_ALLOC1(frag,rc); */
/* } else { */
/* } */
MCA_BMI_IB_FRAG_ALLOC1(bmi, frag,rc);
frag->segment.seg_len = size;
void * user_out;
ib_bmi = (mca_bmi_ib_module_t*) bmi;
/* if(size <= ib_bmi->super.bmi_eager_limit){ */
if(size <= mca_bmi_ib_component.eager_limit){
MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc);
frag->segment.seg_len =
size <= mca_bmi_ib_component.eager_limit ?
size: mca_bmi_ib_component.eager_limit ;
} else {
MCA_BMI_IB_FRAG_ALLOC_MAX(bmi, frag, rc);
frag->segment.seg_len =
size <= mca_bmi_ib_component.max_send_size ?
size: mca_bmi_ib_component.max_send_size ;
}
/* } else { */
/* frag = (mca_bmi_ib_frag_t*) ib_bmi->ib_pool->mpool_alloc(ib_bmi->ib_pool, sizeof(frag) + sizeof(mca_bmi_ib_header_t) + size ,0, &user_out); */
/* frag->base.super.user_data = user_out; */
/* OBJ_CONSTRUCT(frag, mca_bmi_ib_frag_t); */
/* } */
frag->segment.seg_len = size <= ib_bmi->super.bmi_eager_limit ? size : ib_bmi->super.bmi_eager_limit;
return (mca_bmi_base_descriptor_t*)frag;
}
@ -163,7 +186,14 @@ int mca_bmi_ib_free(
mca_bmi_base_descriptor_t* des)
{
mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*)des;
MCA_BMI_IB_FRAG_RETURN1(bmi, frag);
if(frag->size == 0) {
MCA_BMI_IB_FRAG_RETURN_FRAG(bmi, frag);
} else if(frag->size == mca_bmi_ib_component.max_send_size){
MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag);
} else if(frag->size == mca_bmi_ib_component.eager_limit){
MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag);
}
return frag->rc;
}
@ -182,8 +212,111 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
size_t* size
)
{
mca_bmi_ib_module_t* ib_bmi;
mca_bmi_ib_frag_t* frag;
struct iovec iov;
uint32_t iov_count = 1;
uint32_t max_data = *size;
int32_t free_after;
int rc;
void* user_out;
ib_bmi = (mca_bmi_ib_module_t*) bmi;
if( max_data+reserve <= bmi->bmi_eager_limit) {
MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc);
if(NULL == frag) {
return NULL;
}
if(max_data + reserve > frag->size){
max_data = frag->size - reserve;
}
iov.iov_len = max_data;
iov.iov_base = frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
if( rc < 0 ) {
MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag);
return NULL;
}
frag->segment.seg_len = max_data + reserve;
*size = max_data;
return &frag->base;
}else if( max_data + reserve <= ib_bmi->ib_pin_min || 1 == ompi_convertor_need_buffers( convertor) ){
MCA_BMI_IB_FRAG_ALLOC_MAX(bmi, frag, rc);
if(NULL == frag) {
return NULL;
}
if(max_data + reserve > frag->size){
max_data = frag->size - reserve;
}
iov.iov_len = max_data;
iov.iov_base = frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
if( rc < 0 ) {
MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag);
return NULL;
}
frag->segment.seg_len = max_data + reserve;
*size = max_data;
return &frag->base;
} else {
VAPI_mrw_t mr_in, mr_out;
VAPI_ret_t ret;
mca_common_vapi_memhandle_t mem_hndl;
memset(&mr_in, 0, sizeof(VAPI_mrw_t));
memset(&mr_out, 0, sizeof(VAPI_mrw_t));
memset(&mem_hndl, 0, sizeof(mca_common_vapi_memhandle_t));
mem_hndl.hndl = VAPI_INVAL_HNDL;
mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
mr_in.l_key = 0;
mr_in.r_key = 0;
mr_in.pd_hndl = ib_bmi->ptag;
mr_in.type = VAPI_MR;
frag = (mca_bmi_ib_send_frag_frag_t*) ib_bmi->ib_pool->mpool_alloc(ib_bmi->ib_pool, sizeof(frag) + sizeof(mca_bmi_ib_header_t) + size ,0, &user_out);
frag->base.super.user_data = user_out;
OBJ_CONSTRUCT(frag, mca_bmi_ib_send_frag_frag_t);
iov.iov_len = max_data;
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
mr_in.size = max_data;
mr_in.start = (VAPI_virt_addr_t) (MT_virt_addr_t) iov.iov_base;
ret = VAPI_register_mr(
ib_bmi->nic,
&mr_in,
&mem_hndl.hndl,
&mr_out
);
if(VAPI_OK != ret){
ompi_output(0, "error pinning vapi memory\n");
return NULL;
}
mem_hndl.l_key = mr_out.l_key;
mem_hndl.r_key = mr_out.r_key;
return &frag->base;
}
return NULL;
}
/**
@ -244,3 +377,145 @@ int mca_bmi_ib_put( mca_bmi_base_module_t* bmi,
}
static int mca_bmi_ib_alloc_pd(VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t* ptag)
{
VAPI_ret_t ret;
ret = VAPI_alloc_pd(nic, ptag);
if(ret != VAPI_OK) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_alloc_pd");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static int mca_bmi_ib_create_cq(VAPI_hca_hndl_t nic,
VAPI_cq_hndl_t* cq_hndl)
{
uint32_t act_num_cqe = 0;
VAPI_ret_t ret;
ret = VAPI_create_cq(nic, DEFAULT_CQ_SIZE,
cq_hndl, &act_num_cqe);
if( (VAPI_OK != ret) || (0 == act_num_cqe)) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_create_cq");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/*
* Asynchronous event handler to detect unforseen
* events. Usually, such events are catastrophic.
* Should have a robust mechanism to handle these
* events and abort the OMPI application if necessary.
*
*/
static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
VAPI_event_record_t * event_p,
void *priv_data)
{
switch (event_p->type) {
case VAPI_QP_PATH_MIGRATED:
case VAPI_EEC_PATH_MIGRATED:
case VAPI_QP_COMM_ESTABLISHED:
case VAPI_EEC_COMM_ESTABLISHED:
case VAPI_SEND_QUEUE_DRAINED:
case VAPI_PORT_ACTIVE:
{
D_PRINT("Got an asynchronous event: %s\n",
VAPI_event_record_sym(event_p->type));
break;
}
case VAPI_CQ_ERROR:
case VAPI_LOCAL_WQ_INV_REQUEST_ERROR:
case VAPI_LOCAL_WQ_ACCESS_VIOL_ERROR:
case VAPI_LOCAL_WQ_CATASTROPHIC_ERROR:
case VAPI_PATH_MIG_REQ_ERROR:
case VAPI_LOCAL_EEC_CATASTROPHIC_ERROR:
case VAPI_LOCAL_CATASTROPHIC_ERROR:
case VAPI_PORT_ERROR:
{
ompi_output(0, "Got an asynchronous event: %s (%s)",
VAPI_event_record_sym(event_p->type),
VAPI_event_syndrome_sym(event_p->
syndrome));
break;
}
default:
ompi_output(0, "Warning!! Got an undefined "
"asynchronous event\n");
}
}
static int mca_bmi_ib_set_async_handler(VAPI_hca_hndl_t nic,
EVAPI_async_handler_hndl_t *async_handler)
{
VAPI_ret_t ret;
ret = EVAPI_set_async_event_handler(nic,
async_event_handler, 0, async_handler);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "EVAPI_set_async_event_handler");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_bmi_ib_module_init(mca_bmi_ib_module_t *ib_bmi)
{
/* Get HCA handle */
/* if(mca_bmi_ib_get_hca_hndl(ib_bmi->hca_id, &ib_bmi->nic) */
/* != OMPI_SUCCESS) { */
/* return OMPI_ERROR; */
/* } */
/* Allocate a protection domain for this NIC */
if(mca_bmi_ib_alloc_pd(ib_bmi->nic, &ib_bmi->ptag)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
/* Get the properties of the HCA PORT,
* LID etc. are part of the properties */
/* if(mca_bmi_ib_query_hca_port_prop(ib_bmi->nic, &ib_bmi->port) */
/* != OMPI_SUCCESS) { */
/* return OMPI_ERROR; */
/* } */
/* Create Completion Q */
/* We use a single completion Q for sends & recvs
* This saves us overhead of polling 2 separate Qs */
if(mca_bmi_ib_create_cq(ib_bmi->nic, &ib_bmi->cq_hndl)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
/* Attach asynchronous handler */
if(mca_bmi_ib_set_async_handler(ib_bmi->nic,
&ib_bmi->async_handler)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}

18
src/mca/bmi/ib/bmi_ib.h
Просмотреть файл

@ -90,6 +90,10 @@ struct mca_bmi_ib_component_t {
int ib_rr_buf_min;
/**< the minimum number of posted rr */
size_t eager_limit;
size_t max_send_size;
}; typedef struct mca_bmi_ib_component_t mca_bmi_ib_component_t;
extern mca_bmi_ib_component_t mca_bmi_ib_component;
@ -106,6 +110,7 @@ struct mca_bmi_ib_module_t {
bool bmi_inited;
mca_bmi_ib_registration_t ib_reg[256];
VAPI_hca_id_t hca_id; /**< ID of HCA */
IB_port_t port_id; /**< ID of the PORT */
VAPI_hca_port_t port; /**< IB port of this PTL */
VAPI_hca_hndl_t nic; /**< NIC handle */
VAPI_pd_hndl_t ptag; /**< Protection Domain tag */
@ -114,15 +119,22 @@ struct mca_bmi_ib_module_t {
EVAPI_async_handler_hndl_t async_handler;
/**< Async event handler used to detect weird/unknown events */
ompi_free_list_t send_free; /**< free list of buffer descriptors */
ompi_free_list_t send_free_eager; /**< free list of eager buffer descriptors */
ompi_free_list_t send_free_max; /**< free list of max buffer descriptors */
ompi_free_list_t send_free_frag; /**< free list of frags only... used for pining memory */
ompi_free_list_t recv_free; /**< free list of buffer descriptors */
ompi_list_t repost; /**< list of buffers to repost */
mca_mpool_base_module_t* ib_pool; /**< ib memory pool */
uint32_t rr_posted; /**< number of rr posted to the nic*/
VAPI_rr_desc_t* rr_desc_post;
/**< an array to allow posting of rr in one swoop */
size_t ib_inline_max; /**< max size of inline send*/
size_t ib_pin_min; /** < min size to pin memory*/
}; typedef struct mca_bmi_ib_module_t mca_bmi_ib_module_t;
@ -332,6 +344,10 @@ extern void mca_bmi_ib_send_frag_return(
struct mca_bmi_ib_frag_t*
);
int mca_bmi_ib_module_init(mca_bmi_ib_module_t* ib_bmi);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif

156
src/mca/bmi/ib/bmi_ib_component.c
Просмотреть файл

@ -32,7 +32,10 @@
#include "bmi_ib.h"
#include "bmi_ib_frag.h"
#include "bmi_ib_endpoint.h"
#include "mca/bmi/base/base.h"
#include <vapi.h>
#include <vapi_common.h>
#include "datatype/convertor.h"
mca_bmi_ib_component_t mca_bmi_ib_component = {
{
/* First, the mca_base_component_t struct containing meta information
@ -125,16 +128,18 @@ int mca_bmi_ib_component_open(void)
mca_bmi_ib_param_register_int ("exclusivity", 0);
mca_bmi_ib_module.super.bmi_eager_limit =
mca_bmi_ib_param_register_int ("first_frag_size",
(MCA_BMI_IB_FIRST_FRAG_SIZE
(64*1024
- sizeof(mca_bmi_ib_header_t)));
mca_bmi_ib_module.super.bmi_min_send_size =
mca_bmi_ib_param_register_int ("min_send_size",
(MCA_BMI_IB_FIRST_FRAG_SIZE
(64*1024
- sizeof(mca_bmi_ib_header_t)));
mca_bmi_ib_module.super.bmi_max_send_size =
mca_bmi_ib_param_register_int ("max_send_size", 2<<30);
mca_bmi_ib_param_register_int ("max_send_size", 128*1024);
mca_bmi_ib_component.max_send_size = mca_bmi_ib_module.super.bmi_max_send_size;
mca_bmi_ib_component.eager_limit = mca_bmi_ib_module.super.bmi_eager_limit;
return OMPI_SUCCESS;
}
@ -162,37 +167,95 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
{
VAPI_ret_t vapi_ret;
VAPI_hca_id_t* hca_ids;
VAPI_hca_hndl_t hca_hndl;
VAPI_hca_vendor_t hca_vendor;
VAPI_hca_cap_t hca_cap;
VAPI_hca_port_t hca_port;
uint32_t num_hcas;
mca_bmi_base_module_t** bmis;
int i, length;
int i,j,k, length;
mca_common_vapi_hca_pd_t hca_pd;
ompi_list_t bmi_list;
mca_bmi_ib_module_t * ib_bmi;
mca_bmi_base_selected_module_t* ib_selected;
ompi_list_item_t* item;
/* initialization */
*num_bmi_modules = 0;
/* Determine the number of hca's available on the host */
vapi_ret=EVAPI_list_hcas(0, &(mca_bmi_ib_component.ib_num_bmis), NULL);
if( VAPI_EAGAIN != vapi_ret || 0 == mca_bmi_ib_component.ib_num_bmis ) {
/* Determine the number of hca's available on the host */
vapi_ret=EVAPI_list_hcas(0, &num_hcas, NULL);
if( VAPI_EAGAIN != vapi_ret || 0 == num_hcas ) {
ompi_output(0,"No hca's found on this host \n");
return NULL;
}
/* Allocate space for the hca's */
hca_ids = (VAPI_hca_id_t*) malloc(mca_bmi_ib_component.ib_num_bmis * sizeof(VAPI_hca_id_t));
hca_ids = (VAPI_hca_id_t*) malloc(num_hcas * sizeof(VAPI_hca_id_t));
if(NULL == hca_ids) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return NULL;
}
/* obtain a list of the hca's on this host */
vapi_ret=EVAPI_list_hcas(mca_bmi_ib_component.ib_num_bmis, &mca_bmi_ib_component.ib_num_bmis, hca_ids);
vapi_ret=EVAPI_list_hcas(num_hcas, &num_hcas, hca_ids);
if( VAPI_OK != vapi_ret ) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return NULL;
}
/** We must loop through all the hca id's, get there handles and
for each hca we query the number of ports on the hca and set up
a distinct bmi module for each hca port */
OBJ_CONSTRUCT(&bmi_list, ompi_list_t);
for(i = 0; i < num_hcas; i++){
vapi_ret = EVAPI_get_hca_hndl(hca_ids[i], &hca_hndl);
if(VAPI_OK != vapi_ret) {
ompi_output(0, "%s:error getting hca handle\n", __func__);
return OMPI_ERROR;
}
vapi_ret = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap);
if(VAPI_OK != vapi_ret) {
ompi_output(0, "%s:error getting hca properties\n", __func__);
return OMPI_ERROR;
}
/* Note ports are 1 based hence j = 1 */
for(j = 1; j <= hca_cap.phys_port_num; j++){
vapi_ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t) j, &hca_port);
if(VAPI_OK != vapi_ret) {
ompi_output(0, "%s:error getting hca port properties\n", __func__);
return OMPI_ERROR;
}
if( PORT_ACTIVE == hca_port.state ){
ib_bmi = (mca_bmi_ib_module_t*) malloc(sizeof(mca_bmi_ib_module_t));
memcpy(ib_bmi, &mca_bmi_ib_module, sizeof(mca_bmi_ib_module));
ib_selected = OBJ_NEW(mca_bmi_base_selected_module_t);
ib_selected->bmi_module = (mca_bmi_base_module_t*) ib_bmi;
memcpy(ib_bmi->hca_id, hca_ids[i], sizeof(VAPI_hca_id_t));
ib_bmi->nic = hca_hndl;
ib_bmi->port_id = (IB_port_t) j;
ib_bmi->port = hca_port;
ompi_list_append(&bmi_list, (ompi_list_item_t*) ib_selected);
mca_bmi_ib_component.ib_num_bmis ++;
}
}
}
/* Allocate space for bmi modules */
mca_bmi_ib_component.ib_bmis = (mca_bmi_ib_module_t*) malloc(sizeof(mca_bmi_ib_module_t) *
mca_bmi_ib_component.ib_num_bmis);
if(NULL == mca_bmi_ib_component.ib_bmis) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return NULL;
@ -205,23 +268,32 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
}
for(i = 0; i < mca_bmi_ib_component.ib_num_bmis; i++){
item = ompi_list_remove_first(&bmi_list);
ib_selected = (mca_bmi_base_selected_module_t*)item;
ib_bmi = (mca_bmi_ib_module_t*) ib_selected->bmi_module;
memcpy(&(mca_bmi_ib_component.ib_bmis[i]), ib_bmi , sizeof(mca_bmi_ib_module_t));
free(ib_selected);
free(ib_bmi);
/* Initialize each module */
for(i = 0; i < mca_bmi_ib_component.ib_num_bmis; i++) {
mca_bmi_ib_module_t* ib_bmi = &mca_bmi_ib_component.ib_bmis[i];
ib_bmi = &mca_bmi_ib_component.ib_bmis[i];
/* Initialize the modules function pointers */
memcpy(ib_bmi, &mca_bmi_ib_module, sizeof(mca_bmi_ib_module));
/* Initialize module state */
OBJ_CONSTRUCT(&ib_bmi->send_free, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->send_free_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->send_free_max, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->send_free_frag, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->recv_free, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->repost, ompi_list_t);
memcpy(ib_bmi->hca_id, hca_ids[i], sizeof(ib_bmi->hca_id));
if(mca_bmi_ib_module_init(ib_bmi) != OMPI_SUCCESS) {
free(hca_ids);
return NULL;
@ -234,21 +306,41 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
ib_bmi->ib_pool =
mca_mpool_base_module_create(mca_bmi_ib_component.ib_mpool_name, &hca_pd);
if(NULL == ib_bmi->ib_pool) {
ompi_output(0, "%s: error creating vapi memory pool! aborting ib bmi initialization", __func__);
return NULL;
}
/* Initialize pool of send fragments */
length = sizeof(mca_bmi_ib_frag_t) +
sizeof(mca_bmi_ib_header_t) +
ib_bmi->super.bmi_eager_limit;
ompi_free_list_init(&ib_bmi->send_free,
ompi_free_list_init(&ib_bmi->send_free_eager,
length,
OBJ_CLASS(mca_bmi_ib_send_frag_t),
OBJ_CLASS(mca_bmi_ib_send_frag_eager_t),
mca_bmi_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc,
ib_bmi->ib_pool);
length = sizeof(mca_bmi_ib_frag_t) +
sizeof(mca_bmi_ib_header_t) +
ib_bmi->super.bmi_max_send_size;
ompi_free_list_init(&ib_bmi->send_free_max,
length,
OBJ_CLASS(mca_bmi_ib_send_frag_max_t),
mca_bmi_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc,
ib_bmi->ib_pool);
/* Initialize pool of receive fragments */
ompi_free_list_init (&ib_bmi->recv_free,
length,
@ -257,6 +349,22 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
mca_bmi_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc, ib_bmi->ib_pool);
length = sizeof(mca_bmi_ib_frag_t) +
sizeof(mca_bmi_ib_header_t);
ib_bmi->super.bmi_max_send_size;
ompi_free_list_init(&ib_bmi->send_free_frag,
length,
OBJ_CLASS(mca_bmi_ib_send_frag_frag_t),
mca_bmi_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc,
ib_bmi->ib_pool);
/* Initialize the rr_desc_post array for posting of rr*/
ib_bmi->rr_desc_post = (VAPI_rr_desc_t*) malloc((mca_bmi_ib_component.ib_rr_buf_max * sizeof(VAPI_rr_desc_t)));
@ -351,7 +459,6 @@ int mca_bmi_ib_component_progress()
/* Process a completed send */
frag = (mca_bmi_ib_frag_t*) comp.id;
frag->base.des_cbfunc(&ib_bmi->super,(mca_bmi_base_endpoint_t*) &frag->endpoint, &frag->base, frag->rc);
count++;
break;
@ -362,15 +469,16 @@ int mca_bmi_ib_component_progress()
frag->segment.seg_len = comp.byte_len-sizeof(mca_bmi_ib_header_t);
/* advance the segment address past the header and subtract from the length..*/
ib_bmi->ib_reg[frag->hdr->tag].cbfunc(&ib_bmi->super, frag->hdr->tag, &frag->base, ib_bmi->ib_reg[frag->hdr->tag].cbdata);
OMPI_FREE_LIST_RETURN(&ib_bmi->recv_free, (ompi_free_list_item_t*)comp.id);
OMPI_FREE_LIST_RETURN(&ib_bmi->recv_free, (ompi_free_list_item_t*) frag);
if(OMPI_THREAD_ADD32(&ib_bmi->rr_posted, -1) <= mca_bmi_ib_component.ib_rr_buf_min)
mca_bmi_ib_endpoint_post_rr(mca_bmi_ib_component.ib_rr_buf_max - ib_bmi->rr_posted,
((mca_bmi_ib_recv_frag_t*)comp.id)->endpoint);
((mca_bmi_ib_frag_t*)comp.id)->endpoint);
count++;
break;

15
src/mca/bmi/ib/bmi_ib_endpoint.c
Просмотреть файл

@ -39,7 +39,7 @@ static void mca_bmi_ib_endpoint_construct(mca_bmi_base_endpoint_t* endpoint);
static void mca_bmi_ib_endpoint_destruct(mca_bmi_base_endpoint_t* endpoint);
static inline int mca_bmi_ib_endpoint_post_send(mca_bmi_ib_module_t* ib_bmi, mca_bmi_ib_endpoint_t * endpoint, mca_bmi_ib_send_frag_t * frag)
static inline int mca_bmi_ib_endpoint_post_send(mca_bmi_ib_module_t* ib_bmi, mca_bmi_ib_endpoint_t * endpoint, mca_bmi_ib_frag_t * frag)
{
frag->sr_desc.remote_qkey = 0;
@ -483,11 +483,11 @@ int mca_bmi_ib_endpoint_send(
rc = mca_bmi_ib_endpoint_post_send(ib_bmi, endpoint, frag);
if(ib_bmi->rr_posted <= mca_bmi_ib_component.ib_rr_buf_min+1)
mca_bmi_ib_endpoint_post_rr(mca_bmi_ib_component.ib_rr_buf_max - ib_bmi->rr_posted,
endpoint);
/* rc = mca_bmi_ib_post_send(endpoint->endpoint_bmi, endpoint, */
/* &frag->ib_buf, (void*) frag); */
@ -511,7 +511,7 @@ int mca_bmi_ib_endpoint_send(
void mca_bmi_ib_progress_send_frags(mca_bmi_ib_endpoint_t* endpoint)
{
ompi_list_item_t *frag_item;
mca_bmi_ib_send_frag_t *frag;
mca_bmi_ib_frag_t *frag;
mca_bmi_ib_module_t* ib_bmi;
/*Check if endpoint is connected */
if(endpoint->endpoint_state != MCA_BMI_IB_CONNECTED) {
@ -524,7 +524,7 @@ void mca_bmi_ib_progress_send_frags(mca_bmi_ib_endpoint_t* endpoint)
while(!ompi_list_is_empty(&(endpoint->pending_send_frags))) {
frag_item = ompi_list_remove_first(&(endpoint->pending_send_frags));
frag = (mca_bmi_ib_send_frag_t *) frag_item;
frag = (mca_bmi_ib_frag_t *) frag_item;
ib_bmi = endpoint->endpoint_bmi;
/* We need to post this one */
@ -546,9 +546,10 @@ int mca_bmi_ib_endpoint_connect(
mca_bmi_ib_module_t *ib_bmi = endpoint->endpoint_bmi;
/* Connection establishment RC */
rc = mca_bmi_ib_qp_init(ib_bmi->nic,
endpoint->lcl_qp_hndl,
endpoint->rem_qp_num,
endpoint->rem_lid);
endpoint->lcl_qp_hndl,
endpoint->rem_qp_num,
endpoint->rem_lid,
ib_bmi->port_id);
if(rc != OMPI_SUCCESS) {

4
src/mca/bmi/ib/bmi_ib_endpoint.h
Просмотреть файл

@ -119,14 +119,14 @@ static inline int mca_bmi_ib_endpoint_post_rr(int cnt, mca_bmi_ib_endpoint_t *en
int i, rc;
ompi_list_item_t* item;
mca_bmi_ib_recv_frag_t* frag;
mca_bmi_ib_frag_t* frag;
mca_bmi_ib_module_t *ib_bmi = endpoint->endpoint_bmi;
VAPI_rr_desc_t* rr_desc_post = ib_bmi->rr_desc_post;
/* prepare frags and post receive requests */
for(i = 0; i < cnt; i++) {
OMPI_FREE_LIST_WAIT(&ib_bmi->recv_free, item, rc);
frag = (mca_bmi_ib_recv_frag_t*) item;
frag = (mca_bmi_ib_frag_t*) item;
frag->endpoint = endpoint;
frag->sg_entry.len = frag->size;
rr_desc_post[i] = frag->rr_desc;

92
src/mca/bmi/ib/bmi_ib_frag.c
Просмотреть файл

@ -2,10 +2,11 @@
#include "mca/common/vapi/vapi_mem_reg.h"
static void mca_bmi_ib_send_frag_constructor(mca_bmi_ib_frag_t* frag)
{
static void mca_bmi_ib_frag_common_constructor( mca_bmi_ib_frag_t* frag)
{
mca_common_vapi_memhandle_t* mem_hndl = frag->base.super.user_data;
frag->size = MCA_BMI_IB_FIRST_FRAG_SIZE;
frag->hdr = (mca_bmi_ib_header_t*) (frag+1); /* initialize the bmi header to point to start at end of frag */
frag->segment.seg_addr.pval = frag->hdr+1; /* init the segment address to start after the bmi header */
frag->segment.seg_len = frag->size;
@ -14,49 +15,73 @@ static void mca_bmi_ib_send_frag_constructor(mca_bmi_ib_frag_t* frag)
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
frag->base.des_src->seg_key.key64 = (uint64_t) mem_hndl->l_key;
/* TODO - initialize the firstfrag size */
frag->base.des_src->seg_key.key64 = (uint64_t) mem_hndl->l_key;
frag->sg_entry.lkey = mem_hndl->l_key;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
}
static void mca_bmi_ib_send_frag_common_constructor(mca_bmi_ib_frag_t* frag)
{
mca_bmi_ib_frag_common_constructor(frag);
frag->sr_desc.comp_type = VAPI_SIGNALED;
frag->sr_desc.opcode = VAPI_SEND;
frag->sr_desc.remote_qkey = 0;
frag->sr_desc.sg_lst_len = 1;
frag->sr_desc.sg_lst_p = &frag->sg_entry;
frag->sg_entry.lkey = mem_hndl->l_key;
frag->sr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
}
static void mca_bmi_ib_recv_frag_constructor(mca_bmi_ib_frag_t* frag)
static void mca_bmi_ib_recv_frag_common_constructor(mca_bmi_ib_frag_t* frag)
{
mca_common_vapi_memhandle_t* mem_hndl = frag->base.super.user_data;
frag->size = MCA_BMI_IB_FIRST_FRAG_SIZE;
frag->hdr = (mca_bmi_ib_header_t*) (frag+1); /* initialize the bmi header to point to start at end of frag */
frag->segment.seg_addr.pval = frag->hdr+1; /* init the segment address to start after the bmi header */
frag->segment.seg_len = frag->size;
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
frag->base.des_src->seg_key.key64 = (uint64_t) mem_hndl->l_key;
/* TODO - initialize the first frag size */
mca_bmi_ib_frag_common_constructor(frag);
frag->rr_desc.comp_type = VAPI_SIGNALED;
frag->rr_desc.opcode = VAPI_RECEIVE;
frag->rr_desc.sg_lst_len = 1;
frag->rr_desc.sg_lst_p = &frag->sg_entry;
frag->sg_entry.lkey = mem_hndl->l_key;
frag->rr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
}
static void mca_bmi_ib_send_frag_eager_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = mca_bmi_ib_component.eager_limit;
mca_bmi_ib_send_frag_common_constructor(frag);
}
static void mca_bmi_ib_send_frag_max_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = mca_bmi_ib_component.max_send_size;
mca_bmi_ib_send_frag_common_constructor(frag);
}
static void mca_bmi_ib_recv_frag_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = mca_bmi_ib_component.max_send_size;
mca_bmi_ib_recv_frag_common_constructor(frag);
}
static void mca_bmi_ib_send_frag_frag_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = 0;
mca_bmi_ib_send_frag_common_constructor(frag);
}
OBJ_CLASS_INSTANCE(
mca_bmi_ib_frag_t,
mca_bmi_base_descriptor_t,
@ -64,9 +89,22 @@ OBJ_CLASS_INSTANCE(
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_send_frag_t,
mca_bmi_ib_send_frag_eager_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_send_frag_constructor,
mca_bmi_ib_send_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_send_frag_max_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_send_frag_max_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_send_frag_frag_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_send_frag_frag_constructor,
NULL);
OBJ_CLASS_INSTANCE(

57
src/mca/bmi/ib/bmi_ib_frag.h
Просмотреть файл

@ -64,34 +64,75 @@ struct mca_bmi_ib_frag_t {
mca_bmi_ib_header_t *hdr;
};
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_frag_t;
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_send_frag_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_frag_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_send_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_send_frag_eager_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_send_frag_max_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_send_frag_max_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_send_frag_frag_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_send_frag_frag_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_recv_frag_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_frag_t);
OBJ_CLASS_DECLARATION(mca_bmi_ib_send_frag_t);
OBJ_CLASS_DECLARATION(mca_bmi_ib_recv_frag_t);
/*
* Allocate an IB send descriptor
*
*/
#define MCA_BMI_IB_FRAG_ALLOC1(bmi, frag, rc) \
#define MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc) \
{ \
\
ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_bmi_ib_module_t*)bmi)->send_free, item, rc); \
OMPI_FREE_LIST_WAIT(&((mca_bmi_ib_module_t*)bmi)->send_free_eager, item, rc); \
frag = (mca_bmi_ib_frag_t*) item; \
}
#define MCA_BMI_IB_FRAG_RETURN1(bmi, frag) \
#define MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_bmi_ib_module_t*)bmi)->send_free, (ompi_list_item_t*)(frag)); \
OMPI_FREE_LIST_RETURN(&((mca_bmi_ib_module_t*)bmi)->send_free_eager, (ompi_list_item_t*)(frag)); \
}
#define MCA_BMI_IB_FRAG_ALLOC_MAX(bmi, frag, rc) \
{ \
\
ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_bmi_ib_module_t*)bmi)->send_free_max, item, rc); \
frag = (mca_bmi_ib_frag_t*) item; \
}
#define MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_bmi_ib_module_t*)bmi)->send_free_max, (ompi_list_item_t*)(frag)); \
}
#define MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc) \
{ \
\
ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_bmi_ib_module_t*)bmi)->send_free_frag, item, rc); \
frag = (mca_bmi_ib_frag_t*) item; \
}
#define MCA_BMI_IB_FRAG_RETURN_FRAG(bmi, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_bmi_ib_module_t*)bmi)->send_free_frag, (ompi_list_item_t*)(frag)); \
}
struct mca_bmi_ib_module_t;

223
src/mca/bmi/ib/bmi_ib_priv.c
Просмотреть файл

@ -23,50 +23,6 @@
#include "bmi_ib_frag.h"
#include "bmi_ib_endpoint.h"
/*
* Asynchronous event handler to detect unforseen
* events. Usually, such events are catastrophic.
* Should have a robust mechanism to handle these
* events and abort the OMPI application if necessary.
*
*/
static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
VAPI_event_record_t * event_p,
void *priv_data)
{
switch (event_p->type) {
case VAPI_QP_PATH_MIGRATED:
case VAPI_EEC_PATH_MIGRATED:
case VAPI_QP_COMM_ESTABLISHED:
case VAPI_EEC_COMM_ESTABLISHED:
case VAPI_SEND_QUEUE_DRAINED:
case VAPI_PORT_ACTIVE:
{
D_PRINT("Got an asynchronous event: %s\n",
VAPI_event_record_sym(event_p->type));
break;
}
case VAPI_CQ_ERROR:
case VAPI_LOCAL_WQ_INV_REQUEST_ERROR:
case VAPI_LOCAL_WQ_ACCESS_VIOL_ERROR:
case VAPI_LOCAL_WQ_CATASTROPHIC_ERROR:
case VAPI_PATH_MIG_REQ_ERROR:
case VAPI_LOCAL_EEC_CATASTROPHIC_ERROR:
case VAPI_LOCAL_CATASTROPHIC_ERROR:
case VAPI_PORT_ERROR:
{
ompi_output(0, "Got an asynchronous event: %s (%s)",
VAPI_event_record_sym(event_p->type),
VAPI_event_syndrome_sym(event_p->
syndrome));
break;
}
default:
ompi_output(0, "Warning!! Got an undefined "
"asynchronous event\n");
}
}
/*
* This function returns the hca_id for each BMI
@ -79,87 +35,50 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
*/
static int mca_bmi_ib_get_hca_hndl(VAPI_hca_id_t hca_id,
VAPI_hca_hndl_t* hca_hndl)
{
VAPI_ret_t ret;
/* static int mca_bmi_ib_get_hca_hndl(VAPI_hca_id_t hca_id, */
/* VAPI_hca_hndl_t* hca_hndl) */
/* { */
/* VAPI_ret_t ret; */
/* Open the HCA */
ret = EVAPI_get_hca_hndl(hca_id, hca_hndl);
/* /\* Open the HCA *\/ */
/* ret = EVAPI_get_hca_hndl(hca_id, hca_hndl); */
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "EVAPI_get_hca_hndl");
return OMPI_ERROR;
}
/* if(VAPI_OK != ret) { */
/* MCA_BMI_IB_VAPI_RET(ret, "EVAPI_get_hca_hndl"); */
/* return OMPI_ERROR; */
/* } */
return OMPI_SUCCESS;
}
/* return OMPI_SUCCESS; */
/* } */
static int mca_bmi_ib_query_hca_prop(VAPI_hca_hndl_t nic,
VAPI_hca_port_t* port)
{
VAPI_ret_t ret;
/* Querying for port properties */
ret = VAPI_query_hca_port_prop(nic,
(IB_port_t)DEFAULT_PORT,
port);
/* static int mca_bmi_ib_query_hca_prop(VAPI_hca_hndl_t nic) */
/* { */
/* VAPI_hca_vendor_t hca_vendor; */
/* VAPI_hca_cap_t hca_cap; */
/* VAPI_ret_t ret; */
/* ret = VAPI_query_hca_prop(nic, &hca_vendor, &hca_cap); */
/* hca_cap.phys_port_num; */
/* } */
/* static int mca_bmi_ib_query_hca_port_prop(VAPI_hca_hndl_t nic, */
/* VAPI_hca_port_t* port) */
/* { */
/* VAPI_ret_t ret; */
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_query_hca_port_prop");
return OMPI_ERROR;
}
/* /\* Querying for port properties *\/ */
/* ret = VAPI_query_hca_port_prop(nic, */
/* (IB_port_t)DEFAULT_PORT, */
/* port); */
return OMPI_SUCCESS;
}
/* if(VAPI_OK != ret) { */
/* MCA_BMI_IB_VAPI_RET(ret, "VAPI_query_hca_port_prop"); */
/* return OMPI_ERROR; */
/* } */
static int mca_bmi_ib_alloc_pd(VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t* ptag)
{
VAPI_ret_t ret;
/* return OMPI_SUCCESS; */
/* } */
ret = VAPI_alloc_pd(nic, ptag);
if(ret != VAPI_OK) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_alloc_pd");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static int mca_bmi_ib_create_cq(VAPI_hca_hndl_t nic,
VAPI_cq_hndl_t* cq_hndl)
{
uint32_t act_num_cqe = 0;
VAPI_ret_t ret;
ret = VAPI_create_cq(nic, DEFAULT_CQ_SIZE,
cq_hndl, &act_num_cqe);
if( (VAPI_OK != ret) || (0 == act_num_cqe)) {
MCA_BMI_IB_VAPI_RET(ret, "VAPI_create_cq");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static int mca_bmi_ib_set_async_handler(VAPI_hca_hndl_t nic,
EVAPI_async_handler_hndl_t *async_handler)
{
VAPI_ret_t ret;
ret = EVAPI_set_async_event_handler(nic,
async_event_handler, 0, async_handler);
if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_RET(ret, "EVAPI_set_async_event_handler");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_bmi_ib_create_qp(VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag,
@ -209,49 +128,6 @@ int mca_bmi_ib_create_qp(VAPI_hca_hndl_t nic,
return OMPI_SUCCESS;
}
int mca_bmi_ib_module_init(mca_bmi_ib_module_t *ib_bmi)
{
/* Get HCA handle */
if(mca_bmi_ib_get_hca_hndl(ib_bmi->hca_id, &ib_bmi->nic)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
/* Allocate a protection domain for this NIC */
if(mca_bmi_ib_alloc_pd(ib_bmi->nic, &ib_bmi->ptag)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
/* Get the properties of the HCA,
* LID etc. are part of the properties */
if(mca_bmi_ib_query_hca_prop(ib_bmi->nic, &ib_bmi->port)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
/* Create Completion Q */
/* We use a single completion Q for sends & recvs
* This saves us overhead of polling 2 separate Qs */
if(mca_bmi_ib_create_cq(ib_bmi->nic, &ib_bmi->cq_hndl)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
/* Attach asynchronous handler */
if(mca_bmi_ib_set_async_handler(ib_bmi->nic,
&ib_bmi->async_handler)
!= OMPI_SUCCESS) {
return OMPI_ERROR;
}
/* initialize memory region registry */
/* OBJ_CONSTRUCT(&ib_bmi->mem_registry, mca_bmi_ib_mem_registry_t); */
/* mca_bmi_ib_mem_registry_init(&ib_bmi->mem_registry, ib_bmi); */
return OMPI_SUCCESS;
}
int mca_bmi_ib_qp_query(mca_bmi_ib_module_t* ib_bmi, VAPI_qp_hndl_t qp_hndl, VAPI_qp_num_t qp_num)
{
@ -273,9 +149,10 @@ int mca_bmi_ib_qp_query(mca_bmi_ib_module_t* ib_bmi, VAPI_qp_hndl_t qp_hndl, VAP
}
int mca_bmi_ib_qp_init(VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp,
IB_lid_t remote_lid)
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp,
IB_lid_t remote_lid,
IB_port_t port_id)
{
VAPI_ret_t ret;
VAPI_qp_attr_t qp_attr;
@ -288,7 +165,7 @@ int mca_bmi_ib_qp_init(VAPI_hca_hndl_t nic,
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = DEFAULT_PKEY_IX;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = DEFAULT_PORT;
qp_attr.port = port_id;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
@ -433,21 +310,21 @@ int mca_bmi_ib_qp_init(VAPI_hca_hndl_t nic,
/* } */
void mca_bmi_ib_buffer_repost(VAPI_hca_hndl_t nic, void* addr)
{
/* void mca_bmi_ib_buffer_repost(VAPI_hca_hndl_t nic, void* addr) */
/* { */
mca_bmi_ib_recv_frag_t * frag = (mca_bmi_ib_recv_frag_t*)addr;
/* mca_bmi_ib_recv_frag_t * frag = (mca_bmi_ib_recv_frag_t*)addr; */
frag->sg_entry.len = frag->size;
/* frag->sg_entry.len = frag->size; */
frag->ret = VAPI_post_rr(nic, frag->endpoint->lcl_qp_hndl, &(frag->rr_desc));
/* frag->ret = VAPI_post_rr(nic, frag->endpoint->lcl_qp_hndl, &(frag->rr_desc)); */
if(VAPI_OK != frag->ret) {
MCA_BMI_IB_VAPI_RET(frag->ret, "VAPI_post_rr");
ompi_output(0, "Error in buffer reposting");
}
}
/* if(VAPI_OK != frag->ret) { */
/* MCA_BMI_IB_VAPI_RET(frag->ret, "VAPI_post_rr"); */
/* ompi_output(0, "Error in buffer reposting"); */
/* } */
/* } */
/* void mca_bmi_ib_prepare_ack(mca_bmi_ib_module_t *ib_bmi, */
/* void* addr_to_reg, int len_to_reg, */

4
src/mca/bmi/ib/bmi_ib_priv.h
Просмотреть файл

@ -27,7 +27,6 @@
#define NUM_IB_SEND_BUF (1)
#define NUM_IB_RECV_BUF (4)
#define MCA_BMI_IB_FIRST_FRAG_SIZE (65536)
typedef enum {
IB_RECV,
@ -217,6 +216,7 @@ int mca_bmi_ib_qp_init(
VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp,
IB_lid_t remote_lid);
IB_lid_t remote_lid,
IB_port_t port_id);
#endif /* MCA_BMI_IB_PRIV_H */

7
src/mca/mpool/vapi/mpool_vapi_component.c
Просмотреть файл

@ -96,14 +96,17 @@ static int mca_mpool_vapi_open(void)
void* mca_common_vapi_segment_alloc(size_t* size, void* user_in, void** user_out){
mca_mpool_vapi_module_t * mpool_module = (mca_mpool_vapi_module_t*)user_in;
void* addr = (void*)malloc((*size));
VAPI_mrw_t mr_in, mr_out;
memset(&mr_in, 0, sizeof(VAPI_mrw_t));
memset(&mr_out, 0, sizeof(VAPI_mrw_t));
VAPI_ret_t ret;
mca_common_vapi_memhandle_t* mem_hndl;
memset(&mr_in, 0, sizeof(VAPI_mrw_t));
memset(&mr_out, 0, sizeof(VAPI_mrw_t));
*user_out = (void*) malloc(sizeof(mca_common_vapi_memhandle_t));
mem_hndl = (mca_common_vapi_memhandle_t*) *user_out;