ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

More code cleanup.. Also converted post receive requests to macros..

Просмотр исходного кода 
This commit was SVN r6566.
Этот коммит содержится в:
Galen Shipman 2005-07-20 17:43:31 +00:00
родитель 946402b980
Коммит fd969ac833
15 изменённых файлов: 472 добавлений и 402 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

67
ompi/mca/btl/base/btl_base_error.h
Просмотреть файл

@ -20,13 +20,13 @@
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_ERROR(fmt, ...) { \
opal_output(0, "[%s:%d:%d] my_name: [%lu,%lu,%lu] " fmt, __FILE__, __LINE__, __func__, \
opal_output(0, "[%s:%d:%d] my_name: [%lu,%lu,%lu] " fmt "\n", __FILE__, __LINE__, __func__, \
ORTE_NAME_ARGS(orte_process_info.my_name), __VA_ARGS__); \
}
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_ERROR(fmt, args...) { \
opal_output(0, "[%s:%d:%d] my_name: [%lu,%lu,%lu]" fmt, __FILE__, __LINE__, __func__,\
opal_output(0, "[%s:%d:%d] my_name: [%lu,%lu,%lu]" fmt "\n", __FILE__, __LINE__, __func__,\
ORTE_NAME_ARGS(orte_process_info.my_name), ##args); \
}
#else
@ -38,43 +38,40 @@ static inline void BTL_ERROR(char *fmt, ... )
ORTE_NAME_ARGS(orte_process_info.my_name), list);
va_end(list);
}
#define BTL_ERROR printf
#endif
#endif
#ifdef BTL_DEBUG_OUT
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_DEBUG_OUT(fmt, ...) { \
opal_output(0, "[%s:%d:%d " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \
#if 0
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_DEBUG_OUT(fmt, ...) { \
opal_output(0, "[%s:%d:%d] " fmt "\n", __FILE__, __LINE__, __func__, __VA_ARGS__); \
}
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_DEBUG_OUT(fmt, args...) { \
opal_output(0, "[%s:%d:%d] " fmt "\n", __FILE__, __LINE__, __func__, ##args); \
}
#else
static inline void BTL_DEBUG_OUT(char *fmt, ... )
{
va_list list;
va_start(list, fmt);
opal_output(0, "[%s:%d:%d ", fmt, __FILE__, __LINE__, __func__, list);
va_end(list);
}
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_DEBUG_OUT(fmt, args...) { \
opal_output(0, "[%s:%d:%d " fmt, __FILE__, __LINE__, __func__, ##args); \
}
#endif
#endif
#else
static inline void BTL_DEBUG_OUT(char *fmt, ... )
{
va_list list;
va_start(list, fmt);
opal_output(0, "[%s:%d:%d ", fmt, __FILE__, __LINE__, __func__, list);
va_end(list);
}
#define BTL_DEBUG_OUT printf
#endif
#endif
#else
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_DEBUG_OUT(fmt, ...)
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_DEBUG_OUT(fmt, args...)
#else
static inline void BTL_DEBUG_OUT(char *fmt, ... )
{
}
#define BTL_DEBUG_OUT printf
#endif
#endif
#if defined(ACCEPT_C99) && __STDC_VERSION__ >= 199901L
# define BTL_DEBUG_OUT(fmt, ...)
#else
# if defined(__GNUC__) && !defined(__STDC__)
#define BTL_DEBUG_OUT(fmt, args...)
#else
static inline void BTL_DEBUG_OUT(char *fmt, ... )
{
}
#endif
#endif
#endif
#endif

3
ompi/mca/btl/mvapi/Makefile.am
Просмотреть файл

@ -30,8 +30,7 @@ sources = \
btl_mvapi_frag.c \
btl_mvapi_frag.h \
btl_mvapi_proc.c \
btl_mvapi_proc.h \
btl_mvapi_error.h
btl_mvapi_proc.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la

208
ompi/mca/btl/mvapi/btl_mvapi.c
Просмотреть файл

@ -30,6 +30,7 @@
#include "mca/mpool/base/base.h"
#include "mca/mpool/mpool.h"
#include "mca/mpool/mvapi/mpool_mvapi.h"
#include "mca/btl/base/btl_base_error.h"
mca_btl_mvapi_module_t mca_btl_mvapi_module = {
{
@ -58,8 +59,11 @@ mca_btl_mvapi_module_t mca_btl_mvapi_module = {
}
};
/*
* add a proc to this btl module
* creates an endpoint that is setup on the
* first send to the endpoint
*/
int mca_btl_mvapi_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
@ -114,23 +118,29 @@ int mca_btl_mvapi_add_procs(
return OMPI_SUCCESS;
}
/*
* delete the proc as reachable from this btl module
*/
int mca_btl_mvapi_del_procs(struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t ** peers)
{
/* Stub */
DEBUG_OUT("Stub\n");
BTL_DEBUG_OUT("Stub\n");
return OMPI_SUCCESS;
}
/*
*Register callback function to support send/recv semantics
*/
int mca_btl_mvapi_register(
struct mca_btl_base_module_t* btl,
mca_btl_base_tag_t tag,
mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata)
{
/* TODO add register stuff here... */
mca_btl_mvapi_module_t* mvapi_btl = (mca_btl_mvapi_module_t*) btl;
@ -142,11 +152,15 @@ int mca_btl_mvapi_register(
}
/**
* Allocate a segment.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*
* When allocating a segment we pull a pre-alllocated segment
* from one of two free lists, an eager list and a max list
*/
mca_btl_base_descriptor_t* mca_btl_mvapi_alloc(
struct mca_btl_base_module_t* btl,
@ -176,9 +190,11 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_alloc(
}
/**
* Return a segment
*
*
*/
* Return the segment to the appropriate
* preallocated segment list
*/
int mca_btl_mvapi_free(
struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des)
@ -188,23 +204,38 @@ int mca_btl_mvapi_free(
if(frag->size == 0) {
OBJ_RELEASE(frag->vapi_reg);
MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag);
}
else if(frag->size == mca_btl_mvapi_component.max_send_size){
} else if(frag->size == mca_btl_mvapi_component.max_send_size){
MCA_BTL_IB_FRAG_RETURN_MAX(btl, frag);
} else if(frag->size == mca_btl_mvapi_component.eager_limit){
MCA_BTL_IB_FRAG_RETURN_EAGER(btl, frag);
}
} else {
BTL_ERROR("invalid descriptor");
}
return OMPI_SUCCESS;
}
/**
* Pack data and return a descriptor that can be
* register user buffer or pack
* data into pre-registered buffer and return a
* descriptor that can be
* used for send/put.
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
* @param endpoint (IN) BTL peer addressing
*
* prepare source's behavior depends on the following:
* Has a valid memory registration been passed to prepare_src?
* if so we attempt to use the pre-registred user-buffer, if the memory registration
* is to small (only a portion of the user buffer) then we must reregister the user buffer
* Has the user requested the memory to be left pinned?
* if so we insert the memory registration into a memory tree for later lookup, we
* may also remove a previous registration if a MRU (most recently used) list of
* registions is full, this prevents resources from being exhausted.
* Is the requested size larger than the btl's max send size?
* if so and we aren't asked to leave the registration pinned than we register the memory if
* the users buffer is contiguous
* Otherwise we choose from two free lists of pre-registered memory in which to pack the data into.
*
*/
mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
struct mca_btl_base_module_t* btl,
@ -227,14 +258,12 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
mvapi_btl = (mca_btl_mvapi_module_t*) btl;
vapi_reg = (mca_mpool_mvapi_registration_t*) registration;
/** if the data fits in the eager limit and we aren't told to pinn then we
simply pack, if the data fits in the eager limit and the data is non contiguous
then we pack **/
if(NULL != vapi_reg && 0 == ompi_convertor_need_buffers(convertor)){
bool is_leave_pinned = vapi_reg->base_reg.is_leave_pinned;
size_t reg_len;
/* the memory is already pinned and we have contiguous user data */
MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){
return NULL;
@ -245,27 +274,27 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
/* first we will try to find this address in the memory tree (from MPI_Alloc_mem) */
frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
size_t reg_len;
reg_len = (unsigned char*)vapi_reg->base_reg.bound - (unsigned char*)iov.iov_base + 1;
if(frag->segment.seg_len > reg_len) {
/* the pinned region is too small! we have to re-pinn it */
size_t new_len = vapi_reg->base_reg.bound - vapi_reg->base_reg.base + 1
+ frag->segment.seg_len - reg_len;
void * base_addr = vapi_reg->base_reg.base;
rc = mca_mpool_base_remove((void*) vapi_reg->base_reg.base);
if(OMPI_SUCCESS != rc) {
opal_output(0, "%s:%d:%s error removing memory region from memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing memory region from memory pool tree");
return NULL;
}
if(is_leave_pinned) {
if(NULL == opal_list_remove_item(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg)){
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
}
@ -286,19 +315,21 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
if(rc != OMPI_SUCCESS) {
opal_output(0,"%s:%d:%s error inserting memory region into memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error inserting memory region into memory pool tree");
return NULL;
}
OBJ_RETAIN(vapi_reg);
if(is_leave_pinned) {
/* we should leave the memory pinned so put the memory on the MRU list */
vapi_reg->base_reg.is_leave_pinned = is_leave_pinned;
opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
}
}
else if(is_leave_pinned) {
if(NULL == opal_list_remove_item(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg)) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
/* the current memory region is large enough and we should leave the memory pinned */
if(NULL == opal_list_remove_item(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg)) {
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
@ -327,6 +358,10 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
ompi_convertor_need_buffers(convertor) == 0 &&
reserve == 0)
{
/* The user buffer is contigous and we need to leave the buffer pinned or we are asked to send
more than the max send size. Note that the memory was not already pinned because we have
no registration information passed to us */
MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){
return NULL;
@ -344,14 +379,19 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
if(mca_btl_mvapi_component.leave_pinned) {
/* so we need to leave pinned which means we must check and see if we
have room on the MRU list */
if(mca_btl_mvapi_component.reg_mru_len <= mvapi_btl->reg_mru_list.opal_list_length ) {
/* we have the maximum number of entries on the MRU list, time to
pull something off and make room. */
mca_mpool_mvapi_registration_t* old_reg =
(mca_mpool_mvapi_registration_t*)
opal_list_remove_first(&mvapi_btl->reg_mru_list);
if( NULL == old_reg) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
@ -359,7 +399,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
rc = mca_mpool_base_remove((void*) old_reg->base_reg.base);
if(OMPI_SUCCESS != rc) {
opal_output(0,"%s:%d:%s error removing memory region from memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing memory region from memory pool tree");
return NULL;
}
@ -385,6 +425,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
} else {
/* we don't need to leave the memory pinned so just register it.. */
mvapi_btl->ib_pool->mpool_register(mvapi_btl->ib_pool,
iov.iov_base,
max_data,
@ -392,7 +433,6 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
vapi_reg->base_reg.is_leave_pinned = false;
}
/* frag->mem_hndl = vapi_reg->hndl; */
frag->sg_entry.len = max_data;
frag->sg_entry.lkey = vapi_reg->l_key;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) iov.iov_base;
@ -408,7 +448,10 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
return &frag->base;
} else if (max_data+reserve <= btl->btl_eager_limit) {
/* the data is small enough to fit in the eager frag and
either we received no prepinned memory or leave pinned is
not set
*/
MCA_BTL_IB_FRAG_ALLOC_EAGER(btl, frag, rc);
if(NULL == frag) {
return NULL;
@ -434,12 +477,10 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
return &frag->base;
}
/** if the data fits in the max limit and we aren't told to pinn then we
} else if(max_data + reserve <= mvapi_btl->super.btl_max_send_size) {
/** if the data fits in the max limit and we aren't told to pinn then we
simply pack, if the data is non contiguous then we pack **/
else if(max_data + reserve <= mvapi_btl->super.btl_max_send_size) {
MCA_BTL_IB_FRAG_ALLOC_MAX(btl, frag, rc);
if(NULL == frag) {
return NULL;
@ -471,11 +512,21 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
return NULL;
}
/**
* Pack data
* Prepare the dst buffer
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
* prepare dest's behavior depends on the following:
* Has a valid memory registration been passed to prepare_src?
* if so we attempt to use the pre-registred user-buffer, if the memory registration
* is to small (only a portion of the user buffer) then we must reregister the user buffer
* Has the user requested the memory to be left pinned?
* if so we insert the memory registration into a memory tree for later lookup, we
* may also remove a previous registration if a MRU (most recently used) list of
* registions is full, this prevents resources from being exhausted.
*/
mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
struct mca_btl_base_module_t* btl,
@ -506,23 +557,30 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
frag->base.des_flags = 0;
if(NULL!= vapi_reg){
/* the memory is already pinned try to use it if the pinned region is large enough*/
reg_len = (unsigned char*)vapi_reg->base_reg.bound - (unsigned char*)frag->segment.seg_addr.pval + 1;
bool is_leave_pinned = vapi_reg->base_reg.is_leave_pinned;
if(frag->segment.seg_len > reg_len ) {
/* the pinned region is too small! we have to re-pinn it */
size_t new_len = vapi_reg->base_reg.bound - vapi_reg->base_reg.base + 1
+ frag->segment.seg_len - reg_len;
void * base_addr = vapi_reg->base_reg.base;
rc = mca_mpool_base_remove((void*) vapi_reg->base_reg.base);
if(OMPI_SUCCESS != rc) {
opal_output(0,"%s:%d:%s error removing memory region from memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing memory region from memory pool tree");
return NULL;
}
if(is_leave_pinned) {
/* the memory we just un-pinned was marked as leave pinned,
* pull it off the MRU list
*/
if(NULL == opal_list_remove_item(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg)) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
}
@ -541,20 +599,22 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
(mca_mpool_base_registration_t*) vapi_reg);
if(OMPI_SUCCESS != rc) {
opal_output(0,"%s:%d:%s error inserting memory region into memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error inserting memory region into memory pool tree");
return NULL;
}
OBJ_RETAIN(vapi_reg);
if(is_leave_pinned) {
/* we should leave the memory pinned so put the memory on the MRU list */
vapi_reg->base_reg.is_leave_pinned = is_leave_pinned;
opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
}
}
else if(is_leave_pinned){
/* the current memory region is large enough and we should leave the memory pinned */
if(NULL == opal_list_remove_item(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg)) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
@ -562,24 +622,30 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
}
OBJ_RETAIN(vapi_reg);
} else {
/* we didn't get a memory registration passed in, so we have to register the region
* ourselves
*/
if(mca_btl_mvapi_component.leave_pinned) {
/* so we need to leave pinned which means we must check and see if we
have room on the MRU list */
if( mca_btl_mvapi_component.reg_mru_len <= mvapi_btl->reg_mru_list.opal_list_length ) {
/* we have the maximum number of entries on the MRU list, time to
pull something off and make room. */
mca_mpool_mvapi_registration_t* old_reg =
(mca_mpool_mvapi_registration_t*)
opal_list_remove_first(&mvapi_btl->reg_mru_list);
if( NULL == old_reg) {
opal_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
}
rc = mca_mpool_base_remove((void*) old_reg->base_reg.base);
if(OMPI_SUCCESS !=rc ) {
opal_output(0,"%s:%d:%s error removing memory region from memory pool tree", __FILE__, __LINE__, __func__);
BTL_ERROR("error removing memory region from memory pool tree");
return NULL;
}
@ -599,7 +665,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
(void*) (&mvapi_btl->super),
(mca_mpool_base_registration_t*) vapi_reg);
if(OMPI_SUCCESS != rc){
opal_output(0,"%s:%d:%s error inserting memory region into memory pool", __FILE__, __LINE__, __func__);
BTL_ERROR("error inserting memory region into memory pool");
return NULL;
}
@ -607,6 +673,8 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
} else {
/* we don't need to leave the memory pinned so just register it.. */
mvapi_btl->ib_pool->mpool_register(mvapi_btl->ib_pool,
frag->segment.seg_addr.pval,
*size,
@ -616,9 +684,6 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
}
/* frag->mem_hndl = vapi_reg->hndl; */
frag->sg_entry.len = *size;
frag->sg_entry.lkey = vapi_reg->l_key;
@ -640,7 +705,8 @@ int mca_btl_mvapi_finalize(struct mca_btl_base_module_t* btl)
{
mca_btl_mvapi_module_t* mvapi_btl;
mvapi_btl = (mca_btl_mvapi_module_t*) btl;
#if 0
if(mvapi_btl->send_free_eager.fl_num_allocated !=
mvapi_btl->send_free_eager.super.opal_list_length){
opal_output(0, "btl ib send_free_eager frags: %d allocated %d returned \n",
@ -673,15 +739,13 @@ int mca_btl_mvapi_finalize(struct mca_btl_base_module_t* btl)
mvapi_btl->recv_free_max.fl_num_allocated,
mvapi_btl->recv_free_max.super.opal_list_length);
}
#endif
return OMPI_SUCCESS;
}
/*
* Initiate a send. If this is the first fragment, use the fragment
* descriptor allocated with the send requests, otherwise obtain
* one from the free list. Initialize the fragment and foward
* on to the peer.
* Initiate a send.
*/
int mca_btl_mvapi_send(
@ -726,7 +790,8 @@ int mca_btl_mvapi_put( mca_btl_base_module_t* btl,
if(VAPI_OK != frag->ret){
return OMPI_ERROR;
}
mca_btl_mvapi_endpoint_post_rr(endpoint, 1);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(endpoint, 1);
return OMPI_SUCCESS;
@ -753,7 +818,7 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
case VAPI_SEND_QUEUE_DRAINED:
case VAPI_PORT_ACTIVE:
{
DEBUG_OUT("Got an asynchronous event: %s\n",
BTL_DEBUG_OUT("Got an asynchronous event: %s\n",
VAPI_event_record_sym(event_p->type));
break;
}
@ -766,22 +831,23 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
case VAPI_LOCAL_CATASTROPHIC_ERROR:
case VAPI_PORT_ERROR:
{
opal_output(0, "Got an asynchronous event: %s (%s)",
VAPI_event_record_sym(event_p->type),
VAPI_event_syndrome_sym(event_p->
syndrome));
BTL_ERROR("Got an asynchronous event: %s (%s)",
VAPI_event_record_sym(event_p->type),
VAPI_event_syndrome_sym(event_p->syndrome));
break;
}
default:
opal_output(0, "Warning!! Got an undefined "
"asynchronous event\n");
BTL_ERROR("Warning!! Got an undefined "
"asynchronous event");
}
}
/*
* Initialize the btl module by allocating a protection domain
* and creating both the high and low priority completion queues
*/
int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
{
@ -792,7 +858,7 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
ret = VAPI_alloc_pd(mvapi_btl->nic, &mvapi_btl->ptag);
if(ret != VAPI_OK) {
MCA_BTL_IB_VAPI_ERROR(ret, "VAPI_alloc_pd");
BTL_ERROR("error in VAPI_alloc_pd: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}
@ -801,7 +867,7 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
if( VAPI_OK != ret) {
MCA_BTL_IB_VAPI_ERROR(ret, "VAPI_create_cq");
BTL_ERROR("error in VAPI_create_cq: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}
@ -810,13 +876,13 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
if( VAPI_OK != ret) {
MCA_BTL_IB_VAPI_ERROR(ret, "VAPI_create_cq");
BTL_ERROR("error in VAPI_create_cq: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}
if(cqe_cnt <= 0) {
opal_output(0, "%s: error creating completion queue ", __func__);
BTL_ERROR("error creating completion queue ");
return OMPI_ERROR;
}
@ -824,7 +890,7 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl)
async_event_handler, 0, &mvapi_btl->async_handler);
if(VAPI_OK != ret) {
MCA_BTL_IB_VAPI_ERROR(ret, "EVAPI_set_async_event_handler");
BTL_ERROR("error in EVAPI_set_async_event_handler: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}

9
ompi/mca/btl/mvapi/btl_mvapi.h
Просмотреть файл

@ -31,12 +31,17 @@
#include "mca/pml/pml.h"
#include "mca/btl/btl.h"
#include "opal/util/output.h"
#include "mca/mpool/mpool.h"
#include "btl_mvapi_error.h"
#include "mca/mpool/mpool.h"
#include "mca/btl/base/btl_base_error.h"
#include "mca/btl/btl.h"
#include "mca/btl/base/base.h"
#include <vapi.h>
#include <mtl_common.h>
#include <vapi_common.h>
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif

62
ompi/mca/btl/mvapi/btl_mvapi_component.c
Просмотреть файл

@ -110,7 +110,6 @@ int mca_btl_mvapi_component_open(void)
/* initialize objects */
OBJ_CONSTRUCT(&mca_btl_mvapi_component.ib_procs, opal_list_t);
/* OBJ_CONSTRUCT (&mca_btl_mvapi_component.ib_recv_frags, ompi_free_list_t); */
/* register IB component parameters */
mca_btl_mvapi_component.ib_free_list_num =
@ -206,12 +205,6 @@ int mca_btl_mvapi_component_open(void)
mca_base_param_lookup_int(param, &value);
mca_btl_mvapi_component.leave_pinned = value;
mca_btl_mvapi_component.max_send_size = mca_btl_mvapi_module.super.btl_max_send_size;
mca_btl_mvapi_component.eager_limit = mca_btl_mvapi_module.super.btl_eager_limit;
@ -259,7 +252,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
/* Determine the number of hca's available on the host */
vapi_ret=EVAPI_list_hcas(0, &num_hcas, NULL);
if( VAPI_EAGAIN != vapi_ret || 0 == num_hcas ) {
opal_output(0,"No hca's found on this host \n");
BTL_ERROR("No hca's found on this host!");
return NULL;
}
@ -288,14 +281,14 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
for(i = 0; i < num_hcas; i++){
vapi_ret = EVAPI_get_hca_hndl(hca_ids[i], &hca_hndl);
if(VAPI_OK != vapi_ret) {
opal_output(0, "%s:error getting hca handle\n", __func__);
BTL_ERROR("error getting hca handle: %s", VAPI_strerror(vapi_ret));
return NULL;
}
vapi_ret = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap);
if(VAPI_OK != vapi_ret) {
opal_output(0, "%s:error getting hca properties\n", __func__);
BTL_ERROR("error getting hca properties %s", VAPI_strerror(vapi_ret));
return NULL;
}
@ -304,7 +297,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
for(j = 1; j <= hca_cap.phys_port_num; j++){
vapi_ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t) j, &hca_port);
if(VAPI_OK != vapi_ret) {
opal_output(0, "%s:error getting hca port properties\n", __func__);
BTL_ERROR("error getting hca port properties %s", VAPI_strerror(vapi_ret));
return NULL;
}
@ -389,7 +382,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
&hca_pd);
if(NULL == mvapi_btl->ib_pool) {
opal_output(0, "%s: error creating vapi memory pool! aborting ib btl initialization", __func__);
BTL_ERROR("error creating vapi memory pool! aborting mvapi btl initialization");
return NULL;
}
/* Initialize pool of send fragments */
@ -462,11 +455,6 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
/* Initialize the rr_desc_post array for posting of rr*/
mvapi_btl->rr_desc_post = (VAPI_rr_desc_t*) malloc((mca_btl_mvapi_component.ib_rr_buf_max * sizeof(VAPI_rr_desc_t)));
/* This is now done by the memory pool passed to free_list_init.. Initialize the send descriptors */
/* if(mca_btl_mvapi_send_frag_register(mvapi_btl) != OMPI_SUCCESS) { */
/* free(hca_ids); */
/* return NULL; */
/* } */
btls[i] = &mvapi_btl->super;
}
@ -504,20 +492,18 @@ int mca_btl_mvapi_component_progress()
ret = VAPI_poll_cq(mvapi_btl->nic, mvapi_btl->cq_hndl_high, &comp);
if(VAPI_OK == ret) {
if(comp.status != VAPI_SUCCESS) {
opal_output(0, "Got error : %s, Vendor code : %d Frag : %p",
VAPI_wc_status_sym(comp.status),
comp.vendor_err_syndrome, comp.id);
BTL_ERROR("Got error : %s, Vendor code : %d Frag : %p",
VAPI_wc_status_sym(comp.status),
comp.vendor_err_syndrome, comp.id);
return OMPI_ERROR;
}
/* Handle work completions */
switch(comp.opcode) {
case VAPI_CQE_RQ_RDMA_WITH_IMM:
if(comp.imm_data_valid){
opal_output(0, "Got an RQ_RDMA_WITH_IMM!\n");
}
break;
BTL_ERROR("Got an RDMA with Immediate data!, not supported!");
return OMPI_ERROR;
case VAPI_CQE_SQ_RDMA_WRITE:
case VAPI_CQE_SQ_SEND_DATA :
@ -531,9 +517,9 @@ int mca_btl_mvapi_component_progress()
case VAPI_CQE_RQ_SEND_DATA:
/* Process a RECV */
DEBUG_OUT("Got an recv completion" );
BTL_DEBUG_OUT("Got a recv completion");
frag = (mca_btl_mvapi_frag_t*) comp.id;
endpoint = (mca_btl_endpoint_t*) frag->endpoint;
endpoint = (mca_btl_mvapi_endpoint_t*) frag->endpoint;
frag->rc=OMPI_SUCCESS;
frag->segment.seg_len = comp.byte_len-((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);
@ -542,14 +528,13 @@ int mca_btl_mvapi_component_progress()
OMPI_FREE_LIST_RETURN(&(mvapi_btl->recv_free_eager), (opal_list_item_t*) frag);
OPAL_THREAD_ADD32(&endpoint->rr_posted_high, -1);
mca_btl_mvapi_endpoint_post_rr(((mca_btl_mvapi_frag_t*)comp.id)->endpoint, 0);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(((mca_btl_mvapi_frag_t*)comp.id)->endpoint, 0);
count++;
break;
default:
opal_output(0, "Unhandled work completion opcode is %d", comp.opcode);
BTL_ERROR("Unhandled work completion opcode is %d", comp.opcode);
break;
}
}
@ -559,7 +544,7 @@ int mca_btl_mvapi_component_progress()
ret = VAPI_poll_cq(mvapi_btl->nic, mvapi_btl->cq_hndl_low, &comp);
if(VAPI_OK == ret) {
if(comp.status != VAPI_SUCCESS) {
opal_output(0, "Got error : %s, Vendor code : %d Frag : %p",
BTL_ERROR("Got error : %s, Vendor code : %d Frag : %p",
VAPI_wc_status_sym(comp.status),
comp.vendor_err_syndrome, comp.id);
return OMPI_ERROR;
@ -567,6 +552,10 @@ int mca_btl_mvapi_component_progress()
/* Handle n/w completions */
switch(comp.opcode) {
case VAPI_CQE_RQ_RDMA_WITH_IMM:
BTL_ERROR("Got an RDMA with Immediate data!, not supported!");
return OMPI_ERROR;
case VAPI_CQE_SQ_RDMA_WRITE:
case VAPI_CQE_SQ_SEND_DATA :
@ -579,10 +568,9 @@ int mca_btl_mvapi_component_progress()
case VAPI_CQE_RQ_SEND_DATA:
DEBUG_OUT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
BTL_DEBUG_OUT("Got a recv completion");
frag = (mca_btl_mvapi_frag_t*) comp.id;
endpoint = (mca_btl_endpoint_t*) frag->endpoint;
endpoint = (mca_btl_mvapi_endpoint_t*) frag->endpoint;
frag->rc=OMPI_SUCCESS;
frag->segment.seg_len = comp.byte_len-((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);
/* advance the segment address past the header and subtract from the length..*/
@ -591,14 +579,14 @@ int mca_btl_mvapi_component_progress()
OMPI_FREE_LIST_RETURN(&(mvapi_btl->recv_free_max), (opal_list_item_t*) frag);
OPAL_THREAD_ADD32(&endpoint->rr_posted_low, -1);
mca_btl_mvapi_endpoint_post_rr(((mca_btl_mvapi_frag_t*)comp.id)->endpoint, 0);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(((mca_btl_mvapi_frag_t*)comp.id)->endpoint, 0);
count++;
break;
default:
opal_output(0, "Errorneous network completion");
BTL_ERROR("Errorneous network completion");
break;
}
}

151
ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
Просмотреть файл

@ -88,7 +88,8 @@ static inline int mca_btl_mvapi_endpoint_post_send(mca_btl_mvapi_module_t* mvapi
if(VAPI_OK != frag->ret)
return OMPI_ERROR;
mca_btl_mvapi_endpoint_post_rr(endpoint, 1);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(endpoint, 1);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(endpoint, 1);
return OMPI_SUCCESS;
}
@ -178,7 +179,7 @@ static int mca_btl_mvapi_endpoint_send_connect_req(mca_btl_base_endpoint_t* endp
mca_btl_mvapi_endpoint_send_cb, NULL);
DEBUG_OUT("Sending High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
BTL_DEBUG_OUT("Sending High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_prop_high.qp_num,
endpoint->lcl_qp_prop_low.qp_num,
endpoint->endpoint_btl->port.lid);
@ -254,7 +255,7 @@ static int mca_btl_mvapi_endpoint_set_remote_info(mca_btl_base_endpoint_t* endpo
ORTE_ERROR_LOG(rc);
return rc;
}
DEBUG_OUT("Received High Priority QP num = %d, Low Priority QP num %d, LID = %d",
BTL_DEBUG_OUT("Received High Priority QP num = %d, Low Priority QP num %d, LID = %d",
endpoint->rem_qp_num_high,
endpoint->rem_qp_num_low,
endpoint->rem_lid);
@ -286,8 +287,7 @@ static int mca_btl_mvapi_endpoint_start_connect(mca_btl_base_endpoint_t* endpoin
&endpoint->lcl_qp_hndl_high,
&endpoint->lcl_qp_prop_high,
VAPI_TS_RC))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
@ -299,22 +299,21 @@ static int mca_btl_mvapi_endpoint_start_connect(mca_btl_base_endpoint_t* endpoin
endpoint->endpoint_btl->cq_hndl_low,
&endpoint->lcl_qp_hndl_low,
&endpoint->lcl_qp_prop_low,
VAPI_TS_RC))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
VAPI_TS_RC))) {
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
BTL_DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_prop_high.qp_num,
endpoint->lcl_qp_prop_low.qp_num,
mvapi_btl->port.lid);
endpoint->endpoint_btl->port.lid);
/* Send connection info over to remote endpoint */
endpoint->endpoint_state = MCA_BTL_IB_CONNECTING;
if(OMPI_SUCCESS != (rc = mca_btl_mvapi_endpoint_send_connect_req(endpoint))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error sending connect request, error code %d", rc);
return rc;
}
return OMPI_SUCCESS;
@ -337,8 +336,7 @@ static int mca_btl_mvapi_endpoint_reply_start_connect(mca_btl_mvapi_endpoint_t *
&endpoint->lcl_qp_hndl_high,
&endpoint->lcl_qp_prop_high,
VAPI_TS_RC))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
@ -351,15 +349,14 @@ static int mca_btl_mvapi_endpoint_reply_start_connect(mca_btl_mvapi_endpoint_t *
&endpoint->lcl_qp_hndl_low,
&endpoint->lcl_qp_prop_low,
VAPI_TS_RC))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
BTL_DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_prop_high.qp_num,
endpoint->lcl_qp_prop_low.qp_num,
mvapi_btl->port.lid);
endpoint->endpoint_btl->port.lid);
@ -371,15 +368,13 @@ static int mca_btl_mvapi_endpoint_reply_start_connect(mca_btl_mvapi_endpoint_t *
rc = mca_btl_mvapi_endpoint_connect(endpoint);
if(rc != OMPI_SUCCESS) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error in endpoint connect error code is %d", rc);
return rc;
}
/* Send connection info over to remote endpoint */
if(OMPI_SUCCESS != (rc = mca_btl_mvapi_endpoint_send_connect_req(endpoint))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error in endpoint send connect request error code is %d", rc);
return rc;
}
return OMPI_SUCCESS;
@ -435,48 +430,46 @@ static void mca_btl_mvapi_endpoint_recv(
/* Update status */
switch(endpoint_state) {
case MCA_BTL_IB_CLOSED :
/* We had this connection closed before.
* The endpoint is trying to connect. Move the
* status of this connection to CONNECTING,
* and then reply with our QP information */
case MCA_BTL_IB_CLOSED :
/* We had this connection closed before.
* The endpoint is trying to connect. Move the
* status of this connection to CONNECTING,
* and then reply with our QP information */
if(OMPI_SUCCESS != (rc = mca_btl_mvapi_endpoint_reply_start_connect(ib_endpoint, buffer))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
break;
}
/* Setup state as connected */
ib_endpoint->endpoint_state = MCA_BTL_IB_CONNECT_ACK;
if(OMPI_SUCCESS != (rc = mca_btl_mvapi_endpoint_reply_start_connect(ib_endpoint, buffer))) {
BTL_ERROR("error in endpoint reply start connect");
break;
}
case MCA_BTL_IB_CONNECTING :
/* Setup state as connected */
ib_endpoint->endpoint_state = MCA_BTL_IB_CONNECT_ACK;
break;
mca_btl_mvapi_endpoint_set_remote_info(ib_endpoint, buffer);
if(OMPI_SUCCESS != (rc = mca_btl_mvapi_endpoint_connect(ib_endpoint))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
break;
}
case MCA_BTL_IB_CONNECTING :
/* Setup state as connected */
mca_btl_mvapi_endpoint_connected(ib_endpoint);
/* Send him an ack */
mca_btl_mvapi_endpoint_send_connect_ack(ib_endpoint);
mca_btl_mvapi_endpoint_set_remote_info(ib_endpoint, buffer);
if(OMPI_SUCCESS != (rc = mca_btl_mvapi_endpoint_connect(ib_endpoint))) {
BTL_ERROR("endpoint connect error: %d", rc);
break;
}
case MCA_BTL_IB_CONNECT_ACK:
/* Setup state as connected */
mca_btl_mvapi_endpoint_connected(ib_endpoint);
mca_btl_mvapi_endpoint_connected(ib_endpoint);
/* Send him an ack */
mca_btl_mvapi_endpoint_send_connect_ack(ib_endpoint);
break;
break;
case MCA_BTL_IB_CONNECT_ACK:
case MCA_BTL_IB_CONNECTED :
break;
default :
opal_output(0, "Connected -> Connecting not possible.\n");
mca_btl_mvapi_endpoint_connected(ib_endpoint);
break;
case MCA_BTL_IB_CONNECTED :
break;
default :
BTL_ERROR("Invalid endpoint state %d", endpoint_state);
}
break;
@ -490,8 +483,6 @@ static void mca_btl_mvapi_endpoint_recv(
void mca_btl_mvapi_post_recv()
{
DEBUG_OUT("");
orte_rml.recv_buffer_nb(
ORTE_RML_NAME_ANY,
ORTE_RML_TAG_DYNAMIC-1,
@ -519,7 +510,7 @@ int mca_btl_mvapi_endpoint_send(
switch(endpoint->endpoint_state) {
case MCA_BTL_IB_CONNECTING:
DEBUG_OUT("Queing because state is connecting");
BTL_DEBUG_OUT("Queing because state is connecting");
opal_list_append(&endpoint->pending_send_frags,
(opal_list_item_t *)frag);
@ -529,7 +520,7 @@ int mca_btl_mvapi_endpoint_send(
case MCA_BTL_IB_CONNECT_ACK:
DEBUG_OUT("Queuing because waiting for ack");
BTL_DEBUG_OUT("Queuing because waiting for ack");
opal_list_append(&endpoint->pending_send_frags,
(opal_list_item_t *)frag);
@ -539,7 +530,7 @@ int mca_btl_mvapi_endpoint_send(
case MCA_BTL_IB_CLOSED:
DEBUG_OUT("Connection to endpoint closed ... connecting ...");
BTL_DEBUG_OUT("Connection to endpoint closed ... connecting ...");
opal_list_append(&endpoint->pending_send_frags,
(opal_list_item_t *)frag);
@ -558,10 +549,10 @@ int mca_btl_mvapi_endpoint_send(
mvapi_btl = endpoint->endpoint_btl;
DEBUG_OUT("Send to : %d, len : %d, frag : %p",
endpoint->endpoint_proc->proc_guid.vpid,
frag->ib_buf.desc.sg_entry.len,
frag);
BTL_DEBUG_OUT("Send to : %d, len : %d, frag : %p",
endpoint->endpoint_proc->proc_guid.vpid,
frag->sg_entry.len,
frag);
rc = mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag);
@ -598,7 +589,7 @@ void mca_btl_mvapi_progress_send_frags(mca_btl_mvapi_endpoint_t* endpoint)
/* We need to post this one */
if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag))
opal_output(0, "error in mca_btl_mvapi_endpoint_send");
BTL_ERROR("error in mca_btl_mvapi_endpoint_send");
}
}
@ -632,12 +623,18 @@ int mca_btl_mvapi_endpoint_connect(
return rc;
}
mca_btl_mvapi_endpoint_post_rr(endpoint, 0);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(endpoint, 0);
MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(endpoint, 0);
return OMPI_SUCCESS;
}
/*
* Create the queue pair note that this is just the initial
* queue pair creation and we need to get the remote queue pair
* info from the peer before the qp is usable,
*/
int mca_btl_mvapi_endpoint_create_qp(
mca_btl_mvapi_module_t* mvapi_btl,
@ -686,12 +683,17 @@ int mca_btl_mvapi_endpoint_create_qp(
qp_hndl, qp_prop);
if(VAPI_OK != ret) {
MCA_BTL_IB_VAPI_ERROR(ret, "VAPI_create_qp");
BTL_ERROR("error creating the queue pair: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/*
* The queue pair has been created and we have received the remote
* queue pair information from the peer so we init this queue pair
* and are ready to roll.
*/
int mca_btl_mvapi_endpoint_qp_init_query(
mca_btl_mvapi_module_t* mvapi_btl,
@ -727,11 +729,11 @@ int mca_btl_mvapi_endpoint_qp_init_query(
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BTL_IB_VAPI_ERROR(ret, "VAPI_modify_qp");
BTL_ERROR("Error modifying the queue pair: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}
DEBUG_OUT("Modified to init..Qp %d", qp_hndl);
BTL_DEBUG_OUT("Modified to init..Qp %d", qp_hndl);
/********************** INIT --> RTR ************************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
@ -762,11 +764,11 @@ int mca_btl_mvapi_endpoint_qp_init_query(
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BTL_IB_VAPI_ERROR(ret, "VAPI_modify_qp");
BTL_ERROR("Error modifying the queue pair: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}
DEBUG_OUT("Modified to RTR..Qp %d", qp_hndl);
BTL_DEBUG_OUT("Modified to RTR..Qp %d", qp_hndl);
/************** RTS *******************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
@ -787,14 +789,13 @@ int mca_btl_mvapi_endpoint_qp_init_query(
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
MCA_BTL_IB_VAPI_ERROR(ret, "VAPI_modify_qp");
return OMPI_ERROR;
}
DEBUG_OUT("Modified to RTS..Qp %d", qp_hndl);
BTL_DEBUG_OUT("Modified to RTS..Qp %d", qp_hndl);
ret = VAPI_query_qp(nic, qp_hndl, &qp_attr, &qp_attr_mask, &qp_init_attr );
if (ret != VAPI_OK) {
opal_output(0, "error querying the queue pair");
if (ret != VAPI_OK) {
BTL_ERROR("Error modifying the queue pair: %s", VAPI_strerror(ret));
return OMPI_ERROR;
}

227
ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
Просмотреть файл

@ -125,97 +125,154 @@ void mca_btl_mvapi_post_recv(void);
void mca_btl_mvapi_progress_send_frags(mca_btl_mvapi_endpoint_t*);
static inline int mca_btl_mvapi_endpoint_post_rr_sub(int cnt,
mca_btl_mvapi_endpoint_t* endpoint,
ompi_free_list_t* frag_list,
uint32_t* rr_posted,
VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl
)
{
int rc, i;
opal_list_item_t* item;
mca_btl_mvapi_frag_t* frag;
mca_btl_mvapi_module_t *mvapi_btl = endpoint->endpoint_btl;
VAPI_rr_desc_t* rr_desc_post = mvapi_btl->rr_desc_post;
/* prepare frags and post receive requests */
for(i = 0; i < cnt; i++) {
OMPI_FREE_LIST_WAIT(frag_list, item, rc);
frag = (mca_btl_mvapi_frag_t*) item;
frag->endpoint = endpoint;
frag->sg_entry.len = frag->size + ((unsigned char*) frag->segment.seg_addr.pval- (unsigned char*) frag->hdr); /* sizeof(mca_btl_mvapi_header_t); */
rr_desc_post[i] = frag->rr_desc;
}
frag->ret = EVAPI_post_rr_list(nic,
qp_hndl,
cnt,
rr_desc_post);
if(VAPI_OK != frag->ret) {
MCA_BTL_IB_VAPI_ERROR(frag->ret, "EVAPI_post_rr_list");
return OMPI_ERROR;
}
OPAL_THREAD_ADD32(rr_posted, cnt);
return OMPI_SUCCESS;
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(post_rr_high_endpoint, \
post_rr_high_additional) \
{ \
mca_btl_mvapi_module_t * post_rr_high_mvapi_btl = post_rr_high_endpoint->endpoint_btl; \
OPAL_THREAD_LOCK(&post_rr_high_mvapi_btl->ib_lock); \
if(post_rr_high_endpoint->rr_posted_high <= mca_btl_mvapi_component.ib_rr_buf_min+post_rr_high_additional && \
post_rr_high_endpoint->rr_posted_high < mca_btl_mvapi_component.ib_rr_buf_max){ \
MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(mca_btl_mvapi_component.ib_rr_buf_max - \
post_rr_high_endpoint->rr_posted_high, \
post_rr_high_endpoint, \
&post_rr_high_mvapi_btl->recv_free_eager, \
&post_rr_high_endpoint->rr_posted_high, \
post_rr_high_mvapi_btl->nic, \
post_rr_high_endpoint->lcl_qp_hndl_high); \
} \
OPAL_THREAD_UNLOCK(&post_rr_high_mvapi_btl->ib_lock); \
}
static inline int mca_btl_mvapi_endpoint_post_rr( mca_btl_mvapi_endpoint_t * endpoint, int additional){
mca_btl_mvapi_module_t * mvapi_btl = endpoint->endpoint_btl;
int rc;
OPAL_THREAD_LOCK(&endpoint->ib_lock);
if(endpoint->rr_posted_high <= mca_btl_mvapi_component.ib_rr_buf_min+additional && endpoint->rr_posted_high < mca_btl_mvapi_component.ib_rr_buf_max){
rc = mca_btl_mvapi_endpoint_post_rr_sub(mca_btl_mvapi_component.ib_rr_buf_max - endpoint->rr_posted_high,
endpoint,
&mvapi_btl->recv_free_eager,
&endpoint->rr_posted_high,
mvapi_btl->nic,
endpoint->lcl_qp_hndl_high
);
if(rc != OMPI_SUCCESS){
OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock);
return rc;
}
}
if(endpoint->rr_posted_low <= mca_btl_mvapi_component.ib_rr_buf_min+additional && endpoint->rr_posted_low < mca_btl_mvapi_component.ib_rr_buf_max){
rc = mca_btl_mvapi_endpoint_post_rr_sub(mca_btl_mvapi_component.ib_rr_buf_max - endpoint->rr_posted_low,
endpoint,
&mvapi_btl->recv_free_max,
&endpoint->rr_posted_low,
mvapi_btl->nic,
endpoint->lcl_qp_hndl_low
);
if(rc != OMPI_SUCCESS) {
OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock);
return rc;
}
}
OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock);
return OMPI_SUCCESS;
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_LOW(post_rr_low_endpoint, \
post_rr_low_additional) \
{ \
mca_btl_mvapi_module_t * post_rr_low_mvapi_btl = post_rr_low_endpoint->endpoint_btl; \
OPAL_THREAD_LOCK(&post_rr_low_mvapi_btl->ib_lock); \
if(post_rr_low_endpoint->rr_posted_low <= mca_btl_mvapi_component.ib_rr_buf_min+post_rr_low_additional && \
post_rr_low_endpoint->rr_posted_low < mca_btl_mvapi_component.ib_rr_buf_max){ \
MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(mca_btl_mvapi_component.ib_rr_buf_max - \
post_rr_low_endpoint->rr_posted_low, \
post_rr_low_endpoint, \
&post_rr_low_mvapi_btl->recv_free_max, \
&post_rr_low_endpoint->rr_posted_low, \
post_rr_low_mvapi_btl->nic, \
post_rr_low_endpoint->lcl_qp_hndl_low); \
} \
OPAL_THREAD_UNLOCK(&post_rr_low_mvapi_btl->ib_lock); \
}
#define DUMP_ENDPOINT(endpoint_ptr) { \
opal_output(0, "[%s:%d] ", __FILE__, __LINE__); \
opal_output(0, "Dumping endpoint %d state", \
endpoint->endpoint_proc->proc_guid.vpid); \
opal_output(0, "Local QP hndl : %d", \
endpoint_ptr->endpoint_conn->lres->qp_hndl); \
opal_output(0, "Local QP num : %d", \
endpoint_ptr->endpoint_conn->lres->qp_prop.qp_num); \
opal_output(0, "Remote QP num : %d", \
endpoint_ptr->endpoint_conn->rres->qp_num); \
opal_output(0, "Remote LID : %d", \
endpoint_ptr->endpoint_conn->rres->lid); \
#define MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(post_rr_sub_cnt, \
post_rr_sub_endpoint, \
post_rr_sub_frag_list, \
post_rr_sub_rr_posted, \
post_rr_sub_nic, \
post_rr_sub_qp ) \
{\
uint32_t post_rr_sub_i; \
int post_rr_sub_rc; \
opal_list_item_t* post_rr_sub_item; \
mca_btl_mvapi_frag_t* post_rr_sub_frag; \
mca_btl_mvapi_module_t *post_rr_sub_mvapi_btl = post_rr_sub_endpoint->endpoint_btl; \
VAPI_rr_desc_t* post_rr_sub_desc_post = post_rr_sub_mvapi_btl->rr_desc_post; \
for(post_rr_sub_i = 0; post_rr_sub_i < post_rr_sub_cnt; post_rr_sub_i++) { \
OMPI_FREE_LIST_WAIT(post_rr_sub_frag_list, post_rr_sub_item, post_rr_sub_rc); \
post_rr_sub_frag = (mca_btl_mvapi_frag_t*) post_rr_sub_item; \
post_rr_sub_frag->endpoint = post_rr_sub_endpoint; \
post_rr_sub_frag->sg_entry.len = post_rr_sub_frag->size + \
((unsigned char*) post_rr_sub_frag->segment.seg_addr.pval- \
(unsigned char*) post_rr_sub_frag->hdr); \
post_rr_sub_desc_post[post_rr_sub_i] = post_rr_sub_frag->rr_desc; \
}\
post_rr_sub_frag->ret = EVAPI_post_rr_list( post_rr_sub_nic, \
post_rr_sub_qp, \
post_rr_sub_cnt, \
post_rr_sub_desc_post); \
if(VAPI_OK != post_rr_sub_frag->ret) { \
BTL_ERROR("error posting receive descriptors: %s",\
VAPI_strerror(post_rr_sub_frag->ret)); \
} else {\
OPAL_THREAD_ADD32(post_rr_sub_rr_posted, post_rr_sub_cnt); \
}\
}
/* static inline int mca_btl_mvapi_endpoint_post_rr_sub(int cnt, */
/* mca_btl_mvapi_endpoint_t* endpoint, */
/* ompi_free_list_t* frag_list, */
/* uint32_t* rr_posted, */
/* VAPI_hca_hndl_t nic, */
/* VAPI_qp_hndl_t qp_hndl */
/* ) */
/* { */
/* int rc, i; */
/* opal_list_item_t* item; */
/* mca_btl_mvapi_frag_t* frag; */
/* mca_btl_mvapi_module_t *mvapi_btl = endpoint->endpoint_btl; */
/* VAPI_rr_desc_t* rr_desc_post = mvapi_btl->rr_desc_post; */
/* /\* prepare frags and post receive requests *\/ */
/* for(i = 0; i < cnt; i++) { */
/* OMPI_FREE_LIST_WAIT(frag_list, item, rc); */
/* frag = (mca_btl_mvapi_frag_t*) item; */
/* frag->endpoint = endpoint; */
/* frag->sg_entry.len = frag->size + ((unsigned char*) frag->segment.seg_addr.pval- (unsigned char*) frag->hdr); /\* sizeof(mca_btl_mvapi_header_t); *\/ */
/* rr_desc_post[i] = frag->rr_desc; */
/* } */
/* frag->ret = EVAPI_post_rr_list(nic, */
/* qp_hndl, */
/* cnt, */
/* rr_desc_post); */
/* if(VAPI_OK != frag->ret) { */
/* BTL_ERROR("error posting receive descriptors: %s", VAPI_strerror(frag->ret)); */
/* return OMPI_ERROR; */
/* } */
/* OPAL_THREAD_ADD32(rr_posted, cnt); */
/* return OMPI_SUCCESS; */
/* } */
/* static inline int mca_btl_mvapi_endpoint_post_rr( mca_btl_mvapi_endpoint_t * endpoint, int additional){ */
/* mca_btl_mvapi_module_t * mvapi_btl = endpoint->endpoint_btl; */
/* int rc; */
/* OPAL_THREAD_LOCK(&endpoint->ib_lock); */
/* if(endpoint->rr_posted_high <= mca_btl_mvapi_component.ib_rr_buf_min+additional && endpoint->rr_posted_high < mca_btl_mvapi_component.ib_rr_buf_max){ */
/* rc = mca_btl_mvapi_endpoint_post_rr_sub(mca_btl_mvapi_component.ib_rr_buf_max - endpoint->rr_posted_high, */
/* endpoint, */
/* &mvapi_btl->recv_free_eager, */
/* &endpoint->rr_posted_high, */
/* mvapi_btl->nic, */
/* endpoint->lcl_qp_hndl_high */
/* ); */
/* if(rc != OMPI_SUCCESS){ */
/* OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock); */
/* return rc; */
/* } */
/* } */
/* if(endpoint->rr_posted_low <= mca_btl_mvapi_component.ib_rr_buf_min+additional && endpoint->rr_posted_low < mca_btl_mvapi_component.ib_rr_buf_max){ */
/* rc = mca_btl_mvapi_endpoint_post_rr_sub(mca_btl_mvapi_component.ib_rr_buf_max - endpoint->rr_posted_low, */
/* endpoint, */
/* &mvapi_btl->recv_free_max, */
/* &endpoint->rr_posted_low, */
/* mvapi_btl->nic, */
/* endpoint->lcl_qp_hndl_low */
/* ); */
/* if(rc != OMPI_SUCCESS) { */
/* OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock); */
/* return rc; */
/* } */
/* } */
/* OPAL_THREAD_UNLOCK(&mvapi_btl->ib_lock); */
/* return OMPI_SUCCESS; */
/* } */
#if defined(c_plusplus) || defined(__cplusplus)

43
ompi/mca/btl/mvapi/btl_mvapi_error.h
Просмотреть файл

@ -1,43 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_IB_ERROR_H
#define MCA_BTL_IB_ERROR_H
#include <vapi.h>
#include <mtl_common.h>
#include <vapi_common.h>
/*
*
*
*/
#define MCA_BTL_IB_VAPI_ERROR(vapi_ret, func_name) { \
opal_output(0,"[%s:%d] ", __FILE__, __LINE__); \
opal_output(0,"%s : %s",func_name,VAPI_strerror(vapi_ret)); \
}
/* Debug Print */
#if 0
#define DEBUG_OUT(fmt, args...) { \
opal_output(0, "[%s:%d:%s] " fmt, __FILE__, __LINE__, __func__, \
##args); \
}
#else
#define DEBUG_OUT(fmt, args...)
#endif
#endif

17
ompi/mca/btl/mvapi/btl_mvapi_frag.c
Просмотреть файл

@ -1,3 +1,20 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "btl_mvapi_frag.h"
#include "mca/common/vapi/vapi_mem_reg.h"
#include "mca/mpool/mvapi/mpool_mvapi.h"

3
ompi/mca/btl/openib/Makefile.am
Просмотреть файл

@ -27,8 +27,7 @@ sources = \
btl_openib_frag.c \
btl_openib_frag.h \
btl_openib_proc.c \
btl_openib_proc.h \
btl_openib_error.h
btl_openib_proc.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la

32
ompi/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -139,14 +139,13 @@ int mca_btl_openib_del_procs(struct mca_btl_base_module_t* btl,
/*
*Register callback function to support send/recv semantics
*/
int mca_btl_openib_register(
struct mca_btl_base_module_t* btl,
mca_btl_base_tag_t tag,
mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata)
{
/* TODO add register stuff here... */
mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*) btl;
@ -273,7 +272,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
bool is_leave_pinned = openib_reg->base_reg.is_leave_pinned;
size_t reg_len;
/* the memory is already pinned and we have contiguous user data */
/* the memory is already pinned and we have contiguous user data */ o
MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){
@ -526,7 +525,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
}
/**
* Pack data
* Prepare the dst buffer
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
@ -568,11 +567,13 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
frag->base.des_flags = 0;
if(NULL!= openib_reg){
/* the memory is already pinned try to use it if the pinned region is large enough*/
bool is_leave_pinned = openib_reg->base_reg.is_leave_pinned;
reg_len = (unsigned char*)openib_reg->base_reg.bound - (unsigned char*)frag->segment.seg_addr.pval + 1;
if(frag->segment.seg_len > reg_len ) {
/* the pinned region is too small! we have to re-pinn it */
size_t new_len = openib_reg->base_reg.bound - openib_reg->base_reg.base + 1
+ frag->segment.seg_len - reg_len;
void * base_addr = openib_reg->base_reg.base;
@ -584,6 +585,9 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
}
if(is_leave_pinned) {
/* the memory we just un-pinned was marked as leave pinned,
* pull it off the MRU list
*/
if(NULL == opal_list_remove_item(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg)) {
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
@ -610,12 +614,14 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
OBJ_RETAIN(openib_reg);
if(is_leave_pinned) {
/* we should leave the memory pinned so put the memory on the MRU list */
openib_reg->base_reg.is_leave_pinned = is_leave_pinned;
opal_list_append(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg);
}
}
else if(is_leave_pinned){
/* the current memory region is large enough and we should leave the memory pinned */
if(NULL == opal_list_remove_item(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg)) {
BTL_ERROR("error removing item from reg_mru_list");
return NULL;
@ -624,12 +630,18 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
}
OBJ_RETAIN(openib_reg);
} else {
/* we didn't get a memory registration passed in, so we have to register the region
* ourselves
*/
if(mca_btl_openib_component.leave_pinned) {
/* so we need to leave pinned which means we must check and see if we
have room on the MRU list */
if( mca_btl_openib_component.reg_mru_len <= openib_btl->reg_mru_list.opal_list_length ) {
/* we have the maximum number of entries on the MRU list, time to
pull something off and make room. */
mca_mpool_openib_registration_t* old_reg =
(mca_mpool_openib_registration_t*)
opal_list_remove_last(&openib_btl->reg_mru_list);
@ -669,6 +681,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
opal_list_append(&openib_btl->reg_mru_list, (opal_list_item_t*) openib_reg);
} else {
/* we don't need to leave the memory pinned so just register it.. */
openib_btl->ib_pool->mpool_register(openib_btl->ib_pool,
frag->segment.seg_addr.pval,
*size,
@ -801,7 +814,10 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
}
/*
* Initialize the btl module by allocating a protection domain
* and creating both the high and low priority completion queues
*/
int mca_btl_openib_module_init(mca_btl_openib_module_t *openib_btl)
{

11
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -115,7 +115,6 @@ int mca_btl_openib_component_open(void)
/* initialize objects */
OBJ_CONSTRUCT(&mca_btl_openib_component.ib_procs, opal_list_t);
/* OBJ_CONSTRUCT (&mca_btl_openib_component.ib_recv_frags, ompi_free_list_t); */
/* register IB component parameters */
mca_btl_openib_component.ib_free_list_num =
@ -212,12 +211,6 @@ int mca_btl_openib_component_open(void)
mca_base_param_lookup_int(param, &value);
mca_btl_openib_component.leave_pinned = value;
mca_btl_openib_component.max_send_size = mca_btl_openib_module.super.btl_max_send_size;
mca_btl_openib_component.eager_limit = mca_btl_openib_module.super.btl_eager_limit;
@ -409,7 +402,7 @@ mca_btl_base_module_t** mca_btl_openib_component_init(int *num_btl_modules,
&mpool_resources);
if(NULL == openib_btl->ib_pool) {
BTL_ERROR("error creating vapi memory pool! aborting ib btl initialization");
BTL_ERROR("error creating vapi memory pool! aborting openib btl initialization");
return NULL;
}
@ -599,7 +592,7 @@ int mca_btl_openib_component_progress()
case IBV_WC_RECV:
/* process a recv completion (this should only occur for a send not an rdma) */
BTL_DEBUG_OUT( "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
BTL_DEBUG_OUT( "Got a recv completion");
frag = (mca_btl_openib_frag_t*) wc.wr_id;
endpoint = (mca_btl_openib_endpoint_t*) frag->endpoint;
frag->rc=OMPI_SUCCESS;

17
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -327,8 +327,7 @@ static int mca_btl_openib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoi
openib_btl->ib_cq_high,
endpoint->lcl_qp_attr_high,
&endpoint->lcl_qp_high))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
srand48(getpid() * time(NULL));
@ -340,8 +339,7 @@ static int mca_btl_openib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoi
openib_btl->ib_cq_low,
endpoint->lcl_qp_attr_low,
&endpoint->lcl_qp_low))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
endpoint->lcl_psn_low = lrand48() & 0xffffff;
@ -354,8 +352,7 @@ static int mca_btl_openib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoi
/* Send connection info over to remote endpoint */
endpoint->endpoint_state = MCA_BTL_IB_CONNECTING;
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_send_connect_req(endpoint))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error sending connect request, error code %d", rc);
return rc;
}
return OMPI_SUCCESS;
@ -389,7 +386,7 @@ static int mca_btl_openib_endpoint_reply_start_connect(mca_btl_openib_endpoint_t
openib_btl->ib_cq_low,
endpoint->lcl_qp_attr_low,
&endpoint->lcl_qp_low))) {
BTL_ERROR("error createing queue pair, error code %d", rc);
BTL_ERROR("error creating queue pair, error code %d", rc);
return rc;
}
endpoint->lcl_psn_low = lrand48() & 0xffffff;
@ -420,7 +417,8 @@ static int mca_btl_openib_endpoint_reply_start_connect(mca_btl_openib_endpoint_t
}
/*
*
* called when the openib has completed setup via the
* OOB channel
*/
static void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoint)
@ -477,8 +475,7 @@ static void mca_btl_openib_endpoint_recv(
* and then reply with our QP information */
if(OMPI_SUCCESS != (rc = mca_btl_openib_endpoint_reply_start_connect(ib_endpoint, buffer))) {
opal_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
BTL_ERROR("error in endpoint reply start connect");
break;
}

15
ompi/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -195,21 +195,6 @@ void mca_btl_openib_progress_send_frags(mca_btl_openib_endpoint_t*);
OPAL_THREAD_ADD32(post_rr_sub_rr_posted, post_rr_sub_cnt); \
}
#define DUMP_ENDPOINT(endpoint_ptr) { \
opal_output(0, "[%s:%d] ", __FILE__, __LINE__); \
opal_output(0, "Dumping endpoint %d state", \
endpoint->endpoint_proc->proc_guid.vpid); \
opal_output(0, "Local QP hndl : %d", \
endpoint_ptr->endpoint_conn->lres->qp_hndl); \
opal_output(0, "Local QP num : %d", \
endpoint_ptr->endpoint_conn->lres->qp_prop.qp_num); \
opal_output(0, "Remote QP num : %d", \
endpoint_ptr->endpoint_conn->rres->qp_num); \
opal_output(0, "Remote LID : %d", \
endpoint_ptr->endpoint_conn->rres->lid); \
}
#if defined(c_plusplus) || defined(__cplusplus)
}

9
ompi/mca/mpool/mvapi/mpool_mvapi_module.c
Просмотреть файл

@ -19,9 +19,6 @@
#include "opal/util/output.h"
#include "mca/mpool/mvapi/mpool_mvapi.h"
uint32_t mca_mpool_mvapi_registered_cnt=0;
uint32_t mca_mpool_mvapi_unregistered_cnt=0;
/*
* Initializes the mpool module.
@ -68,7 +65,6 @@ int mca_mpool_mvapi_register(mca_mpool_base_module_t* mpool,
size_t size,
mca_mpool_base_registration_t** registration){
mca_mpool_mvapi_registered_cnt++;
mca_mpool_mvapi_module_t * mpool_module = (mca_mpool_mvapi_module_t*) mpool;
mca_mpool_mvapi_registration_t * vapi_reg;
VAPI_mrw_t mr_in, mr_out;
@ -103,9 +99,7 @@ int mca_mpool_mvapi_register(mca_mpool_base_module_t* mpool,
);
if(VAPI_OK != ret){
opal_output(0, "error pinning vapi memory registered called %d times unregistered called %d times\n",
mca_mpool_mvapi_registered_cnt,
mca_mpool_mvapi_unregistered_cnt);
opal_output(0, "error registering memory: %s ", VAPI_strerror(ret));
return OMPI_ERROR;
}
@ -124,7 +118,6 @@ int mca_mpool_mvapi_register(mca_mpool_base_module_t* mpool,
int mca_mpool_mvapi_deregister(mca_mpool_base_module_t* mpool, void *addr, size_t size,
mca_mpool_base_registration_t* registration){
mca_mpool_mvapi_unregistered_cnt++;
VAPI_ret_t ret;
mca_mpool_mvapi_module_t * mpool_mvapi = (mca_mpool_mvapi_module_t*) mpool;
mca_mpool_mvapi_registration_t * vapi_reg;