ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

name changing bmi to btl

Просмотр исходного кода 
This commit was SVN r6223.
Этот коммит содержится в:
Galen Shipman 2005-06-30 02:12:23 +00:00
родитель 6e81b28251
Коммит 408a36c4ee
14 изменённых файлов: 874 добавлений и 874 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

34
src/mca/btl/mvapi/Makefile.am
Просмотреть файл

@ -19,28 +19,28 @@
include $(top_ompi_srcdir)/config/Makefile.options include $(top_ompi_srcdir)/config/Makefile.options
sources = \ sources = \
bmi_ib.c \ btl_ib.c \
bmi_ib.h \ btl_ib.h \
bmi_ib_component.c \ btl_ib_component.c \
bmi_ib_endpoint.c \ btl_ib_endpoint.c \
bmi_ib_endpoint.h \ btl_ib_endpoint.h \
bmi_ib_frag.c \ btl_ib_frag.c \
bmi_ib_frag.h \ btl_ib_frag.h \
bmi_ib_proc.c \ btl_ib_proc.c \
bmi_ib_proc.h \ btl_ib_proc.h \
bmi_ib_error.h btl_ib_error.h
# Make the output library in this directory, and name it either # Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la # mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds). # (for static builds).
if OMPI_BUILD_bmi_ib_DSO if OMPI_BUILD_btl_ib_DSO
lib = lib =
lib_sources = lib_sources =
component = mca_bmi_ib.la component = mca_btl_ib.la
component_sources = $(sources) component_sources = $(sources)
else else
lib = libmca_bmi_ib.la lib = libmca_btl_ib.la
lib_sources = $(sources) lib_sources = $(sources)
component = component =
component_sources = component_sources =
@ -48,9 +48,9 @@ endif
mcacomponentdir = $(libdir)/openmpi mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component) mcacomponent_LTLIBRARIES = $(component)
mca_bmi_ib_la_SOURCES = $(component_sources) mca_btl_ib_la_SOURCES = $(component_sources)
mca_bmi_ib_la_LDFLAGS = -module -avoid-version mca_btl_ib_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(lib) noinst_LTLIBRARIES = $(lib)
libmca_bmi_ib_la_SOURCES = $(lib_sources) libmca_btl_ib_la_SOURCES = $(lib_sources)
libmca_bmi_ib_la_LDFLAGS = -module -avoid-version libmca_btl_ib_la_LDFLAGS = -module -avoid-version

152
src/mca/btl/mvapi/bmi_ib_frag.c
Просмотреть файл

@ -1,152 +0,0 @@
#include "bmi_ib_frag.h"
#include "mca/common/vapi/vapi_mem_reg.h"
#include "mca/mpool/vapi/mpool_vapi.h"
static void mca_bmi_ib_frag_common_constructor( mca_bmi_ib_frag_t* frag)
{
mca_mpool_vapi_registration_t* mem_hndl = (mca_mpool_vapi_registration_t*) frag->base.super.user_data;
frag->hdr = (mca_bmi_ib_header_t*) (frag+1); /* initialize the bmi header to point to start at end of frag */
#if 0
mod = (unsigned long) frag->hdr % MCA_BMI_IB_FRAG_ALIGN;
if(mod != 0) {
frag->hdr = (mca_bmi_ib_header_t*) ((unsigned char*) frag->hdr + (MCA_BMI_IB_FRAG_ALIGN - mod));
}
#endif
frag->segment.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_bmi_ib_header_t); /* init the segment address to start after the bmi header */
#if 0
mod = (frag->segment.seg_addr.lval) % MCA_BMI_IB_FRAG_ALIGN;
if(mod != 0) {
frag->segment.seg_addr.lval += (MCA_BMI_IB_FRAG_ALIGN - mod);
}
#endif
frag->mem_hndl = mem_hndl->hndl;
frag->segment.seg_len = frag->size;
frag->segment.seg_key.key32[0] = (uint32_t) mem_hndl->l_key;
frag->sg_entry.lkey = mem_hndl->l_key;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
frag->base.des_flags = 0;
}
static void mca_bmi_ib_send_frag_common_constructor(mca_bmi_ib_frag_t* frag)
{
mca_bmi_ib_frag_common_constructor(frag);
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->sr_desc.comp_type = VAPI_SIGNALED;
frag->sr_desc.opcode = VAPI_SEND;
frag->sr_desc.remote_qkey = 0;
frag->sr_desc.sg_lst_len = 1;
frag->sr_desc.sg_lst_p = &frag->sg_entry;
frag->sr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
}
static void mca_bmi_ib_recv_frag_common_constructor(mca_bmi_ib_frag_t* frag)
{
mca_bmi_ib_frag_common_constructor(frag);
frag->base.des_dst = &frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->rr_desc.comp_type = VAPI_SIGNALED;
frag->rr_desc.opcode = VAPI_RECEIVE;
frag->rr_desc.sg_lst_len = 1;
frag->rr_desc.sg_lst_p = &frag->sg_entry;
frag->rr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
}
static void mca_bmi_ib_send_frag_eager_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = mca_bmi_ib_component.eager_limit;
mca_bmi_ib_send_frag_common_constructor(frag);
}
static void mca_bmi_ib_send_frag_max_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = mca_bmi_ib_component.max_send_size;
mca_bmi_ib_send_frag_common_constructor(frag);
}
static void mca_bmi_ib_recv_frag_max_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = mca_bmi_ib_component.max_send_size;
mca_bmi_ib_recv_frag_common_constructor(frag);
}
static void mca_bmi_ib_recv_frag_eager_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = mca_bmi_ib_component.eager_limit;
mca_bmi_ib_recv_frag_common_constructor(frag);
}
static void mca_bmi_ib_send_frag_frag_constructor(mca_bmi_ib_frag_t* frag)
{
frag->size = 0;
mca_bmi_ib_send_frag_common_constructor(frag);
}
OBJ_CLASS_INSTANCE(
mca_bmi_ib_frag_t,
mca_bmi_base_descriptor_t,
NULL,
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_send_frag_eager_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_send_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_send_frag_max_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_send_frag_max_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_send_frag_frag_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_send_frag_frag_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_recv_frag_eager_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_recv_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_bmi_ib_recv_frag_max_t,
mca_bmi_base_descriptor_t,
mca_bmi_ib_recv_frag_max_constructor,
NULL);

338
src/mca/btl/mvapi/bmi_ib.c → src/mca/btl/mvapi/btl_ib.c
Просмотреть файл

@ -19,21 +19,21 @@
#include "util/output.h" #include "util/output.h"
#include "util/if.h" #include "util/if.h"
#include "mca/pml/pml.h" #include "mca/pml/pml.h"
#include "mca/bmi/bmi.h" #include "mca/btl/btl.h"
#include "bmi_ib.h" #include "btl_ib.h"
#include "bmi_ib_frag.h" #include "btl_ib_frag.h"
#include "bmi_ib_proc.h" #include "btl_ib_proc.h"
#include "bmi_ib_endpoint.h" #include "btl_ib_endpoint.h"
#include "datatype/convertor.h" #include "datatype/convertor.h"
#include "mca/common/vapi/vapi_mem_reg.h" #include "mca/common/vapi/vapi_mem_reg.h"
#include "mca/mpool/base/base.h" #include "mca/mpool/base/base.h"
#include "mca/mpool/mpool.h" #include "mca/mpool/mpool.h"
#include "mca/mpool/vapi/mpool_vapi.h" #include "mca/mpool/vapi/mpool_vapi.h"
mca_bmi_ib_module_t mca_bmi_ib_module = { mca_btl_ib_module_t mca_btl_ib_module = {
{ {
&mca_bmi_ib_component.super, &mca_btl_ib_component.super,
0, /* max size of first fragment */ 0, /* max size of first fragment */
0, /* min send fragment size */ 0, /* min send fragment size */
0, /* max send fragment size */ 0, /* max send fragment size */
@ -42,41 +42,41 @@ mca_bmi_ib_module_t mca_bmi_ib_module = {
0, /* exclusivity */ 0, /* exclusivity */
0, /* latency */ 0, /* latency */
0, /* bandwidth */ 0, /* bandwidth */
0, /* TODO this should be PUT bmi flags */ 0, /* TODO this should be PUT btl flags */
mca_bmi_ib_add_procs, mca_btl_ib_add_procs,
mca_bmi_ib_del_procs, mca_btl_ib_del_procs,
mca_bmi_ib_register, mca_btl_ib_register,
mca_bmi_ib_finalize, mca_btl_ib_finalize,
/* we need alloc free, pack */ /* we need alloc free, pack */
mca_bmi_ib_alloc, mca_btl_ib_alloc,
mca_bmi_ib_free, mca_btl_ib_free,
mca_bmi_ib_prepare_src, mca_btl_ib_prepare_src,
mca_bmi_ib_prepare_dst, mca_btl_ib_prepare_dst,
mca_bmi_ib_send, mca_btl_ib_send,
mca_bmi_ib_put, mca_btl_ib_put,
NULL /* get */ NULL /* get */
} }
}; };
int mca_bmi_ib_add_procs( int mca_btl_ib_add_procs(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
size_t nprocs, size_t nprocs,
struct ompi_proc_t **ompi_procs, struct ompi_proc_t **ompi_procs,
struct mca_bmi_base_endpoint_t** peers, struct mca_btl_base_endpoint_t** peers,
ompi_bitmap_t* reachable) ompi_bitmap_t* reachable)
{ {
mca_bmi_ib_module_t* ib_bmi = (mca_bmi_ib_module_t*)bmi; mca_btl_ib_module_t* ib_btl = (mca_btl_ib_module_t*)btl;
int i, rc; int i, rc;
for(i = 0; i < (int) nprocs; i++) { for(i = 0; i < (int) nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i]; struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_bmi_ib_proc_t* ib_proc; mca_btl_ib_proc_t* ib_proc;
mca_bmi_base_endpoint_t* ib_peer; mca_btl_base_endpoint_t* ib_peer;
if(NULL == (ib_proc = mca_bmi_ib_proc_create(ompi_proc))) { if(NULL == (ib_proc = mca_btl_ib_proc_create(ompi_proc))) {
return OMPI_ERR_OUT_OF_RESOURCE; return OMPI_ERR_OUT_OF_RESOURCE;
} }
@ -88,18 +88,18 @@ int mca_bmi_ib_add_procs(
OMPI_THREAD_LOCK(&ib_proc->proc_lock); OMPI_THREAD_LOCK(&ib_proc->proc_lock);
/* The bmi_proc datastructure is shared by all IB PTL /* The btl_proc datastructure is shared by all IB PTL
* instances that are trying to reach this destination. * instances that are trying to reach this destination.
* Cache the peer instance on the bmi_proc. * Cache the peer instance on the btl_proc.
*/ */
ib_peer = OBJ_NEW(mca_bmi_ib_endpoint_t); ib_peer = OBJ_NEW(mca_btl_ib_endpoint_t);
if(NULL == ib_peer) { if(NULL == ib_peer) {
OMPI_THREAD_UNLOCK(&module_proc->proc_lock); OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE; return OMPI_ERR_OUT_OF_RESOURCE;
} }
ib_peer->endpoint_bmi = ib_bmi; ib_peer->endpoint_btl = ib_btl;
rc = mca_bmi_ib_proc_insert(ib_proc, ib_peer); rc = mca_btl_ib_proc_insert(ib_proc, ib_peer);
if(rc != OMPI_SUCCESS) { if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(ib_peer); OBJ_RELEASE(ib_peer);
OMPI_THREAD_UNLOCK(&module_proc->proc_lock); OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
@ -114,30 +114,30 @@ int mca_bmi_ib_add_procs(
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
int mca_bmi_ib_del_procs(struct mca_bmi_base_module_t* bmi, int mca_btl_ib_del_procs(struct mca_btl_base_module_t* btl,
size_t nprocs, size_t nprocs,
struct ompi_proc_t **procs, struct ompi_proc_t **procs,
struct mca_bmi_base_endpoint_t ** peers) struct mca_btl_base_endpoint_t ** peers)
{ {
/* Stub */ /* Stub */
DEBUG_OUT("Stub\n"); DEBUG_OUT("Stub\n");
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
int mca_bmi_ib_register( int mca_btl_ib_register(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
mca_bmi_base_tag_t tag, mca_btl_base_tag_t tag,
mca_bmi_base_module_recv_cb_fn_t cbfunc, mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata) void* cbdata)
{ {
/* TODO add register stuff here... */ /* TODO add register stuff here... */
mca_bmi_ib_module_t* ib_bmi = (mca_bmi_ib_module_t*) bmi; mca_btl_ib_module_t* ib_btl = (mca_btl_ib_module_t*) btl;
OMPI_THREAD_LOCK(&ib->bmi.ib_lock); OMPI_THREAD_LOCK(&ib->btl.ib_lock);
ib_bmi->ib_reg[tag].cbfunc = cbfunc; ib_btl->ib_reg[tag].cbfunc = cbfunc;
ib_bmi->ib_reg[tag].cbdata = cbdata; ib_btl->ib_reg[tag].cbdata = cbdata;
OMPI_THREAD_UNLOCK(&ib->bmi.ib_lock); OMPI_THREAD_UNLOCK(&ib->btl.ib_lock);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -145,57 +145,57 @@ int mca_bmi_ib_register(
/** /**
* Allocate a segment. * Allocate a segment.
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @param size (IN) Request segment size. * @param size (IN) Request segment size.
*/ */
mca_bmi_base_descriptor_t* mca_bmi_ib_alloc( mca_btl_base_descriptor_t* mca_btl_ib_alloc(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
size_t size) size_t size)
{ {
mca_bmi_ib_frag_t* frag; mca_btl_ib_frag_t* frag;
mca_bmi_ib_module_t* ib_bmi; mca_btl_ib_module_t* ib_btl;
int rc; int rc;
ib_bmi = (mca_bmi_ib_module_t*) bmi; ib_btl = (mca_btl_ib_module_t*) btl;
if(size <= mca_bmi_ib_component.eager_limit){ if(size <= mca_btl_ib_component.eager_limit){
MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc); MCA_BMI_IB_FRAG_ALLOC_EAGER(btl, frag, rc);
frag->segment.seg_len = frag->segment.seg_len =
size <= mca_bmi_ib_component.eager_limit ? size <= mca_btl_ib_component.eager_limit ?
size: mca_bmi_ib_component.eager_limit ; size: mca_btl_ib_component.eager_limit ;
} else { } else {
MCA_BMI_IB_FRAG_ALLOC_MAX(bmi, frag, rc); MCA_BMI_IB_FRAG_ALLOC_MAX(btl, frag, rc);
frag->segment.seg_len = frag->segment.seg_len =
size <= mca_bmi_ib_component.max_send_size ? size <= mca_btl_ib_component.max_send_size ?
size: mca_bmi_ib_component.max_send_size ; size: mca_btl_ib_component.max_send_size ;
} }
frag->segment.seg_len = size <= ib_bmi->super.bmi_eager_limit ? size : ib_bmi->super.bmi_eager_limit; frag->segment.seg_len = size <= ib_btl->super.btl_eager_limit ? size : ib_btl->super.btl_eager_limit;
frag->base.des_flags = 0; frag->base.des_flags = 0;
return (mca_bmi_base_descriptor_t*)frag; return (mca_btl_base_descriptor_t*)frag;
} }
/** /**
* *
* *
*/ */
int mca_bmi_ib_free( int mca_btl_ib_free(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
mca_bmi_base_descriptor_t* des) mca_btl_base_descriptor_t* des)
{ {
mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*)des; mca_btl_ib_frag_t* frag = (mca_btl_ib_frag_t*)des;
if(frag->size == 0) { if(frag->size == 0) {
MCA_BMI_IB_FRAG_RETURN_FRAG(bmi, frag); MCA_BMI_IB_FRAG_RETURN_FRAG(btl, frag);
OBJ_RELEASE(frag->vapi_reg); OBJ_RELEASE(frag->vapi_reg);
} }
else if(frag->size == mca_bmi_ib_component.max_send_size){ else if(frag->size == mca_btl_ib_component.max_send_size){
MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag); MCA_BMI_IB_FRAG_RETURN_MAX(btl, frag);
} else if(frag->size == mca_bmi_ib_component.eager_limit){ } else if(frag->size == mca_btl_ib_component.eager_limit){
MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag); MCA_BMI_IB_FRAG_RETURN_EAGER(btl, frag);
} }
return OMPI_SUCCESS; return OMPI_SUCCESS;
@ -206,20 +206,20 @@ int mca_bmi_ib_free(
* Pack data and return a descriptor that can be * Pack data and return a descriptor that can be
* used for send/put. * used for send/put.
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @param peer (IN) BMI peer addressing * @param peer (IN) BMI peer addressing
*/ */
mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src( mca_btl_base_descriptor_t* mca_btl_ib_prepare_src(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_base_endpoint_t* endpoint, struct mca_btl_base_endpoint_t* endpoint,
mca_mpool_base_registration_t* registration, mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor, struct ompi_convertor_t* convertor,
size_t reserve, size_t reserve,
size_t* size size_t* size
) )
{ {
mca_bmi_ib_module_t* ib_bmi; mca_btl_ib_module_t* ib_btl;
mca_bmi_ib_frag_t* frag; mca_btl_ib_frag_t* frag;
mca_mpool_vapi_registration_t * vapi_reg; mca_mpool_vapi_registration_t * vapi_reg;
struct iovec iov; struct iovec iov;
int32_t iov_count = 1; int32_t iov_count = 1;
@ -228,7 +228,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
int rc; int rc;
ib_bmi = (mca_bmi_ib_module_t*) bmi; ib_btl = (mca_btl_ib_module_t*) btl;
vapi_reg = (mca_mpool_vapi_registration_t*) registration; vapi_reg = (mca_mpool_vapi_registration_t*) registration;
/** if the data fits in the eager limit and we aren't told to pinn then we /** if the data fits in the eager limit and we aren't told to pinn then we
@ -238,7 +238,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
if(NULL != vapi_reg && 0 == ompi_convertor_need_buffers(convertor)){ if(NULL != vapi_reg && 0 == ompi_convertor_need_buffers(convertor)){
bool is_leave_pinned = vapi_reg->is_leave_pinned; bool is_leave_pinned = vapi_reg->is_leave_pinned;
MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); MCA_BMI_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){ if(NULL == frag){
return NULL; return NULL;
} }
@ -267,14 +267,14 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
} }
if(is_leave_pinned) { if(is_leave_pinned) {
if(NULL == ompi_list_remove_item(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg)){ if(NULL == ompi_list_remove_item(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg)){
ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__); ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
return NULL; return NULL;
} }
} }
OBJ_RELEASE(vapi_reg); OBJ_RELEASE(vapi_reg);
ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, ib_btl->ib_pool->mpool_register(ib_btl->ib_pool,
base_addr, base_addr,
new_len, new_len,
(mca_mpool_base_registration_t**) &vapi_reg); (mca_mpool_base_registration_t**) &vapi_reg);
@ -283,8 +283,8 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
rc = mca_mpool_base_insert(vapi_reg->base, rc = mca_mpool_base_insert(vapi_reg->base,
vapi_reg->bound - vapi_reg->base + 1, vapi_reg->bound - vapi_reg->base + 1,
ib_bmi->ib_pool, ib_btl->ib_pool,
(void*) (&ib_bmi->super), (void*) (&ib_btl->super),
(mca_mpool_base_registration_t*) vapi_reg); (mca_mpool_base_registration_t*) vapi_reg);
@ -296,15 +296,15 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
OBJ_RETAIN(vapi_reg); OBJ_RETAIN(vapi_reg);
if(is_leave_pinned) { if(is_leave_pinned) {
vapi_reg->is_leave_pinned = is_leave_pinned; vapi_reg->is_leave_pinned = is_leave_pinned;
ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); ompi_list_append(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg);
} }
} }
else if(is_leave_pinned) { else if(is_leave_pinned) {
if(NULL == ompi_list_remove_item(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg)) { if(NULL == ompi_list_remove_item(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg)) {
ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__); ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
return NULL; return NULL;
} }
ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); ompi_list_append(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg);
} }
frag->mem_hndl = vapi_reg->hndl; frag->mem_hndl = vapi_reg->hndl;
@ -324,11 +324,11 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
return &frag->base; return &frag->base;
} else if((mca_bmi_ib_component.leave_pinned || max_data > bmi->bmi_max_send_size) && } else if((mca_btl_ib_component.leave_pinned || max_data > btl->btl_max_send_size) &&
ompi_convertor_need_buffers(convertor) == 0 && ompi_convertor_need_buffers(convertor) == 0 &&
reserve == 0) reserve == 0)
{ {
MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); MCA_BMI_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){ if(NULL == frag){
return NULL; return NULL;
} }
@ -344,12 +344,12 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
frag->base.des_flags = 0; frag->base.des_flags = 0;
if(mca_bmi_ib_component.leave_pinned) { if(mca_btl_ib_component.leave_pinned) {
if(mca_bmi_ib_component.reg_mru_len <= ib_bmi->reg_mru_list.ompi_list_length ) { if(mca_btl_ib_component.reg_mru_len <= ib_btl->reg_mru_list.ompi_list_length ) {
mca_mpool_vapi_registration_t* old_reg = mca_mpool_vapi_registration_t* old_reg =
(mca_mpool_vapi_registration_t*) (mca_mpool_vapi_registration_t*)
ompi_list_remove_last(&ib_bmi->reg_mru_list); ompi_list_remove_last(&ib_btl->reg_mru_list);
if( NULL == old_reg) { if( NULL == old_reg) {
ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__); ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
@ -367,15 +367,15 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
OBJ_RELEASE(old_reg); OBJ_RELEASE(old_reg);
} }
ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, ib_btl->ib_pool->mpool_register(ib_btl->ib_pool,
iov.iov_base, iov.iov_base,
max_data, max_data,
(mca_mpool_base_registration_t**) &vapi_reg); (mca_mpool_base_registration_t**) &vapi_reg);
rc = mca_mpool_base_insert(vapi_reg->base, rc = mca_mpool_base_insert(vapi_reg->base,
vapi_reg->bound - vapi_reg->base + 1, vapi_reg->bound - vapi_reg->base + 1,
ib_bmi->ib_pool, ib_btl->ib_pool,
(void*) (&ib_bmi->super), (void*) (&ib_btl->super),
(mca_mpool_base_registration_t*) vapi_reg); (mca_mpool_base_registration_t*) vapi_reg);
if(rc != OMPI_SUCCESS) if(rc != OMPI_SUCCESS)
return NULL; return NULL;
@ -383,10 +383,10 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
vapi_reg->is_leave_pinned = true; vapi_reg->is_leave_pinned = true;
ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); ompi_list_append(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg);
} else { } else {
ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, ib_btl->ib_pool->mpool_register(ib_btl->ib_pool,
iov.iov_base, iov.iov_base,
max_data, max_data,
(mca_mpool_base_registration_t**) &vapi_reg); (mca_mpool_base_registration_t**) &vapi_reg);
@ -408,9 +408,9 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
OBJ_RETAIN(vapi_reg); OBJ_RETAIN(vapi_reg);
return &frag->base; return &frag->base;
} else if (max_data+reserve <= bmi->bmi_eager_limit) { } else if (max_data+reserve <= btl->btl_eager_limit) {
MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc); MCA_BMI_IB_FRAG_ALLOC_EAGER(btl, frag, rc);
if(NULL == frag) { if(NULL == frag) {
return NULL; return NULL;
} }
@ -421,7 +421,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
*size = max_data; *size = max_data;
if( rc < 0 ) { if( rc < 0 ) {
MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag); MCA_BMI_IB_FRAG_RETURN_EAGER(btl, frag);
return NULL; return NULL;
} }
@ -439,9 +439,9 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
/** if the data fits in the max limit and we aren't told to pinn then we /** if the data fits in the max limit and we aren't told to pinn then we
simply pack, if the data is non contiguous then we pack **/ simply pack, if the data is non contiguous then we pack **/
else if(max_data + reserve <= ib_bmi->super.bmi_max_send_size) { else if(max_data + reserve <= ib_btl->super.btl_max_send_size) {
MCA_BMI_IB_FRAG_ALLOC_MAX(bmi, frag, rc); MCA_BMI_IB_FRAG_ALLOC_MAX(btl, frag, rc);
if(NULL == frag) { if(NULL == frag) {
return NULL; return NULL;
} }
@ -455,7 +455,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
*size = max_data; *size = max_data;
if( rc < 0 ) { if( rc < 0 ) {
MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag); MCA_BMI_IB_FRAG_RETURN_MAX(btl, frag);
return NULL; return NULL;
} }
@ -475,27 +475,27 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
/** /**
* Pack data * Pack data
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @param peer (IN) BMI peer addressing * @param peer (IN) BMI peer addressing
*/ */
mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst( mca_btl_base_descriptor_t* mca_btl_ib_prepare_dst(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_base_endpoint_t* endpoint, struct mca_btl_base_endpoint_t* endpoint,
mca_mpool_base_registration_t* registration, mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor, struct ompi_convertor_t* convertor,
size_t reserve, size_t reserve,
size_t* size) size_t* size)
{ {
mca_bmi_ib_module_t* ib_bmi; mca_btl_ib_module_t* ib_btl;
mca_bmi_ib_frag_t* frag; mca_btl_ib_frag_t* frag;
mca_mpool_vapi_registration_t * vapi_reg; mca_mpool_vapi_registration_t * vapi_reg;
int rc; int rc;
size_t reg_len; size_t reg_len;
ib_bmi = (mca_bmi_ib_module_t*) bmi; ib_btl = (mca_btl_ib_module_t*) btl;
vapi_reg = (mca_mpool_vapi_registration_t*) registration; vapi_reg = (mca_mpool_vapi_registration_t*) registration;
MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); MCA_BMI_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
if(NULL == frag){ if(NULL == frag){
return NULL; return NULL;
@ -522,14 +522,14 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
} }
if(is_leave_pinned) { if(is_leave_pinned) {
if(NULL == ompi_list_remove_item(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg)) { if(NULL == ompi_list_remove_item(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg)) {
ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__); ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
return NULL; return NULL;
} }
} }
OBJ_RELEASE(vapi_reg); OBJ_RELEASE(vapi_reg);
ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, ib_btl->ib_pool->mpool_register(ib_btl->ib_pool,
base_addr, base_addr,
new_len, new_len,
(mca_mpool_base_registration_t**) &vapi_reg); (mca_mpool_base_registration_t**) &vapi_reg);
@ -537,8 +537,8 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
rc = mca_mpool_base_insert(vapi_reg->base, rc = mca_mpool_base_insert(vapi_reg->base,
vapi_reg->bound - vapi_reg->base + 1, vapi_reg->bound - vapi_reg->base + 1,
ib_bmi->ib_pool, ib_btl->ib_pool,
(void*) (&ib_bmi->super), (void*) (&ib_btl->super),
(mca_mpool_base_registration_t*) vapi_reg); (mca_mpool_base_registration_t*) vapi_reg);
if(OMPI_SUCCESS != rc) { if(OMPI_SUCCESS != rc) {
@ -549,27 +549,27 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
if(is_leave_pinned) { if(is_leave_pinned) {
vapi_reg->is_leave_pinned = is_leave_pinned; vapi_reg->is_leave_pinned = is_leave_pinned;
ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); ompi_list_append(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg);
} }
} }
else if(is_leave_pinned){ else if(is_leave_pinned){
if(NULL == ompi_list_remove_item(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg)) { if(NULL == ompi_list_remove_item(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg)) {
ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__); ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
return NULL; return NULL;
} }
ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); ompi_list_append(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg);
} }
} else { } else {
if(mca_bmi_ib_component.leave_pinned) { if(mca_btl_ib_component.leave_pinned) {
if( mca_bmi_ib_component.reg_mru_len <= ib_bmi->reg_mru_list.ompi_list_length ) { if( mca_btl_ib_component.reg_mru_len <= ib_btl->reg_mru_list.ompi_list_length ) {
mca_mpool_vapi_registration_t* old_reg = mca_mpool_vapi_registration_t* old_reg =
(mca_mpool_vapi_registration_t*) (mca_mpool_vapi_registration_t*)
ompi_list_remove_last(&ib_bmi->reg_mru_list); ompi_list_remove_last(&ib_btl->reg_mru_list);
if( NULL == old_reg) { if( NULL == old_reg) {
ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__); ompi_output(0,"%s:%d:%s error removing item from reg_mru_list", __FILE__, __LINE__, __func__);
@ -585,7 +585,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
OBJ_RELEASE(old_reg); OBJ_RELEASE(old_reg);
} }
ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, ib_btl->ib_pool->mpool_register(ib_btl->ib_pool,
frag->segment.seg_addr.pval, frag->segment.seg_addr.pval,
*size, *size,
(mca_mpool_base_registration_t**) &vapi_reg); (mca_mpool_base_registration_t**) &vapi_reg);
@ -594,8 +594,8 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
rc = mca_mpool_base_insert(vapi_reg->base, rc = mca_mpool_base_insert(vapi_reg->base,
vapi_reg->bound - vapi_reg->base + 1, vapi_reg->bound - vapi_reg->base + 1,
ib_bmi->ib_pool, ib_btl->ib_pool,
(void*) (&ib_bmi->super), (void*) (&ib_btl->super),
(mca_mpool_base_registration_t*) vapi_reg); (mca_mpool_base_registration_t*) vapi_reg);
if(OMPI_SUCCESS != rc){ if(OMPI_SUCCESS != rc){
ompi_output(0,"%s:%d:%s error inserting memory region into memory pool", __FILE__, __LINE__, __func__); ompi_output(0,"%s:%d:%s error inserting memory region into memory pool", __FILE__, __LINE__, __func__);
@ -603,10 +603,10 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
} }
OBJ_RETAIN(vapi_reg); OBJ_RETAIN(vapi_reg);
ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); ompi_list_append(&ib_btl->reg_mru_list, (ompi_list_item_t*) vapi_reg);
} else { } else {
ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, ib_btl->ib_pool->mpool_register(ib_btl->ib_pool,
frag->segment.seg_addr.pval, frag->segment.seg_addr.pval,
*size, *size,
(mca_mpool_base_registration_t**) &vapi_reg); (mca_mpool_base_registration_t**) &vapi_reg);
@ -635,42 +635,42 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
} }
int mca_bmi_ib_finalize(struct mca_bmi_base_module_t* bmi) int mca_btl_ib_finalize(struct mca_btl_base_module_t* btl)
{ {
mca_bmi_ib_module_t* ib_bmi; mca_btl_ib_module_t* ib_btl;
ib_bmi = (mca_bmi_ib_module_t*) bmi; ib_btl = (mca_btl_ib_module_t*) btl;
if(ib_bmi->send_free_eager.fl_num_allocated != if(ib_btl->send_free_eager.fl_num_allocated !=
ib_bmi->send_free_eager.super.ompi_list_length){ ib_btl->send_free_eager.super.ompi_list_length){
ompi_output(0, "bmi ib send_free_eager frags: %d allocated %d returned \n", ompi_output(0, "btl ib send_free_eager frags: %d allocated %d returned \n",
ib_bmi->send_free_eager.fl_num_allocated, ib_btl->send_free_eager.fl_num_allocated,
ib_bmi->send_free_eager.super.ompi_list_length); ib_btl->send_free_eager.super.ompi_list_length);
} }
if(ib_bmi->send_free_max.fl_num_allocated != if(ib_btl->send_free_max.fl_num_allocated !=
ib_bmi->send_free_max.super.ompi_list_length){ ib_btl->send_free_max.super.ompi_list_length){
ompi_output(0, "bmi ib send_free_max frags: %d allocated %d returned \n", ompi_output(0, "btl ib send_free_max frags: %d allocated %d returned \n",
ib_bmi->send_free_max.fl_num_allocated, ib_btl->send_free_max.fl_num_allocated,
ib_bmi->send_free_max.super.ompi_list_length); ib_btl->send_free_max.super.ompi_list_length);
} }
if(ib_bmi->send_free_frag.fl_num_allocated != if(ib_btl->send_free_frag.fl_num_allocated !=
ib_bmi->send_free_frag.super.ompi_list_length){ ib_btl->send_free_frag.super.ompi_list_length){
ompi_output(0, "bmi ib send_free_frag frags: %d allocated %d returned \n", ompi_output(0, "btl ib send_free_frag frags: %d allocated %d returned \n",
ib_bmi->send_free_frag.fl_num_allocated, ib_btl->send_free_frag.fl_num_allocated,
ib_bmi->send_free_frag.super.ompi_list_length); ib_btl->send_free_frag.super.ompi_list_length);
} }
if(ib_bmi->recv_free_eager.fl_num_allocated != if(ib_btl->recv_free_eager.fl_num_allocated !=
ib_bmi->recv_free_eager.super.ompi_list_length){ ib_btl->recv_free_eager.super.ompi_list_length){
ompi_output(0, "bmi ib recv_free_eager frags: %d allocated %d returned \n", ompi_output(0, "btl ib recv_free_eager frags: %d allocated %d returned \n",
ib_bmi->recv_free_eager.fl_num_allocated, ib_btl->recv_free_eager.fl_num_allocated,
ib_bmi->recv_free_eager.super.ompi_list_length); ib_btl->recv_free_eager.super.ompi_list_length);
} }
if(ib_bmi->recv_free_max.fl_num_allocated != if(ib_btl->recv_free_max.fl_num_allocated !=
ib_bmi->recv_free_max.super.ompi_list_length){ ib_btl->recv_free_max.super.ompi_list_length){
ompi_output(0, "bmi ib recv_free_max frags: %d allocated %d returned \n", ompi_output(0, "btl ib recv_free_max frags: %d allocated %d returned \n",
ib_bmi->recv_free_max.fl_num_allocated, ib_btl->recv_free_max.fl_num_allocated,
ib_bmi->recv_free_max.super.ompi_list_length); ib_btl->recv_free_max.super.ompi_list_length);
} }
return OMPI_SUCCESS; return OMPI_SUCCESS;
@ -683,20 +683,20 @@ int mca_bmi_ib_finalize(struct mca_bmi_base_module_t* bmi)
* on to the peer. * on to the peer.
*/ */
int mca_bmi_ib_send( int mca_btl_ib_send(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_base_endpoint_t* endpoint, struct mca_btl_base_endpoint_t* endpoint,
struct mca_bmi_base_descriptor_t* descriptor, struct mca_btl_base_descriptor_t* descriptor,
mca_bmi_base_tag_t tag) mca_btl_base_tag_t tag)
{ {
mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*)descriptor; mca_btl_ib_frag_t* frag = (mca_btl_ib_frag_t*)descriptor;
frag->endpoint = endpoint; frag->endpoint = endpoint;
frag->hdr->tag = tag; frag->hdr->tag = tag;
frag->type = MCA_BMI_IB_FRAG_SEND; frag->type = MCA_BMI_IB_FRAG_SEND;
frag->rc = mca_bmi_ib_endpoint_send(endpoint, frag); frag->rc = mca_btl_ib_endpoint_send(endpoint, frag);
return frag->rc; return frag->rc;
} }
@ -705,12 +705,12 @@ int mca_bmi_ib_send(
* RDMA local buffer to remote buffer address. * RDMA local buffer to remote buffer address.
*/ */
int mca_bmi_ib_put( mca_bmi_base_module_t* bmi, int mca_btl_ib_put( mca_btl_base_module_t* btl,
mca_bmi_base_endpoint_t* endpoint, mca_btl_base_endpoint_t* endpoint,
mca_bmi_base_descriptor_t* descriptor) mca_btl_base_descriptor_t* descriptor)
{ {
mca_bmi_ib_module_t* ib_bmi = (mca_bmi_ib_module_t*) bmi; mca_btl_ib_module_t* ib_btl = (mca_btl_ib_module_t*) btl;
mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*) descriptor; mca_btl_ib_frag_t* frag = (mca_btl_ib_frag_t*) descriptor;
frag->endpoint = endpoint; frag->endpoint = endpoint;
frag->sr_desc.opcode = VAPI_RDMA_WRITE; frag->sr_desc.opcode = VAPI_RDMA_WRITE;
@ -720,13 +720,13 @@ int mca_bmi_ib_put( mca_bmi_base_module_t* bmi,
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->base.des_src->seg_addr.pval; frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->base.des_src->seg_addr.pval;
frag->sg_entry.len = frag->base.des_src->seg_len; frag->sg_entry.len = frag->base.des_src->seg_len;
frag->ret = VAPI_post_sr(ib_bmi->nic, frag->ret = VAPI_post_sr(ib_btl->nic,
endpoint->lcl_qp_hndl_low, endpoint->lcl_qp_hndl_low,
&frag->sr_desc); &frag->sr_desc);
if(VAPI_OK != frag->ret){ if(VAPI_OK != frag->ret){
return OMPI_ERROR; return OMPI_ERROR;
} }
mca_bmi_ib_endpoint_post_rr(endpoint, 1); mca_btl_ib_endpoint_post_rr(endpoint, 1);
return OMPI_SUCCESS; return OMPI_SUCCESS;
@ -782,22 +782,22 @@ static void async_event_handler(VAPI_hca_hndl_t hca_hndl,
int mca_bmi_ib_module_init(mca_bmi_ib_module_t *ib_bmi) int mca_btl_ib_module_init(mca_btl_ib_module_t *ib_btl)
{ {
/* Allocate Protection Domain */ /* Allocate Protection Domain */
VAPI_ret_t ret; VAPI_ret_t ret;
uint32_t cqe_cnt = 0; uint32_t cqe_cnt = 0;
ret = VAPI_alloc_pd(ib_bmi->nic, &ib_bmi->ptag); ret = VAPI_alloc_pd(ib_btl->nic, &ib_btl->ptag);
if(ret != VAPI_OK) { if(ret != VAPI_OK) {
MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_alloc_pd"); MCA_BMI_IB_VAPI_ERROR(ret, "VAPI_alloc_pd");
return OMPI_ERROR; return OMPI_ERROR;
} }
ret = VAPI_create_cq(ib_bmi->nic, ib_bmi->ib_cq_size, ret = VAPI_create_cq(ib_btl->nic, ib_btl->ib_cq_size,
&ib_bmi->cq_hndl_low, &cqe_cnt); &ib_btl->cq_hndl_low, &cqe_cnt);
if( VAPI_OK != ret) { if( VAPI_OK != ret) {
@ -805,8 +805,8 @@ int mca_bmi_ib_module_init(mca_bmi_ib_module_t *ib_bmi)
return OMPI_ERROR; return OMPI_ERROR;
} }
ret = VAPI_create_cq(ib_bmi->nic, ib_bmi->ib_cq_size, ret = VAPI_create_cq(ib_btl->nic, ib_btl->ib_cq_size,
&ib_bmi->cq_hndl_high, &cqe_cnt); &ib_btl->cq_hndl_high, &cqe_cnt);
if( VAPI_OK != ret) { if( VAPI_OK != ret) {
@ -820,8 +820,8 @@ int mca_bmi_ib_module_init(mca_bmi_ib_module_t *ib_bmi)
return OMPI_ERROR; return OMPI_ERROR;
} }
ret = EVAPI_set_async_event_handler(ib_bmi->nic, ret = EVAPI_set_async_event_handler(ib_btl->nic,
async_event_handler, 0, &ib_bmi->async_handler); async_event_handler, 0, &ib_btl->async_handler);
if(VAPI_OK != ret) { if(VAPI_OK != ret) {
MCA_BMI_IB_VAPI_ERROR(ret, "EVAPI_set_async_event_handler"); MCA_BMI_IB_VAPI_ERROR(ret, "EVAPI_set_async_event_handler");

150
src/mca/btl/mvapi/bmi_ib.h → src/mca/btl/mvapi/btl_ib.h
Просмотреть файл

@ -29,13 +29,13 @@
#include "class/ompi_bitmap.h" #include "class/ompi_bitmap.h"
#include "event/event.h" #include "event/event.h"
#include "mca/pml/pml.h" #include "mca/pml/pml.h"
#include "mca/bmi/bmi.h" #include "mca/btl/btl.h"
#include "util/output.h" #include "util/output.h"
#include "mca/mpool/mpool.h" #include "mca/mpool/mpool.h"
#include "bmi_ib_error.h" #include "btl_ib_error.h"
#include "mca/bmi/bmi.h" #include "mca/btl/btl.h"
#include "mca/bmi/base/base.h" #include "mca/btl/base/base.h"
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)
extern "C" { extern "C" {
@ -47,13 +47,13 @@ extern "C" {
* Infiniband (IB) BMI component. * Infiniband (IB) BMI component.
*/ */
struct mca_bmi_ib_component_t { struct mca_btl_ib_component_t {
mca_bmi_base_component_1_0_0_t super; /**< base BMI component */ mca_btl_base_component_1_0_0_t super; /**< base BMI component */
uint32_t ib_num_bmis; uint32_t ib_num_btls;
/**< number of hcas available to the IB component */ /**< number of hcas available to the IB component */
struct mca_bmi_ib_module_t *ib_bmis; struct mca_btl_ib_module_t *ib_btls;
/**< array of available PTLs */ /**< array of available PTLs */
int ib_free_list_num; int ib_free_list_num;
@ -96,21 +96,21 @@ struct mca_bmi_ib_component_t {
uint32_t reg_mru_len; uint32_t reg_mru_len;
}; typedef struct mca_bmi_ib_component_t mca_bmi_ib_component_t; }; typedef struct mca_btl_ib_component_t mca_btl_ib_component_t;
extern mca_bmi_ib_component_t mca_bmi_ib_component; extern mca_btl_ib_component_t mca_btl_ib_component;
typedef mca_bmi_base_recv_reg_t mca_bmi_ib_recv_reg_t; typedef mca_btl_base_recv_reg_t mca_btl_ib_recv_reg_t;
/** /**
* IB PTL Interface * IB PTL Interface
*/ */
struct mca_bmi_ib_module_t { struct mca_btl_ib_module_t {
mca_bmi_base_module_t super; /**< base PTL interface */ mca_btl_base_module_t super; /**< base PTL interface */
bool bmi_inited; bool btl_inited;
mca_bmi_ib_recv_reg_t ib_reg[256]; mca_btl_ib_recv_reg_t ib_reg[256];
VAPI_hca_id_t hca_id; /**< ID of HCA */ VAPI_hca_id_t hca_id; /**< ID of HCA */
IB_port_t port_id; /**< ID of the PORT */ IB_port_t port_id; /**< ID of the PORT */
VAPI_hca_port_t port; /**< IB port of this PTL */ VAPI_hca_port_t port; /**< IB port of this PTL */
@ -166,26 +166,26 @@ struct mca_bmi_ib_module_t {
uint32_t ib_src_path_bits; uint32_t ib_src_path_bits;
}; typedef struct mca_bmi_ib_module_t mca_bmi_ib_module_t; }; typedef struct mca_btl_ib_module_t mca_btl_ib_module_t;
struct mca_bmi_ib_frag_t; struct mca_btl_ib_frag_t;
extern mca_bmi_ib_module_t mca_bmi_ib_module; extern mca_btl_ib_module_t mca_btl_ib_module;
/** /**
* Register IB component parameters with the MCA framework * Register IB component parameters with the MCA framework
*/ */
extern int mca_bmi_ib_component_open(void); extern int mca_btl_ib_component_open(void);
/** /**
* Any final cleanup before being unloaded. * Any final cleanup before being unloaded.
*/ */
extern int mca_bmi_ib_component_close(void); extern int mca_btl_ib_component_close(void);
/** /**
* IB component initialization. * IB component initialization.
* *
* @param num_bmi_modules (OUT) Number of BMIs returned in BMI array. * @param num_btl_modules (OUT) Number of BMIs returned in BMI array.
* @param allow_multi_user_threads (OUT) Flag indicating wether BMI supports user threads (TRUE) * @param allow_multi_user_threads (OUT) Flag indicating wether BMI supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether BMI uses threads (TRUE) * @param have_hidden_threads (OUT) Flag indicating wether BMI uses threads (TRUE)
* *
@ -195,8 +195,8 @@ extern int mca_bmi_ib_component_close(void);
* (3) publish BMI addressing info * (3) publish BMI addressing info
* *
*/ */
extern mca_bmi_base_module_t** mca_bmi_ib_component_init( extern mca_btl_base_module_t** mca_btl_ib_component_init(
int *num_bmi_modules, int *num_btl_modules,
bool allow_multi_user_threads, bool allow_multi_user_threads,
bool have_hidden_threads bool have_hidden_threads
); );
@ -205,7 +205,7 @@ extern mca_bmi_base_module_t** mca_bmi_ib_component_init(
/** /**
* IB component progress. * IB component progress.
*/ */
extern int mca_bmi_ib_component_progress( extern int mca_btl_ib_component_progress(
void void
); );
@ -214,7 +214,7 @@ extern int mca_bmi_ib_component_progress(
* Register a callback function that is called on receipt * Register a callback function that is called on receipt
* of a fragment. * of a fragment.
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @return Status indicating if cleanup was successful * @return Status indicating if cleanup was successful
* *
* When the process list changes, the PML notifies the BMI of the * When the process list changes, the PML notifies the BMI of the
@ -222,10 +222,10 @@ extern int mca_bmi_ib_component_progress(
* resources associated with the peer. * resources associated with the peer.
*/ */
int mca_bmi_ib_register( int mca_btl_ib_register(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
mca_bmi_base_tag_t tag, mca_btl_base_tag_t tag,
mca_bmi_base_module_recv_cb_fn_t cbfunc, mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata void* cbdata
); );
@ -233,19 +233,19 @@ int mca_bmi_ib_register(
/** /**
* Cleanup any resources held by the BMI. * Cleanup any resources held by the BMI.
* *
* @param bmi BMI instance. * @param btl BMI instance.
* @return OMPI_SUCCESS or error status on failure. * @return OMPI_SUCCESS or error status on failure.
*/ */
extern int mca_bmi_ib_finalize( extern int mca_btl_ib_finalize(
struct mca_bmi_base_module_t* bmi struct mca_btl_base_module_t* btl
); );
/** /**
* PML->BMI notification of change in the process list. * PML->BMI notification of change in the process list.
* *
* @param bmi (IN) * @param btl (IN)
* @param nprocs (IN) Number of processes * @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes * @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info. * @param peers (OUT) Set of (optional) peer addressing info.
@ -254,97 +254,97 @@ extern int mca_bmi_ib_finalize(
* *
*/ */
extern int mca_bmi_ib_add_procs( extern int mca_btl_ib_add_procs(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
size_t nprocs, size_t nprocs,
struct ompi_proc_t **procs, struct ompi_proc_t **procs,
struct mca_bmi_base_endpoint_t** peers, struct mca_btl_base_endpoint_t** peers,
ompi_bitmap_t* reachable ompi_bitmap_t* reachable
); );
/** /**
* PML->BMI notification of change in the process list. * PML->BMI notification of change in the process list.
* *
* @param bmi (IN) BMI instance * @param btl (IN) BMI instance
* @param nproc (IN) Number of processes. * @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes. * @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures. * @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful * @return Status indicating if cleanup was successful
* *
*/ */
extern int mca_bmi_ib_del_procs( extern int mca_btl_ib_del_procs(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
size_t nprocs, size_t nprocs,
struct ompi_proc_t **procs, struct ompi_proc_t **procs,
struct mca_bmi_base_endpoint_t** peers struct mca_btl_base_endpoint_t** peers
); );
/** /**
* PML->BMI Initiate a send of the specified size. * PML->BMI Initiate a send of the specified size.
* *
* @param bmi (IN) BMI instance * @param btl (IN) BMI instance
* @param bmi_base_peer (IN) BMI peer addressing * @param btl_base_peer (IN) BMI peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via mca_bmi_base_request_alloc_fn_t) * @param send_request (IN/OUT) Send request (allocated by PML via mca_btl_base_request_alloc_fn_t)
* @param size (IN) Number of bytes PML is requesting BMI to deliver * @param size (IN) Number of bytes PML is requesting BMI to deliver
* @param flags (IN) Flags that should be passed to the peer via the message header. * @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the BMI was able to queue one or more fragments * @param request (OUT) OMPI_SUCCESS if the BMI was able to queue one or more fragments
*/ */
extern int mca_bmi_ib_send( extern int mca_btl_ib_send(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_base_endpoint_t* bmi_peer, struct mca_btl_base_endpoint_t* btl_peer,
struct mca_bmi_base_descriptor_t* descriptor, struct mca_btl_base_descriptor_t* descriptor,
mca_bmi_base_tag_t tag mca_btl_base_tag_t tag
); );
/** /**
* PML->BMI Initiate a put of the specified size. * PML->BMI Initiate a put of the specified size.
* *
* @param bmi (IN) BMI instance * @param btl (IN) BMI instance
* @param bmi_base_peer (IN) BMI peer addressing * @param btl_base_peer (IN) BMI peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via mca_bmi_base_request_alloc_fn_t) * @param send_request (IN/OUT) Send request (allocated by PML via mca_btl_base_request_alloc_fn_t)
* @param size (IN) Number of bytes PML is requesting BMI to deliver * @param size (IN) Number of bytes PML is requesting BMI to deliver
* @param flags (IN) Flags that should be passed to the peer via the message header. * @param flags (IN) Flags that should be passed to the peer via the message header.
* @param request (OUT) OMPI_SUCCESS if the BMI was able to queue one or more fragments * @param request (OUT) OMPI_SUCCESS if the BMI was able to queue one or more fragments
*/ */
extern int mca_bmi_ib_put( extern int mca_btl_ib_put(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_base_endpoint_t* bmi_peer, struct mca_btl_base_endpoint_t* btl_peer,
struct mca_bmi_base_descriptor_t* decriptor struct mca_btl_base_descriptor_t* decriptor
); );
/** /**
* Allocate a descriptor. * Allocate a descriptor.
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @param size (IN) Requested descriptor size. * @param size (IN) Requested descriptor size.
*/ */
extern mca_bmi_base_descriptor_t* mca_bmi_ib_alloc( extern mca_btl_base_descriptor_t* mca_btl_ib_alloc(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
size_t size); size_t size);
/** /**
* Return a segment allocated by this BMI. * Return a segment allocated by this BMI.
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @param descriptor (IN) Allocated descriptor. * @param descriptor (IN) Allocated descriptor.
*/ */
extern int mca_bmi_ib_free( extern int mca_btl_ib_free(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
mca_bmi_base_descriptor_t* des); mca_btl_base_descriptor_t* des);
/** /**
* Pack data and return a descriptor that can be * Pack data and return a descriptor that can be
* used for send/put. * used for send/put.
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @param peer (IN) BMI peer addressing * @param peer (IN) BMI peer addressing
*/ */
mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src( mca_btl_base_descriptor_t* mca_btl_ib_prepare_src(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_base_endpoint_t* peer, struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration, mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor, struct ompi_convertor_t* convertor,
size_t reserve, size_t reserve,
@ -354,12 +354,12 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
/** /**
* Allocate a descriptor initialized for RDMA write. * Allocate a descriptor initialized for RDMA write.
* *
* @param bmi (IN) BMI module * @param btl (IN) BMI module
* @param peer (IN) BMI peer addressing * @param peer (IN) BMI peer addressing
*/ */
extern mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst( extern mca_btl_base_descriptor_t* mca_btl_ib_prepare_dst(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_base_endpoint_t* peer, struct mca_btl_base_endpoint_t* peer,
mca_mpool_base_registration_t* registration, mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor, struct ompi_convertor_t* convertor,
size_t reserve, size_t reserve,
@ -367,17 +367,17 @@ extern mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
/** /**
* Return a send fragment to the modules free list. * Return a send fragment to the modules free list.
* *
* @param bmi (IN) BMI instance * @param btl (IN) BMI instance
* @param frag (IN) IB send fragment * @param frag (IN) IB send fragment
* *
*/ */
extern void mca_bmi_ib_send_frag_return( extern void mca_btl_ib_send_frag_return(
struct mca_bmi_base_module_t* bmi, struct mca_btl_base_module_t* btl,
struct mca_bmi_ib_frag_t* struct mca_btl_ib_frag_t*
); );
int mca_bmi_ib_module_init(mca_bmi_ib_module_t* ib_bmi); int mca_btl_ib_module_init(mca_btl_ib_module_t* ib_btl);
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)

400
src/mca/btl/mvapi/bmi_ib_component.c → src/mca/btl/mvapi/btl_ib_component.c
Просмотреть файл

@ -22,23 +22,23 @@
#include "util/argv.h" #include "util/argv.h"
#include "util/output.h" #include "util/output.h"
#include "mca/pml/pml.h" #include "mca/pml/pml.h"
#include "mca/bmi/bmi.h" #include "mca/btl/btl.h"
#include "mca/base/mca_base_param.h" #include "mca/base/mca_base_param.h"
#include "mca/base/mca_base_module_exchange.h" #include "mca/base/mca_base_module_exchange.h"
#include "mca/errmgr/errmgr.h" #include "mca/errmgr/errmgr.h"
#include "mca/common/vapi/vapi_mem_reg.h" #include "mca/common/vapi/vapi_mem_reg.h"
#include "mca/mpool/base/base.h" #include "mca/mpool/base/base.h"
#include "bmi_ib.h" #include "btl_ib.h"
#include "bmi_ib_frag.h" #include "btl_ib_frag.h"
#include "bmi_ib_endpoint.h" #include "btl_ib_endpoint.h"
#include "mca/bmi/base/base.h" #include "mca/btl/base/base.h"
#include <vapi.h> #include <vapi.h>
#include <vapi_common.h> #include <vapi_common.h>
#include "datatype/convertor.h" #include "datatype/convertor.h"
#include "mca/mpool/vapi/mpool_vapi.h" #include "mca/mpool/vapi/mpool_vapi.h"
mca_bmi_ib_component_t mca_bmi_ib_component = { mca_btl_ib_component_t mca_btl_ib_component = {
{ {
/* First, the mca_base_component_t struct containing meta information /* First, the mca_base_component_t struct containing meta information
about the component itself */ about the component itself */
@ -53,8 +53,8 @@ mca_bmi_ib_component_t mca_bmi_ib_component = {
1, /* MCA component major version */ 1, /* MCA component major version */
0, /* MCA component minor version */ 0, /* MCA component minor version */
0, /* MCA component release version */ 0, /* MCA component release version */
mca_bmi_ib_component_open, /* component open */ mca_btl_ib_component_open, /* component open */
mca_bmi_ib_component_close /* component close */ mca_btl_ib_component_close /* component close */
}, },
/* Next the MCA v1.0.0 component meta data */ /* Next the MCA v1.0.0 component meta data */
@ -65,8 +65,8 @@ mca_bmi_ib_component_t mca_bmi_ib_component = {
false false
}, },
mca_bmi_ib_component_init, mca_btl_ib_component_init,
mca_bmi_ib_component_progress, mca_btl_ib_component_progress,
} }
}; };
@ -75,21 +75,21 @@ mca_bmi_ib_component_t mca_bmi_ib_component = {
* utility routines for parameter registration * utility routines for parameter registration
*/ */
static inline char* mca_bmi_ib_param_register_string( static inline char* mca_btl_ib_param_register_string(
const char* param_name, const char* param_name,
const char* default_value) const char* default_value)
{ {
char *param_value; char *param_value;
int id = mca_base_param_register_string("bmi","ib",param_name,NULL,default_value); int id = mca_base_param_register_string("btl","ib",param_name,NULL,default_value);
mca_base_param_lookup_string(id, &param_value); mca_base_param_lookup_string(id, &param_value);
return param_value; return param_value;
} }
static inline int mca_bmi_ib_param_register_int( static inline int mca_btl_ib_param_register_int(
const char* param_name, const char* param_name,
int default_value) int default_value)
{ {
int id = mca_base_param_register_int("bmi","ib",param_name,NULL,default_value); int id = mca_base_param_register_int("btl","ib",param_name,NULL,default_value);
int param_value = default_value; int param_value = default_value;
mca_base_param_lookup_int(id,&param_value); mca_base_param_lookup_int(id,&param_value);
return param_value; return param_value;
@ -100,113 +100,113 @@ static inline int mca_bmi_ib_param_register_int(
* component parameters. * component parameters.
*/ */
int mca_bmi_ib_component_open(void) int mca_btl_ib_component_open(void)
{ {
int param, value; int param, value;
/* initialize state */ /* initialize state */
mca_bmi_ib_component.ib_num_bmis=0; mca_btl_ib_component.ib_num_btls=0;
mca_bmi_ib_component.ib_bmis=NULL; mca_btl_ib_component.ib_btls=NULL;
/* initialize objects */ /* initialize objects */
OBJ_CONSTRUCT(&mca_bmi_ib_component.ib_procs, ompi_list_t); OBJ_CONSTRUCT(&mca_btl_ib_component.ib_procs, ompi_list_t);
/* OBJ_CONSTRUCT (&mca_bmi_ib_component.ib_recv_frags, ompi_free_list_t); */ /* OBJ_CONSTRUCT (&mca_btl_ib_component.ib_recv_frags, ompi_free_list_t); */
/* register IB component parameters */ /* register IB component parameters */
mca_bmi_ib_component.ib_free_list_num = mca_btl_ib_component.ib_free_list_num =
mca_bmi_ib_param_register_int ("free_list_num", 8); mca_btl_ib_param_register_int ("free_list_num", 8);
mca_bmi_ib_component.ib_free_list_max = mca_btl_ib_component.ib_free_list_max =
mca_bmi_ib_param_register_int ("free_list_max", 1024); mca_btl_ib_param_register_int ("free_list_max", 1024);
mca_bmi_ib_component.ib_free_list_inc = mca_btl_ib_component.ib_free_list_inc =
mca_bmi_ib_param_register_int ("free_list_inc", 32); mca_btl_ib_param_register_int ("free_list_inc", 32);
mca_bmi_ib_component.ib_mem_registry_hints_log_size = mca_btl_ib_component.ib_mem_registry_hints_log_size =
mca_bmi_ib_param_register_int ("hints_log_size", 8); mca_btl_ib_param_register_int ("hints_log_size", 8);
mca_bmi_ib_component.ib_mpool_name = mca_btl_ib_component.ib_mpool_name =
mca_bmi_ib_param_register_string("mpool", "ib"); mca_btl_ib_param_register_string("mpool", "ib");
mca_bmi_ib_component.ib_rr_buf_max = mca_btl_ib_component.ib_rr_buf_max =
mca_bmi_ib_param_register_int("rr_buf_max", 16); mca_btl_ib_param_register_int("rr_buf_max", 16);
mca_bmi_ib_component.ib_rr_buf_min = mca_btl_ib_component.ib_rr_buf_min =
mca_bmi_ib_param_register_int("rr_buf_min", 8); mca_btl_ib_param_register_int("rr_buf_min", 8);
mca_bmi_ib_component.reg_mru_len = mca_btl_ib_component.reg_mru_len =
mca_bmi_ib_param_register_int("reg_mru_len", 16); mca_btl_ib_param_register_int("reg_mru_len", 16);
mca_bmi_ib_module.super.bmi_exclusivity = mca_btl_ib_module.super.btl_exclusivity =
mca_bmi_ib_param_register_int ("exclusivity", 0); mca_btl_ib_param_register_int ("exclusivity", 0);
mca_bmi_ib_module.super.bmi_eager_limit = mca_btl_ib_module.super.btl_eager_limit =
mca_bmi_ib_param_register_int ("eager_limit", (64*1024)) mca_btl_ib_param_register_int ("eager_limit", (64*1024))
- sizeof(mca_bmi_ib_header_t); - sizeof(mca_btl_ib_header_t);
mca_bmi_ib_module.super.bmi_min_send_size = mca_btl_ib_module.super.btl_min_send_size =
mca_bmi_ib_param_register_int ("min_send_size", (64*1024)) mca_btl_ib_param_register_int ("min_send_size", (64*1024))
- sizeof(mca_bmi_ib_header_t); - sizeof(mca_btl_ib_header_t);
mca_bmi_ib_module.super.bmi_max_send_size = mca_btl_ib_module.super.btl_max_send_size =
mca_bmi_ib_param_register_int ("max_send_size", (128*1024)) mca_btl_ib_param_register_int ("max_send_size", (128*1024))
- sizeof(mca_bmi_ib_header_t); - sizeof(mca_btl_ib_header_t);
mca_bmi_ib_module.ib_pin_min = mca_btl_ib_module.ib_pin_min =
mca_bmi_ib_param_register_int("ib_pin_min", 128*1024); mca_btl_ib_param_register_int("ib_pin_min", 128*1024);
mca_bmi_ib_module.ib_cq_size = mca_btl_ib_module.ib_cq_size =
mca_bmi_ib_param_register_int("ib_cq_size", mca_btl_ib_param_register_int("ib_cq_size",
40000); 40000);
mca_bmi_ib_module.ib_wq_size = mca_btl_ib_module.ib_wq_size =
mca_bmi_ib_param_register_int("ib_wq_size", mca_btl_ib_param_register_int("ib_wq_size",
10000); 10000);
mca_bmi_ib_module.ib_sg_list_size = mca_btl_ib_module.ib_sg_list_size =
mca_bmi_ib_param_register_int("ib_sg_list_size", mca_btl_ib_param_register_int("ib_sg_list_size",
1); 1);
mca_bmi_ib_module.ib_pkey_ix = mca_btl_ib_module.ib_pkey_ix =
mca_bmi_ib_param_register_int("ib_pkey_ix", mca_btl_ib_param_register_int("ib_pkey_ix",
0); 0);
mca_bmi_ib_module.ib_psn = mca_btl_ib_module.ib_psn =
mca_bmi_ib_param_register_int("ib_psn", mca_btl_ib_param_register_int("ib_psn",
0); 0);
mca_bmi_ib_module.ib_qp_ous_rd_atom = mca_btl_ib_module.ib_qp_ous_rd_atom =
mca_bmi_ib_param_register_int("ib_qp_ous_rd_atom", mca_btl_ib_param_register_int("ib_qp_ous_rd_atom",
1); 1);
mca_bmi_ib_module.ib_mtu = mca_btl_ib_module.ib_mtu =
mca_bmi_ib_param_register_int("ib_mtu", mca_btl_ib_param_register_int("ib_mtu",
MTU1024); MTU1024);
mca_bmi_ib_module.ib_min_rnr_timer = mca_btl_ib_module.ib_min_rnr_timer =
mca_bmi_ib_param_register_int("ib_min_rnr_timer", mca_btl_ib_param_register_int("ib_min_rnr_timer",
5); 5);
mca_bmi_ib_module.ib_timeout = mca_btl_ib_module.ib_timeout =
mca_bmi_ib_param_register_int("ib_timeout", mca_btl_ib_param_register_int("ib_timeout",
10); 10);
mca_bmi_ib_module.ib_retry_count = mca_btl_ib_module.ib_retry_count =
mca_bmi_ib_param_register_int("ib_retry_count", mca_btl_ib_param_register_int("ib_retry_count",
7); 7);
mca_bmi_ib_module.ib_rnr_retry = mca_btl_ib_module.ib_rnr_retry =
mca_bmi_ib_param_register_int("ib_rnr_retry", mca_btl_ib_param_register_int("ib_rnr_retry",
7); 7);
mca_bmi_ib_module.ib_max_rdma_dst_ops = mca_btl_ib_module.ib_max_rdma_dst_ops =
mca_bmi_ib_param_register_int("ib_max_rdma_dst_ops", mca_btl_ib_param_register_int("ib_max_rdma_dst_ops",
16); 16);
mca_bmi_ib_module.ib_service_level = mca_btl_ib_module.ib_service_level =
mca_bmi_ib_param_register_int("ib_service_level", mca_btl_ib_param_register_int("ib_service_level",
0); 0);
mca_bmi_ib_module.ib_static_rate = mca_btl_ib_module.ib_static_rate =
mca_bmi_ib_param_register_int("ib_static_rate", mca_btl_ib_param_register_int("ib_static_rate",
0); 0);
mca_bmi_ib_module.ib_src_path_bits = mca_btl_ib_module.ib_src_path_bits =
mca_bmi_ib_param_register_int("ib_src_path_bits", mca_btl_ib_param_register_int("ib_src_path_bits",
0); 0);
mca_bmi_ib_module.super.bmi_min_rdma_size = mca_btl_ib_module.super.btl_min_rdma_size =
mca_bmi_ib_param_register_int("min_rdma_size", mca_btl_ib_param_register_int("min_rdma_size",
1024*1024); 1024*1024);
mca_bmi_ib_module.super.bmi_max_rdma_size = mca_btl_ib_module.super.btl_max_rdma_size =
mca_bmi_ib_param_register_int("max_rdma_size", mca_btl_ib_param_register_int("max_rdma_size",
1024*1024); 1024*1024);
mca_bmi_ib_module.super.bmi_flags = mca_btl_ib_module.super.btl_flags =
mca_bmi_ib_param_register_int("flags", mca_btl_ib_param_register_int("flags",
MCA_BMI_FLAGS_RDMA); MCA_BMI_FLAGS_RDMA);
param = mca_base_param_find("mpi", NULL, "leave_pinned"); param = mca_base_param_find("mpi", NULL, "leave_pinned");
mca_base_param_lookup_int(param, &value); mca_base_param_lookup_int(param, &value);
mca_bmi_ib_component.leave_pinned = value; mca_btl_ib_component.leave_pinned = value;
@ -214,8 +214,8 @@ int mca_bmi_ib_component_open(void)
mca_bmi_ib_component.max_send_size = mca_bmi_ib_module.super.bmi_max_send_size; mca_btl_ib_component.max_send_size = mca_btl_ib_module.super.btl_max_send_size;
mca_bmi_ib_component.eager_limit = mca_bmi_ib_module.super.bmi_eager_limit; mca_btl_ib_component.eager_limit = mca_btl_ib_module.super.btl_eager_limit;
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -224,7 +224,7 @@ int mca_bmi_ib_component_open(void)
* component cleanup - sanity checking of queue lengths * component cleanup - sanity checking of queue lengths
*/ */
int mca_bmi_ib_component_close(void) int mca_btl_ib_component_close(void)
{ {
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -237,7 +237,7 @@ int mca_bmi_ib_component_close(void)
* (3) register BMI parameters with the MCA * (3) register BMI parameters with the MCA
*/ */
mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules, mca_btl_base_module_t** mca_btl_ib_component_init(int *num_btl_modules,
bool enable_progress_threads, bool enable_progress_threads,
bool enable_mpi_threads) bool enable_mpi_threads)
{ {
@ -248,15 +248,15 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
VAPI_hca_cap_t hca_cap; VAPI_hca_cap_t hca_cap;
VAPI_hca_port_t hca_port; VAPI_hca_port_t hca_port;
uint32_t num_hcas; uint32_t num_hcas;
mca_bmi_base_module_t** bmis; mca_btl_base_module_t** btls;
uint32_t i,j, length; uint32_t i,j, length;
struct mca_mpool_base_resources_t hca_pd; struct mca_mpool_base_resources_t hca_pd;
ompi_list_t bmi_list; ompi_list_t btl_list;
mca_bmi_ib_module_t * ib_bmi; mca_btl_ib_module_t * ib_btl;
mca_bmi_base_selected_module_t* ib_selected; mca_btl_base_selected_module_t* ib_selected;
ompi_list_item_t* item; ompi_list_item_t* item;
/* initialization */ /* initialization */
*num_bmi_modules = 0; *num_btl_modules = 0;
/* Determine the number of hca's available on the host */ /* Determine the number of hca's available on the host */
vapi_ret=EVAPI_list_hcas(0, &num_hcas, NULL); vapi_ret=EVAPI_list_hcas(0, &num_hcas, NULL);
@ -281,10 +281,10 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
/** We must loop through all the hca id's, get there handles and /** We must loop through all the hca id's, get there handles and
for each hca we query the number of ports on the hca and set up for each hca we query the number of ports on the hca and set up
a distinct bmi module for each hca port */ a distinct btl module for each hca port */
OBJ_CONSTRUCT(&bmi_list, ompi_list_t); OBJ_CONSTRUCT(&btl_list, ompi_list_t);
OBJ_CONSTRUCT(&mca_bmi_ib_component.ib_lock, ompi_mutex_t); OBJ_CONSTRUCT(&mca_btl_ib_component.ib_lock, ompi_mutex_t);
for(i = 0; i < num_hcas; i++){ for(i = 0; i < num_hcas; i++){
@ -312,17 +312,17 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
if( PORT_ACTIVE == hca_port.state ){ if( PORT_ACTIVE == hca_port.state ){
ib_bmi = (mca_bmi_ib_module_t*) malloc(sizeof(mca_bmi_ib_module_t)); ib_btl = (mca_btl_ib_module_t*) malloc(sizeof(mca_btl_ib_module_t));
memcpy(ib_bmi, &mca_bmi_ib_module, sizeof(mca_bmi_ib_module)); memcpy(ib_btl, &mca_btl_ib_module, sizeof(mca_btl_ib_module));
ib_selected = OBJ_NEW(mca_bmi_base_selected_module_t); ib_selected = OBJ_NEW(mca_btl_base_selected_module_t);
ib_selected->bmi_module = (mca_bmi_base_module_t*) ib_bmi; ib_selected->btl_module = (mca_btl_base_module_t*) ib_btl;
memcpy(ib_bmi->hca_id, hca_ids[i], sizeof(VAPI_hca_id_t)); memcpy(ib_btl->hca_id, hca_ids[i], sizeof(VAPI_hca_id_t));
ib_bmi->nic = hca_hndl; ib_btl->nic = hca_hndl;
ib_bmi->port_id = (IB_port_t) j; ib_btl->port_id = (IB_port_t) j;
ib_bmi->port = hca_port; ib_btl->port = hca_port;
ompi_list_append(&bmi_list, (ompi_list_item_t*) ib_selected); ompi_list_append(&btl_list, (ompi_list_item_t*) ib_selected);
mca_bmi_ib_component.ib_num_bmis ++; mca_btl_ib_component.ib_num_btls ++;
} }
} }
@ -330,154 +330,154 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
} }
/* Allocate space for bmi modules */ /* Allocate space for btl modules */
mca_bmi_ib_component.ib_bmis = (mca_bmi_ib_module_t*) malloc(sizeof(mca_bmi_ib_module_t) * mca_btl_ib_component.ib_btls = (mca_btl_ib_module_t*) malloc(sizeof(mca_btl_ib_module_t) *
mca_bmi_ib_component.ib_num_bmis); mca_btl_ib_component.ib_num_btls);
if(NULL == mca_bmi_ib_component.ib_bmis) { if(NULL == mca_btl_ib_component.ib_btls) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return NULL; return NULL;
} }
bmis = (struct mca_bmi_base_module_t**) btls = (struct mca_btl_base_module_t**)
malloc(mca_bmi_ib_component.ib_num_bmis * sizeof(struct mca_bmi_ib_module_t*)); malloc(mca_btl_ib_component.ib_num_btls * sizeof(struct mca_btl_ib_module_t*));
if(NULL == bmis) { if(NULL == btls) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return NULL; return NULL;
} }
for(i = 0; i < mca_bmi_ib_component.ib_num_bmis; i++){ for(i = 0; i < mca_btl_ib_component.ib_num_btls; i++){
item = ompi_list_remove_first(&bmi_list); item = ompi_list_remove_first(&btl_list);
ib_selected = (mca_bmi_base_selected_module_t*)item; ib_selected = (mca_btl_base_selected_module_t*)item;
ib_bmi = (mca_bmi_ib_module_t*) ib_selected->bmi_module; ib_btl = (mca_btl_ib_module_t*) ib_selected->btl_module;
memcpy(&(mca_bmi_ib_component.ib_bmis[i]), ib_bmi , sizeof(mca_bmi_ib_module_t)); memcpy(&(mca_btl_ib_component.ib_btls[i]), ib_btl , sizeof(mca_btl_ib_module_t));
free(ib_selected); free(ib_selected);
free(ib_bmi); free(ib_btl);
ib_bmi = &mca_bmi_ib_component.ib_bmis[i]; ib_btl = &mca_btl_ib_component.ib_btls[i];
/* Initialize the modules function pointers */ /* Initialize the modules function pointers */
/* Initialize module state */ /* Initialize module state */
OBJ_CONSTRUCT(&ib_bmi->ib_lock, ompi_mutex_t); OBJ_CONSTRUCT(&ib_btl->ib_lock, ompi_mutex_t);
OBJ_CONSTRUCT(&ib_bmi->send_free_eager, ompi_free_list_t); OBJ_CONSTRUCT(&ib_btl->send_free_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->send_free_max, ompi_free_list_t); OBJ_CONSTRUCT(&ib_btl->send_free_max, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->send_free_frag, ompi_free_list_t); OBJ_CONSTRUCT(&ib_btl->send_free_frag, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->recv_free_eager, ompi_free_list_t); OBJ_CONSTRUCT(&ib_btl->recv_free_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->recv_free_max, ompi_free_list_t); OBJ_CONSTRUCT(&ib_btl->recv_free_max, ompi_free_list_t);
OBJ_CONSTRUCT(&ib_bmi->repost, ompi_list_t); OBJ_CONSTRUCT(&ib_btl->repost, ompi_list_t);
OBJ_CONSTRUCT(&ib_bmi->reg_mru_list, ompi_list_t); OBJ_CONSTRUCT(&ib_btl->reg_mru_list, ompi_list_t);
if(mca_bmi_ib_module_init(ib_bmi) != OMPI_SUCCESS) { if(mca_btl_ib_module_init(ib_btl) != OMPI_SUCCESS) {
free(hca_ids); free(hca_ids);
return NULL; return NULL;
} }
hca_pd.hca = ib_bmi->nic; hca_pd.hca = ib_btl->nic;
hca_pd.pd_tag = ib_bmi->ptag; hca_pd.pd_tag = ib_btl->ptag;
/* initialize the memory pool using the hca */ /* initialize the memory pool using the hca */
ib_bmi->ib_pool = ib_btl->ib_pool =
mca_mpool_base_module_create(mca_bmi_ib_component.ib_mpool_name, mca_mpool_base_module_create(mca_btl_ib_component.ib_mpool_name,
&ib_bmi->super, &ib_btl->super,
&hca_pd); &hca_pd);
if(NULL == ib_bmi->ib_pool) { if(NULL == ib_btl->ib_pool) {
ompi_output(0, "%s: error creating vapi memory pool! aborting ib bmi initialization", __func__); ompi_output(0, "%s: error creating vapi memory pool! aborting ib btl initialization", __func__);
return NULL; return NULL;
} }
/* Initialize pool of send fragments */ /* Initialize pool of send fragments */
length = sizeof(mca_bmi_ib_frag_t) + length = sizeof(mca_btl_ib_frag_t) +
sizeof(mca_bmi_ib_header_t) + sizeof(mca_btl_ib_header_t) +
ib_bmi->super.bmi_eager_limit+ ib_btl->super.btl_eager_limit+
2*MCA_BMI_IB_FRAG_ALIGN; 2*MCA_BMI_IB_FRAG_ALIGN;
ompi_free_list_init(&ib_bmi->send_free_eager, ompi_free_list_init(&ib_btl->send_free_eager,
length, length,
OBJ_CLASS(mca_bmi_ib_send_frag_eager_t), OBJ_CLASS(mca_btl_ib_send_frag_eager_t),
mca_bmi_ib_component.ib_free_list_num, mca_btl_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max, mca_btl_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc, mca_btl_ib_component.ib_free_list_inc,
ib_bmi->ib_pool); ib_btl->ib_pool);
ompi_free_list_init(&ib_bmi->recv_free_eager, ompi_free_list_init(&ib_btl->recv_free_eager,
length, length,
OBJ_CLASS(mca_bmi_ib_recv_frag_eager_t), OBJ_CLASS(mca_btl_ib_recv_frag_eager_t),
mca_bmi_ib_component.ib_free_list_num, mca_btl_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max, mca_btl_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc, mca_btl_ib_component.ib_free_list_inc,
ib_bmi->ib_pool); ib_btl->ib_pool);
length = sizeof(mca_bmi_ib_frag_t) + length = sizeof(mca_btl_ib_frag_t) +
sizeof(mca_bmi_ib_header_t) + sizeof(mca_btl_ib_header_t) +
ib_bmi->super.bmi_max_send_size+ ib_btl->super.btl_max_send_size+
2*MCA_BMI_IB_FRAG_ALIGN; 2*MCA_BMI_IB_FRAG_ALIGN;
ompi_free_list_init(&ib_bmi->send_free_max, ompi_free_list_init(&ib_btl->send_free_max,
length, length,
OBJ_CLASS(mca_bmi_ib_send_frag_max_t), OBJ_CLASS(mca_btl_ib_send_frag_max_t),
mca_bmi_ib_component.ib_free_list_num, mca_btl_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max, mca_btl_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc, mca_btl_ib_component.ib_free_list_inc,
ib_bmi->ib_pool); ib_btl->ib_pool);
/* Initialize pool of receive fragments */ /* Initialize pool of receive fragments */
ompi_free_list_init (&ib_bmi->recv_free_max, ompi_free_list_init (&ib_btl->recv_free_max,
length, length,
OBJ_CLASS (mca_bmi_ib_recv_frag_max_t), OBJ_CLASS (mca_btl_ib_recv_frag_max_t),
mca_bmi_ib_component.ib_free_list_num, mca_btl_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max, mca_btl_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc, ib_bmi->ib_pool); mca_btl_ib_component.ib_free_list_inc, ib_btl->ib_pool);
length = sizeof(mca_bmi_ib_frag_t) + length = sizeof(mca_btl_ib_frag_t) +
sizeof(mca_bmi_ib_header_t)+ sizeof(mca_btl_ib_header_t)+
2*MCA_BMI_IB_FRAG_ALIGN; 2*MCA_BMI_IB_FRAG_ALIGN;
ompi_free_list_init(&ib_bmi->send_free_frag, ompi_free_list_init(&ib_btl->send_free_frag,
length, length,
OBJ_CLASS(mca_bmi_ib_send_frag_frag_t), OBJ_CLASS(mca_btl_ib_send_frag_frag_t),
mca_bmi_ib_component.ib_free_list_num, mca_btl_ib_component.ib_free_list_num,
mca_bmi_ib_component.ib_free_list_max, mca_btl_ib_component.ib_free_list_max,
mca_bmi_ib_component.ib_free_list_inc, mca_btl_ib_component.ib_free_list_inc,
ib_bmi->ib_pool); ib_btl->ib_pool);
/* Initialize the rr_desc_post array for posting of rr*/ /* Initialize the rr_desc_post array for posting of rr*/
ib_bmi->rr_desc_post = (VAPI_rr_desc_t*) malloc((mca_bmi_ib_component.ib_rr_buf_max * sizeof(VAPI_rr_desc_t))); ib_btl->rr_desc_post = (VAPI_rr_desc_t*) malloc((mca_btl_ib_component.ib_rr_buf_max * sizeof(VAPI_rr_desc_t)));
/* This is now done by the memory pool passed to free_list_init.. Initialize the send descriptors */ /* This is now done by the memory pool passed to free_list_init.. Initialize the send descriptors */
/* if(mca_bmi_ib_send_frag_register(ib_bmi) != OMPI_SUCCESS) { */ /* if(mca_btl_ib_send_frag_register(ib_btl) != OMPI_SUCCESS) { */
/* free(hca_ids); */ /* free(hca_ids); */
/* return NULL; */ /* return NULL; */
/* } */ /* } */
bmis[i] = &ib_bmi->super; btls[i] = &ib_btl->super;
} }
/* Post OOB receive to support dynamic connection setup */ /* Post OOB receive to support dynamic connection setup */
mca_bmi_ib_post_recv(); mca_btl_ib_post_recv();
*num_bmi_modules = mca_bmi_ib_component.ib_num_bmis; *num_btl_modules = mca_btl_ib_component.ib_num_btls;
free(hca_ids); free(hca_ids);
return bmis; return btls;
} }
/* /*
@ -485,19 +485,19 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
*/ */
int mca_bmi_ib_component_progress() int mca_btl_ib_component_progress()
{ {
uint32_t i; uint32_t i;
int count = 0; int count = 0;
mca_bmi_ib_frag_t* frag; mca_btl_ib_frag_t* frag;
/* Poll for completions */ /* Poll for completions */
for(i = 0; i < mca_bmi_ib_component.ib_num_bmis; i++) { for(i = 0; i < mca_btl_ib_component.ib_num_btls; i++) {
VAPI_ret_t ret; VAPI_ret_t ret;
VAPI_wc_desc_t comp; VAPI_wc_desc_t comp;
mca_bmi_ib_module_t* ib_bmi = &mca_bmi_ib_component.ib_bmis[i]; mca_btl_ib_module_t* ib_btl = &mca_btl_ib_component.ib_btls[i];
do{ do{
ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_high, &comp); ret = VAPI_poll_cq(ib_btl->nic, ib_btl->cq_hndl_high, &comp);
if(VAPI_OK == ret) { if(VAPI_OK == ret) {
if(comp.status != VAPI_SUCCESS) { if(comp.status != VAPI_SUCCESS) {
ompi_output(0, "Got error : %s, Vendor code : %d Frag : %p", ompi_output(0, "Got error : %s, Vendor code : %d Frag : %p",
@ -518,25 +518,25 @@ int mca_bmi_ib_component_progress()
case VAPI_CQE_SQ_SEND_DATA : case VAPI_CQE_SQ_SEND_DATA :
/* Process a completed send */ /* Process a completed send */
frag = (mca_bmi_ib_frag_t*) comp.id; frag = (mca_btl_ib_frag_t*) comp.id;
frag->rc = OMPI_SUCCESS; frag->rc = OMPI_SUCCESS;
frag->base.des_cbfunc(&ib_bmi->super, frag->endpoint, &frag->base, frag->rc); frag->base.des_cbfunc(&ib_btl->super, frag->endpoint, &frag->base, frag->rc);
count++; count++;
break; break;
case VAPI_CQE_RQ_SEND_DATA: case VAPI_CQE_RQ_SEND_DATA:
DEBUG_OUT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__); DEBUG_OUT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
frag = (mca_bmi_ib_frag_t*) comp.id; frag = (mca_btl_ib_frag_t*) comp.id;
frag->rc=OMPI_SUCCESS; frag->rc=OMPI_SUCCESS;
frag->segment.seg_len = comp.byte_len-((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr); frag->segment.seg_len = comp.byte_len-((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);
/* advance the segment address past the header and subtract from the length..*/ /* advance the segment address past the header and subtract from the length..*/
ib_bmi->ib_reg[frag->hdr->tag].cbfunc(&ib_bmi->super, frag->hdr->tag, &frag->base, ib_bmi->ib_reg[frag->hdr->tag].cbdata); ib_btl->ib_reg[frag->hdr->tag].cbfunc(&ib_btl->super, frag->hdr->tag, &frag->base, ib_btl->ib_reg[frag->hdr->tag].cbdata);
OMPI_FREE_LIST_RETURN(&(ib_bmi->recv_free_eager), (ompi_list_item_t*) frag); OMPI_FREE_LIST_RETURN(&(ib_btl->recv_free_eager), (ompi_list_item_t*) frag);
OMPI_THREAD_ADD32(&ib_bmi->rr_posted_high, -1); OMPI_THREAD_ADD32(&ib_btl->rr_posted_high, -1);
mca_bmi_ib_endpoint_post_rr(((mca_bmi_ib_frag_t*)comp.id)->endpoint, 0); mca_btl_ib_endpoint_post_rr(((mca_btl_ib_frag_t*)comp.id)->endpoint, 0);
count++; count++;
break; break;
@ -549,7 +549,7 @@ int mca_bmi_ib_component_progress()
} }
while(VAPI_OK == ret); while(VAPI_OK == ret);
ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_low, &comp); ret = VAPI_poll_cq(ib_btl->nic, ib_btl->cq_hndl_low, &comp);
if(VAPI_OK == ret) { if(VAPI_OK == ret) {
if(comp.status != VAPI_SUCCESS) { if(comp.status != VAPI_SUCCESS) {
ompi_output(0, "Got error : %s, Vendor code : %d Frag : %p", ompi_output(0, "Got error : %s, Vendor code : %d Frag : %p",
@ -564,26 +564,26 @@ int mca_bmi_ib_component_progress()
case VAPI_CQE_SQ_SEND_DATA : case VAPI_CQE_SQ_SEND_DATA :
/* Process a completed send */ /* Process a completed send */
frag = (mca_bmi_ib_frag_t*) comp.id; frag = (mca_btl_ib_frag_t*) comp.id;
frag->rc = OMPI_SUCCESS; frag->rc = OMPI_SUCCESS;
frag->base.des_cbfunc(&ib_bmi->super, frag->endpoint, &frag->base, frag->rc); frag->base.des_cbfunc(&ib_btl->super, frag->endpoint, &frag->base, frag->rc);
count++; count++;
break; break;
case VAPI_CQE_RQ_SEND_DATA: case VAPI_CQE_RQ_SEND_DATA:
DEBUG_OUT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__); DEBUG_OUT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
frag = (mca_bmi_ib_frag_t*) comp.id; frag = (mca_btl_ib_frag_t*) comp.id;
frag->rc=OMPI_SUCCESS; frag->rc=OMPI_SUCCESS;
frag->segment.seg_len = comp.byte_len-((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr); frag->segment.seg_len = comp.byte_len-((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);
/* advance the segment address past the header and subtract from the length..*/ /* advance the segment address past the header and subtract from the length..*/
ib_bmi->ib_reg[frag->hdr->tag].cbfunc(&ib_bmi->super, frag->hdr->tag, &frag->base, ib_bmi->ib_reg[frag->hdr->tag].cbdata); ib_btl->ib_reg[frag->hdr->tag].cbfunc(&ib_btl->super, frag->hdr->tag, &frag->base, ib_btl->ib_reg[frag->hdr->tag].cbdata);
OMPI_FREE_LIST_RETURN(&(ib_bmi->recv_free_max), (ompi_list_item_t*) frag); OMPI_FREE_LIST_RETURN(&(ib_btl->recv_free_max), (ompi_list_item_t*) frag);
OMPI_THREAD_ADD32(&ib_bmi->rr_posted_low, -1); OMPI_THREAD_ADD32(&ib_btl->rr_posted_low, -1);
mca_bmi_ib_endpoint_post_rr(((mca_bmi_ib_frag_t*)comp.id)->endpoint, 0); mca_btl_ib_endpoint_post_rr(((mca_btl_ib_frag_t*)comp.id)->endpoint, 0);
count++; count++;
break; break;

240
src/mca/btl/mvapi/bmi_ib_endpoint.c → src/mca/btl/mvapi/btl_ib_endpoint.c
Просмотреть файл

@ -25,17 +25,17 @@
#include "mca/rml/rml.h" #include "mca/rml/rml.h"
#include "mca/errmgr/errmgr.h" #include "mca/errmgr/errmgr.h"
#include "dps/dps.h" #include "dps/dps.h"
#include "bmi_ib.h" #include "btl_ib.h"
#include "bmi_ib_endpoint.h" #include "btl_ib_endpoint.h"
#include "bmi_ib_proc.h" #include "btl_ib_proc.h"
#include "bmi_ib_frag.h" #include "btl_ib_frag.h"
#include "class/ompi_free_list.h" #include "class/ompi_free_list.h"
static void mca_bmi_ib_endpoint_construct(mca_bmi_base_endpoint_t* endpoint); static void mca_btl_ib_endpoint_construct(mca_btl_base_endpoint_t* endpoint);
static void mca_bmi_ib_endpoint_destruct(mca_bmi_base_endpoint_t* endpoint); static void mca_btl_ib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint);
int mca_bmi_ib_endpoint_create_qp( int mca_btl_ib_endpoint_create_qp(
mca_bmi_ib_module_t* ib_bmi, mca_btl_ib_module_t* ib_btl,
VAPI_hca_hndl_t nic, VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag, VAPI_pd_hndl_t ptag,
VAPI_cq_hndl_t cq_hndl, VAPI_cq_hndl_t cq_hndl,
@ -44,9 +44,9 @@ int mca_bmi_ib_endpoint_create_qp(
int transport_type); int transport_type);
int mca_bmi_ib_endpoint_qp_init_query( int mca_btl_ib_endpoint_qp_init_query(
mca_bmi_ib_module_t* ib_bmi, mca_btl_ib_module_t* ib_btl,
VAPI_hca_hndl_t nic, VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl, VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp_num, VAPI_qp_num_t remote_qp_num,
@ -55,14 +55,14 @@ int mca_bmi_ib_endpoint_qp_init_query(
); );
static inline int mca_bmi_ib_endpoint_post_send(mca_bmi_ib_module_t* ib_bmi, mca_bmi_ib_endpoint_t * endpoint, mca_bmi_ib_frag_t * frag) static inline int mca_btl_ib_endpoint_post_send(mca_btl_ib_module_t* ib_btl, mca_btl_ib_endpoint_t * endpoint, mca_btl_ib_frag_t * frag)
{ {
frag->sr_desc.remote_qkey = 0; frag->sr_desc.remote_qkey = 0;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr; frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
VAPI_qp_hndl_t qp_hndl; VAPI_qp_hndl_t qp_hndl;
if(frag->base.des_flags && MCA_BMI_DES_FLAGS_PRIORITY && frag->size <= ib_bmi->super.bmi_eager_limit){ if(frag->base.des_flags && MCA_BMI_DES_FLAGS_PRIORITY && frag->size <= ib_btl->super.btl_eager_limit){
frag->sr_desc.remote_qp = endpoint->rem_qp_num_high; frag->sr_desc.remote_qp = endpoint->rem_qp_num_high;
qp_hndl = endpoint->lcl_qp_hndl_high; qp_hndl = endpoint->lcl_qp_hndl_high;
} else { } else {
@ -70,15 +70,15 @@ static inline int mca_bmi_ib_endpoint_post_send(mca_bmi_ib_module_t* ib_bmi, mca
qp_hndl = endpoint->lcl_qp_hndl_low; qp_hndl = endpoint->lcl_qp_hndl_low;
} }
frag->sr_desc.opcode = VAPI_SEND; frag->sr_desc.opcode = VAPI_SEND;
frag->sg_entry.len = frag->segment.seg_len + ((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr); /* sizeof(mca_bmi_ib_header_t); */ frag->sg_entry.len = frag->segment.seg_len + ((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr); /* sizeof(mca_btl_ib_header_t); */
if(frag->sg_entry.len <= ib_bmi->ib_inline_max) { if(frag->sg_entry.len <= ib_btl->ib_inline_max) {
frag->ret = EVAPI_post_inline_sr(ib_bmi->nic, frag->ret = EVAPI_post_inline_sr(ib_btl->nic,
qp_hndl, qp_hndl,
&frag->sr_desc); &frag->sr_desc);
}else { }else {
frag->ret = VAPI_post_sr(ib_bmi->nic, frag->ret = VAPI_post_sr(ib_btl->nic,
qp_hndl, qp_hndl,
&frag->sr_desc); &frag->sr_desc);
} }
@ -88,25 +88,25 @@ static inline int mca_bmi_ib_endpoint_post_send(mca_bmi_ib_module_t* ib_bmi, mca
if(VAPI_OK != frag->ret) if(VAPI_OK != frag->ret)
return OMPI_ERROR; return OMPI_ERROR;
mca_bmi_ib_endpoint_post_rr(endpoint, 1); mca_btl_ib_endpoint_post_rr(endpoint, 1);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
OBJ_CLASS_INSTANCE(mca_bmi_ib_endpoint_t, OBJ_CLASS_INSTANCE(mca_btl_ib_endpoint_t,
ompi_list_item_t, mca_bmi_ib_endpoint_construct, ompi_list_item_t, mca_btl_ib_endpoint_construct,
mca_bmi_ib_endpoint_destruct); mca_btl_ib_endpoint_destruct);
/* /*
* Initialize state of the endpoint instance. * Initialize state of the endpoint instance.
* *
*/ */
static void mca_bmi_ib_endpoint_construct(mca_bmi_base_endpoint_t* endpoint) static void mca_btl_ib_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
{ {
endpoint->endpoint_bmi = 0; endpoint->endpoint_btl = 0;
endpoint->endpoint_proc = 0; endpoint->endpoint_proc = 0;
endpoint->endpoint_tstamp = 0.0; endpoint->endpoint_tstamp = 0.0;
endpoint->endpoint_state = MCA_BMI_IB_CLOSED; endpoint->endpoint_state = MCA_BMI_IB_CLOSED;
@ -121,7 +121,7 @@ static void mca_bmi_ib_endpoint_construct(mca_bmi_base_endpoint_t* endpoint)
* *
*/ */
static void mca_bmi_ib_endpoint_destruct(mca_bmi_base_endpoint_t* endpoint) static void mca_btl_ib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
{ {
} }
@ -130,7 +130,7 @@ static void mca_bmi_ib_endpoint_destruct(mca_bmi_base_endpoint_t* endpoint)
* *
*/ */
static void mca_bmi_ib_endpoint_send_cb( static void mca_btl_ib_endpoint_send_cb(
int status, int status,
orte_process_name_t* endpoint, orte_process_name_t* endpoint,
orte_buffer_t* buffer, orte_buffer_t* buffer,
@ -141,7 +141,7 @@ static void mca_bmi_ib_endpoint_send_cb(
} }
static int mca_bmi_ib_endpoint_send_connect_req(mca_bmi_base_endpoint_t* endpoint) static int mca_btl_ib_endpoint_send_connect_req(mca_btl_base_endpoint_t* endpoint)
{ {
orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t); orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t);
int rc; int rc;
@ -163,7 +163,7 @@ static int mca_bmi_ib_endpoint_send_connect_req(mca_bmi_base_endpoint_t* endpoin
ORTE_ERROR_LOG(rc); ORTE_ERROR_LOG(rc);
return rc; return rc;
} }
rc = orte_dps.pack(buffer, &endpoint->endpoint_bmi->port.lid, 1, ORTE_UINT32); rc = orte_dps.pack(buffer, &endpoint->endpoint_btl->port.lid, 1, ORTE_UINT32);
if(rc != ORTE_SUCCESS) { if(rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc); ORTE_ERROR_LOG(rc);
return rc; return rc;
@ -171,13 +171,13 @@ static int mca_bmi_ib_endpoint_send_connect_req(mca_bmi_base_endpoint_t* endpoin
/* send to endpoint */ /* send to endpoint */
rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_guid, buffer, ORTE_RML_TAG_DYNAMIC-1, 0, rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_guid, buffer, ORTE_RML_TAG_DYNAMIC-1, 0,
mca_bmi_ib_endpoint_send_cb, NULL); mca_btl_ib_endpoint_send_cb, NULL);
DEBUG_OUT("Sending High Priority QP num = %d, Low Priority QP num = %d, LID = %d", DEBUG_OUT("Sending High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_prop_high.qp_num, endpoint->lcl_qp_prop_high.qp_num,
endpoint->lcl_qp_prop_low.qp_num, endpoint->lcl_qp_prop_low.qp_num,
endpoint->endpoint_bmi->port.lid); endpoint->endpoint_btl->port.lid);
if(rc < 0) { if(rc < 0) {
ORTE_ERROR_LOG(rc); ORTE_ERROR_LOG(rc);
@ -191,7 +191,7 @@ static int mca_bmi_ib_endpoint_send_connect_req(mca_bmi_base_endpoint_t* endpoin
* *
*/ */
static int mca_bmi_ib_endpoint_send_connect_ack(mca_bmi_base_endpoint_t* endpoint) static int mca_btl_ib_endpoint_send_connect_ack(mca_btl_base_endpoint_t* endpoint)
{ {
orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t); orte_buffer_t* buffer = OBJ_NEW(orte_buffer_t);
int rc; int rc;
@ -213,7 +213,7 @@ static int mca_bmi_ib_endpoint_send_connect_ack(mca_bmi_base_endpoint_t* endpoin
/* send to endpoint */ /* send to endpoint */
rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_guid, buffer, ORTE_RML_TAG_DYNAMIC-1, 0, rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_guid, buffer, ORTE_RML_TAG_DYNAMIC-1, 0,
mca_bmi_ib_endpoint_send_cb, NULL); mca_btl_ib_endpoint_send_cb, NULL);
if(rc < 0) { if(rc < 0) {
ORTE_ERROR_LOG(rc); ORTE_ERROR_LOG(rc);
} }
@ -228,7 +228,7 @@ static int mca_bmi_ib_endpoint_send_connect_ack(mca_bmi_base_endpoint_t* endpoin
* setup. * setup.
* *
*/ */
static int mca_bmi_ib_endpoint_set_remote_info(mca_bmi_base_endpoint_t* endpoint, orte_buffer_t* buffer) static int mca_btl_ib_endpoint_set_remote_info(mca_btl_base_endpoint_t* endpoint, orte_buffer_t* buffer)
{ {
int rc; int rc;
@ -269,16 +269,16 @@ static int mca_bmi_ib_endpoint_set_remote_info(mca_bmi_base_endpoint_t* endpoint
* *
*/ */
static int mca_bmi_ib_endpoint_start_connect(mca_bmi_base_endpoint_t* endpoint) static int mca_btl_ib_endpoint_start_connect(mca_btl_base_endpoint_t* endpoint)
{ {
int rc; int rc;
/* Create the High Priority Queue Pair */ /* Create the High Priority Queue Pair */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_create_qp(endpoint->endpoint_bmi, if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_create_qp(endpoint->endpoint_btl,
endpoint->endpoint_bmi->nic, endpoint->endpoint_btl->nic,
endpoint->endpoint_bmi->ptag, endpoint->endpoint_btl->ptag,
endpoint->endpoint_bmi->cq_hndl_high, endpoint->endpoint_btl->cq_hndl_high,
&endpoint->lcl_qp_hndl_high, &endpoint->lcl_qp_hndl_high,
&endpoint->lcl_qp_prop_high, &endpoint->lcl_qp_prop_high,
VAPI_TS_RC))) { VAPI_TS_RC))) {
@ -289,10 +289,10 @@ static int mca_bmi_ib_endpoint_start_connect(mca_bmi_base_endpoint_t* endpoint)
/* Create the Low Priority Queue Pair */ /* Create the Low Priority Queue Pair */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_create_qp(endpoint->endpoint_bmi, if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_create_qp(endpoint->endpoint_btl,
endpoint->endpoint_bmi->nic, endpoint->endpoint_btl->nic,
endpoint->endpoint_bmi->ptag, endpoint->endpoint_btl->ptag,
endpoint->endpoint_bmi->cq_hndl_low, endpoint->endpoint_btl->cq_hndl_low,
&endpoint->lcl_qp_hndl_low, &endpoint->lcl_qp_hndl_low,
&endpoint->lcl_qp_prop_low, &endpoint->lcl_qp_prop_low,
VAPI_TS_RC))) { VAPI_TS_RC))) {
@ -304,11 +304,11 @@ static int mca_bmi_ib_endpoint_start_connect(mca_bmi_base_endpoint_t* endpoint)
DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d", DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_prop_high.qp_num, endpoint->lcl_qp_prop_high.qp_num,
endpoint->lcl_qp_prop_low.qp_num, endpoint->lcl_qp_prop_low.qp_num,
ib_bmi->port.lid); ib_btl->port.lid);
/* Send connection info over to remote endpoint */ /* Send connection info over to remote endpoint */
endpoint->endpoint_state = MCA_BMI_IB_CONNECTING; endpoint->endpoint_state = MCA_BMI_IB_CONNECTING;
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_send_connect_req(endpoint))) { if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_send_connect_req(endpoint))) {
ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n", ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc); ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
return rc; return rc;
@ -320,16 +320,16 @@ static int mca_bmi_ib_endpoint_start_connect(mca_bmi_base_endpoint_t* endpoint)
* Reply to a `start - connect' message * Reply to a `start - connect' message
* *
*/ */
static int mca_bmi_ib_endpoint_reply_start_connect(mca_bmi_ib_endpoint_t *endpoint, orte_buffer_t* buffer) static int mca_btl_ib_endpoint_reply_start_connect(mca_btl_ib_endpoint_t *endpoint, orte_buffer_t* buffer)
{ {
int rc; int rc;
/* Create the High Priority Queue Pair */ /* Create the High Priority Queue Pair */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_create_qp(endpoint->endpoint_bmi, if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_create_qp(endpoint->endpoint_btl,
endpoint->endpoint_bmi->nic, endpoint->endpoint_btl->nic,
endpoint->endpoint_bmi->ptag, endpoint->endpoint_btl->ptag,
endpoint->endpoint_bmi->cq_hndl_high, endpoint->endpoint_btl->cq_hndl_high,
&endpoint->lcl_qp_hndl_high, &endpoint->lcl_qp_hndl_high,
&endpoint->lcl_qp_prop_high, &endpoint->lcl_qp_prop_high,
VAPI_TS_RC))) { VAPI_TS_RC))) {
@ -340,10 +340,10 @@ static int mca_bmi_ib_endpoint_reply_start_connect(mca_bmi_ib_endpoint_t *endpoi
/* Create the Low Priority Queue Pair */ /* Create the Low Priority Queue Pair */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_create_qp(endpoint->endpoint_bmi, if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_create_qp(endpoint->endpoint_btl,
endpoint->endpoint_bmi->nic, endpoint->endpoint_btl->nic,
endpoint->endpoint_bmi->ptag, endpoint->endpoint_btl->ptag,
endpoint->endpoint_bmi->cq_hndl_low, endpoint->endpoint_btl->cq_hndl_low,
&endpoint->lcl_qp_hndl_low, &endpoint->lcl_qp_hndl_low,
&endpoint->lcl_qp_prop_low, &endpoint->lcl_qp_prop_low,
VAPI_TS_RC))) { VAPI_TS_RC))) {
@ -355,17 +355,17 @@ static int mca_bmi_ib_endpoint_reply_start_connect(mca_bmi_ib_endpoint_t *endpoi
DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d", DEBUG_OUT("Initialized High Priority QP num = %d, Low Priority QP num = %d, LID = %d",
endpoint->lcl_qp_prop_high.qp_num, endpoint->lcl_qp_prop_high.qp_num,
endpoint->lcl_qp_prop_low.qp_num, endpoint->lcl_qp_prop_low.qp_num,
ib_bmi->port.lid); ib_btl->port.lid);
/* Set the remote side info */ /* Set the remote side info */
mca_bmi_ib_endpoint_set_remote_info(endpoint, buffer); mca_btl_ib_endpoint_set_remote_info(endpoint, buffer);
/* Connect to endpoint */ /* Connect to endpoint */
rc = mca_bmi_ib_endpoint_connect(endpoint); rc = mca_btl_ib_endpoint_connect(endpoint);
if(rc != OMPI_SUCCESS) { if(rc != OMPI_SUCCESS) {
ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n", ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc); ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
@ -373,7 +373,7 @@ static int mca_bmi_ib_endpoint_reply_start_connect(mca_bmi_ib_endpoint_t *endpoi
} }
/* Send connection info over to remote endpoint */ /* Send connection info over to remote endpoint */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_send_connect_req(endpoint))) { if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_send_connect_req(endpoint))) {
ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n", ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc); ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
return rc; return rc;
@ -385,10 +385,10 @@ static int mca_bmi_ib_endpoint_reply_start_connect(mca_bmi_ib_endpoint_t *endpoi
* *
*/ */
static void mca_bmi_ib_endpoint_connected(mca_bmi_ib_endpoint_t *endpoint) static void mca_btl_ib_endpoint_connected(mca_btl_ib_endpoint_t *endpoint)
{ {
endpoint->endpoint_state = MCA_BMI_IB_CONNECTED; endpoint->endpoint_state = MCA_BMI_IB_CONNECTED;
mca_bmi_ib_progress_send_frags(endpoint); mca_btl_ib_progress_send_frags(endpoint);
} }
/* /*
@ -400,23 +400,23 @@ static void mca_bmi_ib_endpoint_connected(mca_bmi_ib_endpoint_t *endpoint)
* *
*/ */
static void mca_bmi_ib_endpoint_recv( static void mca_btl_ib_endpoint_recv(
int status, int status,
orte_process_name_t* endpoint, orte_process_name_t* endpoint,
orte_buffer_t* buffer, orte_buffer_t* buffer,
orte_rml_tag_t tag, orte_rml_tag_t tag,
void* cbdata) void* cbdata)
{ {
mca_bmi_ib_proc_t *ib_proc; mca_btl_ib_proc_t *ib_proc;
mca_bmi_ib_endpoint_t *ib_endpoint; mca_btl_ib_endpoint_t *ib_endpoint;
int endpoint_state; int endpoint_state;
int rc; int rc;
for(ib_proc = (mca_bmi_ib_proc_t*) for(ib_proc = (mca_btl_ib_proc_t*)
ompi_list_get_first(&mca_bmi_ib_component.ib_procs); ompi_list_get_first(&mca_btl_ib_component.ib_procs);
ib_proc != (mca_bmi_ib_proc_t*) ib_proc != (mca_btl_ib_proc_t*)
ompi_list_get_end(&mca_bmi_ib_component.ib_procs); ompi_list_get_end(&mca_btl_ib_component.ib_procs);
ib_proc = (mca_bmi_ib_proc_t*)ompi_list_get_next(ib_proc)) { ib_proc = (mca_btl_ib_proc_t*)ompi_list_get_next(ib_proc)) {
if(ib_proc->proc_guid.vpid == endpoint->vpid) { if(ib_proc->proc_guid.vpid == endpoint->vpid) {
@ -437,7 +437,7 @@ static void mca_bmi_ib_endpoint_recv(
* status of this connection to CONNECTING, * status of this connection to CONNECTING,
* and then reply with our QP information */ * and then reply with our QP information */
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_reply_start_connect(ib_endpoint, buffer))) { if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_reply_start_connect(ib_endpoint, buffer))) {
ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n", ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc); ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
break; break;
@ -449,23 +449,23 @@ static void mca_bmi_ib_endpoint_recv(
case MCA_BMI_IB_CONNECTING : case MCA_BMI_IB_CONNECTING :
mca_bmi_ib_endpoint_set_remote_info(ib_endpoint, buffer); mca_btl_ib_endpoint_set_remote_info(ib_endpoint, buffer);
if(OMPI_SUCCESS != (rc = mca_bmi_ib_endpoint_connect(ib_endpoint))) { if(OMPI_SUCCESS != (rc = mca_btl_ib_endpoint_connect(ib_endpoint))) {
ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n", ompi_output(0, "[%lu,%lu,%lu] %s:%d errcode %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc); ORTE_NAME_ARGS(orte_process_info.my_name), __FILE__,__LINE__,rc);
break; break;
} }
/* Setup state as connected */ /* Setup state as connected */
mca_bmi_ib_endpoint_connected(ib_endpoint); mca_btl_ib_endpoint_connected(ib_endpoint);
/* Send him an ack */ /* Send him an ack */
mca_bmi_ib_endpoint_send_connect_ack(ib_endpoint); mca_btl_ib_endpoint_send_connect_ack(ib_endpoint);
break; break;
case MCA_BMI_IB_CONNECT_ACK: case MCA_BMI_IB_CONNECT_ACK:
mca_bmi_ib_endpoint_connected(ib_endpoint); mca_btl_ib_endpoint_connected(ib_endpoint);
break; break;
@ -481,10 +481,10 @@ static void mca_bmi_ib_endpoint_recv(
/* Okay, now that we are done receiving, /* Okay, now that we are done receiving,
* re-post the buffer */ * re-post the buffer */
mca_bmi_ib_post_recv(); mca_btl_ib_post_recv();
} }
void mca_bmi_ib_post_recv() void mca_btl_ib_post_recv()
{ {
DEBUG_OUT(""); DEBUG_OUT("");
@ -492,7 +492,7 @@ void mca_bmi_ib_post_recv()
ORTE_RML_NAME_ANY, ORTE_RML_NAME_ANY,
ORTE_RML_TAG_DYNAMIC-1, ORTE_RML_TAG_DYNAMIC-1,
0, 0,
mca_bmi_ib_endpoint_recv, mca_btl_ib_endpoint_recv,
NULL); NULL);
} }
@ -502,13 +502,13 @@ void mca_bmi_ib_post_recv()
* connected, queue the fragment and start the connection as required. * connected, queue the fragment and start the connection as required.
*/ */
int mca_bmi_ib_endpoint_send( int mca_btl_ib_endpoint_send(
mca_bmi_base_endpoint_t* endpoint, mca_btl_base_endpoint_t* endpoint,
mca_bmi_ib_frag_t* frag mca_btl_ib_frag_t* frag
) )
{ {
int rc; int rc;
mca_bmi_ib_module_t *ib_bmi; mca_btl_ib_module_t *ib_btl;
OMPI_THREAD_LOCK(&endpoint->endpoint_send_lock); OMPI_THREAD_LOCK(&endpoint->endpoint_send_lock);
@ -540,7 +540,7 @@ int mca_bmi_ib_endpoint_send(
ompi_list_append(&endpoint->pending_send_frags, ompi_list_append(&endpoint->pending_send_frags,
(ompi_list_item_t *)frag); (ompi_list_item_t *)frag);
rc = mca_bmi_ib_endpoint_start_connect(endpoint); rc = mca_btl_ib_endpoint_start_connect(endpoint);
break; break;
@ -551,7 +551,7 @@ int mca_bmi_ib_endpoint_send(
case MCA_BMI_IB_CONNECTED: case MCA_BMI_IB_CONNECTED:
{ {
ib_bmi = endpoint->endpoint_bmi; ib_btl = endpoint->endpoint_btl;
DEBUG_OUT("Send to : %d, len : %d, frag : %p", DEBUG_OUT("Send to : %d, len : %d, frag : %p",
@ -559,7 +559,7 @@ int mca_bmi_ib_endpoint_send(
frag->ib_buf.desc.sg_entry.len, frag->ib_buf.desc.sg_entry.len,
frag); frag);
rc = mca_bmi_ib_endpoint_post_send(ib_bmi, endpoint, frag); rc = mca_btl_ib_endpoint_post_send(ib_btl, endpoint, frag);
break; break;
} }
@ -573,11 +573,11 @@ int mca_bmi_ib_endpoint_send(
return rc; return rc;
} }
void mca_bmi_ib_progress_send_frags(mca_bmi_ib_endpoint_t* endpoint) void mca_btl_ib_progress_send_frags(mca_btl_ib_endpoint_t* endpoint)
{ {
ompi_list_item_t *frag_item; ompi_list_item_t *frag_item;
mca_bmi_ib_frag_t *frag; mca_btl_ib_frag_t *frag;
mca_bmi_ib_module_t* ib_bmi; mca_btl_ib_module_t* ib_btl;
/*Check if endpoint is connected */ /*Check if endpoint is connected */
if(endpoint->endpoint_state != MCA_BMI_IB_CONNECTED) { if(endpoint->endpoint_state != MCA_BMI_IB_CONNECTED) {
@ -589,12 +589,12 @@ void mca_bmi_ib_progress_send_frags(mca_bmi_ib_endpoint_t* endpoint)
while(!ompi_list_is_empty(&(endpoint->pending_send_frags))) { while(!ompi_list_is_empty(&(endpoint->pending_send_frags))) {
frag_item = ompi_list_remove_first(&(endpoint->pending_send_frags)); frag_item = ompi_list_remove_first(&(endpoint->pending_send_frags));
frag = (mca_bmi_ib_frag_t *) frag_item; frag = (mca_btl_ib_frag_t *) frag_item;
ib_bmi = endpoint->endpoint_bmi; ib_btl = endpoint->endpoint_btl;
/* We need to post this one */ /* We need to post this one */
if(OMPI_SUCCESS != mca_bmi_ib_endpoint_post_send(ib_bmi, endpoint, frag)) if(OMPI_SUCCESS != mca_btl_ib_endpoint_post_send(ib_btl, endpoint, frag))
ompi_output(0, "error in mca_bmi_ib_endpoint_send"); ompi_output(0, "error in mca_btl_ib_endpoint_send");
} }
} }
@ -604,39 +604,39 @@ void mca_bmi_ib_progress_send_frags(mca_bmi_ib_endpoint_t* endpoint)
* Complete connection to endpoint. * Complete connection to endpoint.
*/ */
int mca_bmi_ib_endpoint_connect( int mca_btl_ib_endpoint_connect(
mca_bmi_ib_endpoint_t *endpoint) mca_btl_ib_endpoint_t *endpoint)
{ {
int rc; int rc;
/* Connection establishment RC */ /* Connection establishment RC */
rc = mca_bmi_ib_endpoint_qp_init_query(endpoint->endpoint_bmi, rc = mca_btl_ib_endpoint_qp_init_query(endpoint->endpoint_btl,
endpoint->endpoint_bmi->nic, endpoint->endpoint_btl->nic,
endpoint->lcl_qp_hndl_high, endpoint->lcl_qp_hndl_high,
endpoint->rem_qp_num_high, endpoint->rem_qp_num_high,
endpoint->rem_lid, endpoint->rem_lid,
endpoint->endpoint_bmi->port_id); endpoint->endpoint_btl->port_id);
rc = mca_bmi_ib_endpoint_qp_init_query(endpoint->endpoint_bmi, rc = mca_btl_ib_endpoint_qp_init_query(endpoint->endpoint_btl,
endpoint->endpoint_bmi->nic, endpoint->endpoint_btl->nic,
endpoint->lcl_qp_hndl_low, endpoint->lcl_qp_hndl_low,
endpoint->rem_qp_num_low, endpoint->rem_qp_num_low,
endpoint->rem_lid, endpoint->rem_lid,
endpoint->endpoint_bmi->port_id); endpoint->endpoint_btl->port_id);
if(rc != OMPI_SUCCESS) { if(rc != OMPI_SUCCESS) {
return rc; return rc;
} }
mca_bmi_ib_endpoint_post_rr(endpoint, 0); mca_btl_ib_endpoint_post_rr(endpoint, 0);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
int mca_bmi_ib_endpoint_create_qp( int mca_btl_ib_endpoint_create_qp(
mca_bmi_ib_module_t* ib_bmi, mca_btl_ib_module_t* ib_btl,
VAPI_hca_hndl_t nic, VAPI_hca_hndl_t nic,
VAPI_pd_hndl_t ptag, VAPI_pd_hndl_t ptag,
VAPI_cq_hndl_t cq_hndl, VAPI_cq_hndl_t cq_hndl,
@ -651,10 +651,10 @@ int mca_bmi_ib_endpoint_create_qp(
switch(transport_type) { switch(transport_type) {
case VAPI_TS_RC: /* Set up RC qp parameters */ case VAPI_TS_RC: /* Set up RC qp parameters */
qp_init_attr.cap.max_oust_wr_rq = ib_bmi->ib_wq_size; qp_init_attr.cap.max_oust_wr_rq = ib_btl->ib_wq_size;
qp_init_attr.cap.max_oust_wr_sq = ib_bmi->ib_wq_size; qp_init_attr.cap.max_oust_wr_sq = ib_btl->ib_wq_size;
qp_init_attr.cap.max_sg_size_rq = ib_bmi->ib_sg_list_size; qp_init_attr.cap.max_sg_size_rq = ib_btl->ib_sg_list_size;
qp_init_attr.cap.max_sg_size_sq = ib_bmi->ib_sg_list_size; qp_init_attr.cap.max_sg_size_sq = ib_btl->ib_sg_list_size;
qp_init_attr.pd_hndl = ptag; qp_init_attr.pd_hndl = ptag;
/* We don't have Reliable Datagram Handle right now */ /* We don't have Reliable Datagram Handle right now */
qp_init_attr.rdd_hndl = 0; qp_init_attr.rdd_hndl = 0;
@ -688,9 +688,9 @@ int mca_bmi_ib_endpoint_create_qp(
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
int mca_bmi_ib_endpoint_qp_init_query( int mca_btl_ib_endpoint_qp_init_query(
mca_bmi_ib_module_t* ib_bmi, mca_btl_ib_module_t* ib_btl,
VAPI_hca_hndl_t nic, VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl, VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp_num, VAPI_qp_num_t remote_qp_num,
@ -712,7 +712,7 @@ int mca_bmi_ib_endpoint_qp_init_query(
QP_ATTR_MASK_CLR_ALL(qp_attr_mask); QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT; qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = ib_bmi->ib_pkey_ix; qp_attr.pkey_ix = ib_btl->ib_pkey_ix;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = port_id; qp_attr.port = port_id;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
@ -733,21 +733,21 @@ int mca_bmi_ib_endpoint_qp_init_query(
QP_ATTR_MASK_CLR_ALL(qp_attr_mask); QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR; qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.qp_ous_rd_atom = ib_bmi->ib_qp_ous_rd_atom; qp_attr.qp_ous_rd_atom = ib_btl->ib_qp_ous_rd_atom;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.path_mtu = ib_bmi->ib_mtu; qp_attr.path_mtu = ib_btl->ib_mtu;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
qp_attr.rq_psn = ib_bmi->ib_psn; qp_attr.rq_psn = ib_btl->ib_psn;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN);
qp_attr.pkey_ix = ib_bmi->ib_pkey_ix; qp_attr.pkey_ix = ib_btl->ib_pkey_ix;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.min_rnr_timer = ib_bmi->ib_min_rnr_timer; qp_attr.min_rnr_timer = ib_btl->ib_min_rnr_timer;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER);
qp_attr.av.sl = ib_bmi->ib_service_level; qp_attr.av.sl = ib_btl->ib_service_level;
qp_attr.av.grh_flag = FALSE; qp_attr.av.grh_flag = FALSE;
qp_attr.av.static_rate = ib_bmi->ib_static_rate; qp_attr.av.static_rate = ib_btl->ib_static_rate;
qp_attr.av.src_path_bits = ib_bmi->ib_src_path_bits; qp_attr.av.src_path_bits = ib_btl->ib_src_path_bits;
qp_attr.dest_qp_num = remote_qp_num; qp_attr.dest_qp_num = remote_qp_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM);
@ -768,15 +768,15 @@ int mca_bmi_ib_endpoint_qp_init_query(
QP_ATTR_MASK_CLR_ALL(qp_attr_mask); QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS; qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = ib_bmi->ib_psn; qp_attr.sq_psn = ib_btl->ib_psn;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
qp_attr.timeout = ib_bmi->ib_timeout; qp_attr.timeout = ib_btl->ib_timeout;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT);
qp_attr.retry_count = ib_bmi->ib_retry_count; qp_attr.retry_count = ib_btl->ib_retry_count;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = ib_bmi->ib_rnr_retry; qp_attr.rnr_retry = ib_btl->ib_rnr_retry;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY);
qp_attr.ous_dst_rd_atom = ib_bmi->ib_max_rdma_dst_ops; qp_attr.ous_dst_rd_atom = ib_btl->ib_max_rdma_dst_ops;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM); QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM);
ret = VAPI_modify_qp(nic, qp_hndl, ret = VAPI_modify_qp(nic, qp_hndl,
@ -794,7 +794,7 @@ int mca_bmi_ib_endpoint_qp_init_query(
return OMPI_ERROR; return OMPI_ERROR;
} }
ib_bmi->ib_inline_max = qp_init_attr.cap.max_inline_data_sq; ib_btl->ib_inline_max = qp_init_attr.cap.max_inline_data_sq;
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }

78
src/mca/btl/mvapi/bmi_ib_endpoint.h → src/mca/btl/mvapi/btl_ib_endpoint.h
Просмотреть файл

@ -20,14 +20,14 @@
#include "class/ompi_list.h" #include "class/ompi_list.h"
#include "event/event.h" #include "event/event.h"
#include "mca/pml/pml.h" #include "mca/pml/pml.h"
#include "mca/bmi/bmi.h" #include "mca/btl/btl.h"
#include "bmi_ib_frag.h" #include "btl_ib_frag.h"
#include "bmi_ib.h" #include "btl_ib.h"
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)
extern "C" { extern "C" {
#endif #endif
#define MAX_POST_RR (16) #define MAX_POST_RR (16)
OBJ_CLASS_DECLARATION(mca_bmi_ib_endpoint_t); OBJ_CLASS_DECLARATION(mca_btl_ib_endpoint_t);
/** /**
* State of IB endpoint connection. * State of IB endpoint connection.
@ -52,25 +52,25 @@ typedef enum {
/* Maximum number of retries have been used. /* Maximum number of retries have been used.
* Report failure on send to upper layer */ * Report failure on send to upper layer */
MCA_BMI_IB_FAILED MCA_BMI_IB_FAILED
} mca_bmi_ib_endpoint_state_t; } mca_btl_ib_endpoint_state_t;
/** /**
* An abstraction that represents a connection to a endpoint process. * An abstraction that represents a connection to a endpoint process.
* An instance of mca_bmi_base_endpoint_t is associated w/ each process * An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BMI pair at startup. However, connections to the endpoint * and BMI pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis: * are established dynamically on an as-needed basis:
*/ */
struct mca_bmi_base_endpoint_t { struct mca_btl_base_endpoint_t {
ompi_list_item_t super; ompi_list_item_t super;
struct mca_bmi_ib_module_t* endpoint_bmi; struct mca_btl_ib_module_t* endpoint_btl;
/**< BMI instance that created this connection */ /**< BMI instance that created this connection */
struct mca_bmi_ib_proc_t* endpoint_proc; struct mca_btl_ib_proc_t* endpoint_proc;
/**< proc structure corresponding to endpoint */ /**< proc structure corresponding to endpoint */
mca_bmi_ib_endpoint_state_t endpoint_state; mca_btl_ib_endpoint_state_t endpoint_state;
/**< current state of the connection */ /**< current state of the connection */
size_t endpoint_retries; size_t endpoint_retries;
@ -111,18 +111,18 @@ struct mca_bmi_base_endpoint_t {
}; };
typedef struct mca_bmi_base_endpoint_t mca_bmi_base_endpoint_t; typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
typedef mca_bmi_base_endpoint_t mca_bmi_ib_endpoint_t; typedef mca_btl_base_endpoint_t mca_btl_ib_endpoint_t;
int mca_bmi_ib_endpoint_send(mca_bmi_base_endpoint_t* endpoint, struct mca_bmi_ib_frag_t* frag); int mca_btl_ib_endpoint_send(mca_btl_base_endpoint_t* endpoint, struct mca_btl_ib_frag_t* frag);
int mca_bmi_ib_endpoint_connect(mca_bmi_base_endpoint_t*); int mca_btl_ib_endpoint_connect(mca_btl_base_endpoint_t*);
void mca_bmi_ib_post_recv(void); void mca_btl_ib_post_recv(void);
void mca_bmi_ib_progress_send_frags(mca_bmi_ib_endpoint_t*); void mca_btl_ib_progress_send_frags(mca_btl_ib_endpoint_t*);
static inline int mca_bmi_ib_endpoint_post_rr_sub(int cnt, static inline int mca_btl_ib_endpoint_post_rr_sub(int cnt,
mca_bmi_ib_endpoint_t* endpoint, mca_btl_ib_endpoint_t* endpoint,
ompi_free_list_t* frag_list, ompi_free_list_t* frag_list,
uint32_t* rr_posted, uint32_t* rr_posted,
VAPI_hca_hndl_t nic, VAPI_hca_hndl_t nic,
@ -132,16 +132,16 @@ static inline int mca_bmi_ib_endpoint_post_rr_sub(int cnt,
int rc, i; int rc, i;
ompi_list_item_t* item; ompi_list_item_t* item;
mca_bmi_ib_frag_t* frag; mca_btl_ib_frag_t* frag;
mca_bmi_ib_module_t *ib_bmi = endpoint->endpoint_bmi; mca_btl_ib_module_t *ib_btl = endpoint->endpoint_btl;
VAPI_rr_desc_t* rr_desc_post = ib_bmi->rr_desc_post; VAPI_rr_desc_t* rr_desc_post = ib_btl->rr_desc_post;
/* prepare frags and post receive requests */ /* prepare frags and post receive requests */
for(i = 0; i < cnt; i++) { for(i = 0; i < cnt; i++) {
OMPI_FREE_LIST_WAIT(frag_list, item, rc); OMPI_FREE_LIST_WAIT(frag_list, item, rc);
frag = (mca_bmi_ib_frag_t*) item; frag = (mca_btl_ib_frag_t*) item;
frag->endpoint = endpoint; frag->endpoint = endpoint;
frag->sg_entry.len = frag->size + ((unsigned char*) frag->segment.seg_addr.pval- (unsigned char*) frag->hdr); /* sizeof(mca_bmi_ib_header_t); */ frag->sg_entry.len = frag->size + ((unsigned char*) frag->segment.seg_addr.pval- (unsigned char*) frag->hdr); /* sizeof(mca_btl_ib_header_t); */
rr_desc_post[i] = frag->rr_desc; rr_desc_post[i] = frag->rr_desc;
} }
@ -158,41 +158,41 @@ static inline int mca_bmi_ib_endpoint_post_rr_sub(int cnt,
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
static inline int mca_bmi_ib_endpoint_post_rr( mca_bmi_ib_endpoint_t * endpoint, int additional){ static inline int mca_btl_ib_endpoint_post_rr( mca_btl_ib_endpoint_t * endpoint, int additional){
mca_bmi_ib_module_t * ib_bmi = endpoint->endpoint_bmi; mca_btl_ib_module_t * ib_btl = endpoint->endpoint_btl;
int rc; int rc;
OMPI_THREAD_LOCK(&ib_bmi->ib_lock); OMPI_THREAD_LOCK(&ib_btl->ib_lock);
if(ib_bmi->rr_posted_high <= mca_bmi_ib_component.ib_rr_buf_min+additional && ib_bmi->rr_posted_high < mca_bmi_ib_component.ib_rr_buf_max){ if(ib_btl->rr_posted_high <= mca_btl_ib_component.ib_rr_buf_min+additional && ib_btl->rr_posted_high < mca_btl_ib_component.ib_rr_buf_max){
rc = mca_bmi_ib_endpoint_post_rr_sub(mca_bmi_ib_component.ib_rr_buf_max - ib_bmi->rr_posted_high, rc = mca_btl_ib_endpoint_post_rr_sub(mca_btl_ib_component.ib_rr_buf_max - ib_btl->rr_posted_high,
endpoint, endpoint,
&ib_bmi->recv_free_eager, &ib_btl->recv_free_eager,
&ib_bmi->rr_posted_high, &ib_btl->rr_posted_high,
ib_bmi->nic, ib_btl->nic,
endpoint->lcl_qp_hndl_high endpoint->lcl_qp_hndl_high
); );
if(rc != OMPI_SUCCESS){ if(rc != OMPI_SUCCESS){
OMPI_THREAD_UNLOCK(&ib_bmi->ib_lock); OMPI_THREAD_UNLOCK(&ib_btl->ib_lock);
return rc; return rc;
} }
} }
if(ib_bmi->rr_posted_low <= mca_bmi_ib_component.ib_rr_buf_min+additional && ib_bmi->rr_posted_low < mca_bmi_ib_component.ib_rr_buf_max){ if(ib_btl->rr_posted_low <= mca_btl_ib_component.ib_rr_buf_min+additional && ib_btl->rr_posted_low < mca_btl_ib_component.ib_rr_buf_max){
rc = mca_bmi_ib_endpoint_post_rr_sub(mca_bmi_ib_component.ib_rr_buf_max - ib_bmi->rr_posted_low, rc = mca_btl_ib_endpoint_post_rr_sub(mca_btl_ib_component.ib_rr_buf_max - ib_btl->rr_posted_low,
endpoint, endpoint,
&ib_bmi->recv_free_max, &ib_btl->recv_free_max,
&ib_bmi->rr_posted_low, &ib_btl->rr_posted_low,
ib_bmi->nic, ib_btl->nic,
endpoint->lcl_qp_hndl_low endpoint->lcl_qp_hndl_low
); );
if(rc != OMPI_SUCCESS) { if(rc != OMPI_SUCCESS) {
OMPI_THREAD_UNLOCK(&ib_bmi->ib_lock); OMPI_THREAD_UNLOCK(&ib_btl->ib_lock);
return rc; return rc;
} }
} }
OMPI_THREAD_UNLOCK(&ib_bmi->ib_lock); OMPI_THREAD_UNLOCK(&ib_btl->ib_lock);
return OMPI_SUCCESS; return OMPI_SUCCESS;

0
src/mca/btl/mvapi/bmi_ib_error.h → src/mca/btl/mvapi/btl_ib_error.h
Просмотреть файл

152
src/mca/btl/mvapi/btl_ib_frag.c Обычный файл
Просмотреть файл

@ -0,0 +1,152 @@
#include "btl_ib_frag.h"
#include "mca/common/vapi/vapi_mem_reg.h"
#include "mca/mpool/vapi/mpool_vapi.h"
static void mca_btl_ib_frag_common_constructor( mca_btl_ib_frag_t* frag)
{
mca_mpool_vapi_registration_t* mem_hndl = (mca_mpool_vapi_registration_t*) frag->base.super.user_data;
frag->hdr = (mca_btl_ib_header_t*) (frag+1); /* initialize the btl header to point to start at end of frag */
#if 0
mod = (unsigned long) frag->hdr % MCA_BMI_IB_FRAG_ALIGN;
if(mod != 0) {
frag->hdr = (mca_btl_ib_header_t*) ((unsigned char*) frag->hdr + (MCA_BMI_IB_FRAG_ALIGN - mod));
}
#endif
frag->segment.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_btl_ib_header_t); /* init the segment address to start after the btl header */
#if 0
mod = (frag->segment.seg_addr.lval) % MCA_BMI_IB_FRAG_ALIGN;
if(mod != 0) {
frag->segment.seg_addr.lval += (MCA_BMI_IB_FRAG_ALIGN - mod);
}
#endif
frag->mem_hndl = mem_hndl->hndl;
frag->segment.seg_len = frag->size;
frag->segment.seg_key.key32[0] = (uint32_t) mem_hndl->l_key;
frag->sg_entry.lkey = mem_hndl->l_key;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
frag->base.des_flags = 0;
}
static void mca_btl_ib_send_frag_common_constructor(mca_btl_ib_frag_t* frag)
{
mca_btl_ib_frag_common_constructor(frag);
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->sr_desc.comp_type = VAPI_SIGNALED;
frag->sr_desc.opcode = VAPI_SEND;
frag->sr_desc.remote_qkey = 0;
frag->sr_desc.sg_lst_len = 1;
frag->sr_desc.sg_lst_p = &frag->sg_entry;
frag->sr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
}
static void mca_btl_ib_recv_frag_common_constructor(mca_btl_ib_frag_t* frag)
{
mca_btl_ib_frag_common_constructor(frag);
frag->base.des_dst = &frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->rr_desc.comp_type = VAPI_SIGNALED;
frag->rr_desc.opcode = VAPI_RECEIVE;
frag->rr_desc.sg_lst_len = 1;
frag->rr_desc.sg_lst_p = &frag->sg_entry;
frag->rr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
}
static void mca_btl_ib_send_frag_eager_constructor(mca_btl_ib_frag_t* frag)
{
frag->size = mca_btl_ib_component.eager_limit;
mca_btl_ib_send_frag_common_constructor(frag);
}
static void mca_btl_ib_send_frag_max_constructor(mca_btl_ib_frag_t* frag)
{
frag->size = mca_btl_ib_component.max_send_size;
mca_btl_ib_send_frag_common_constructor(frag);
}
static void mca_btl_ib_recv_frag_max_constructor(mca_btl_ib_frag_t* frag)
{
frag->size = mca_btl_ib_component.max_send_size;
mca_btl_ib_recv_frag_common_constructor(frag);
}
static void mca_btl_ib_recv_frag_eager_constructor(mca_btl_ib_frag_t* frag)
{
frag->size = mca_btl_ib_component.eager_limit;
mca_btl_ib_recv_frag_common_constructor(frag);
}
static void mca_btl_ib_send_frag_frag_constructor(mca_btl_ib_frag_t* frag)
{
frag->size = 0;
mca_btl_ib_send_frag_common_constructor(frag);
}
OBJ_CLASS_INSTANCE(
mca_btl_ib_frag_t,
mca_btl_base_descriptor_t,
NULL,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_ib_send_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_ib_send_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_ib_send_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_ib_send_frag_max_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_ib_send_frag_frag_t,
mca_btl_base_descriptor_t,
mca_btl_ib_send_frag_frag_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_ib_recv_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_ib_recv_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_ib_recv_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_ib_recv_frag_max_constructor,
NULL);

76
src/mca/btl/mvapi/bmi_ib_frag.h → src/mca/btl/mvapi/btl_ib_frag.h
Просмотреть файл

@ -20,7 +20,7 @@
#define MCA_BMI_IB_FRAG_ALIGN (8) #define MCA_BMI_IB_FRAG_ALIGN (8)
#include "ompi_config.h" #include "ompi_config.h"
#include "bmi_ib.h" #include "btl_ib.h"
#include <vapi.h> #include <vapi.h>
#include <mtl_common.h> #include <mtl_common.h>
@ -30,9 +30,9 @@
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)
extern "C" { extern "C" {
#endif #endif
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_bmi_ib_frag_t); OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_ib_frag_t);
typedef mca_bmi_base_header_t mca_bmi_ib_header_t; typedef mca_btl_base_header_t mca_btl_ib_header_t;
typedef enum { typedef enum {
@ -40,17 +40,17 @@ typedef enum {
MCA_BMI_IB_FRAG_PUT, MCA_BMI_IB_FRAG_PUT,
MCA_BMI_IB_FRAG_GET, MCA_BMI_IB_FRAG_GET,
MCA_BMI_IB_FRAG_ACK MCA_BMI_IB_FRAG_ACK
} mca_bmi_ib_frag_type_t; } mca_btl_ib_frag_type_t;
/** /**
* IB send fragment derived type. * IB send fragment derived type.
*/ */
struct mca_bmi_ib_frag_t { struct mca_btl_ib_frag_t {
mca_bmi_base_descriptor_t base; mca_btl_base_descriptor_t base;
mca_bmi_base_segment_t segment; mca_btl_base_segment_t segment;
struct mca_bmi_base_endpoint_t *endpoint; struct mca_btl_base_endpoint_t *endpoint;
mca_bmi_ib_frag_type_t type; mca_btl_ib_frag_type_t type;
size_t size; size_t size;
int rc; int rc;
@ -61,32 +61,32 @@ struct mca_bmi_ib_frag_t {
VAPI_sg_lst_entry_t sg_entry; VAPI_sg_lst_entry_t sg_entry;
VAPI_mr_hndl_t mem_hndl; VAPI_mr_hndl_t mem_hndl;
VAPI_ret_t ret; VAPI_ret_t ret;
mca_bmi_ib_header_t *hdr; mca_btl_ib_header_t *hdr;
mca_mpool_vapi_registration_t * vapi_reg; mca_mpool_vapi_registration_t * vapi_reg;
}; };
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_frag_t; typedef struct mca_btl_ib_frag_t mca_btl_ib_frag_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_frag_t); OBJ_CLASS_DECLARATION(mca_btl_ib_frag_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_send_frag_eager_t; typedef struct mca_btl_ib_frag_t mca_btl_ib_send_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_send_frag_eager_t); OBJ_CLASS_DECLARATION(mca_btl_ib_send_frag_eager_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_send_frag_max_t; typedef struct mca_btl_ib_frag_t mca_btl_ib_send_frag_max_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_send_frag_max_t); OBJ_CLASS_DECLARATION(mca_btl_ib_send_frag_max_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_send_frag_frag_t; typedef struct mca_btl_ib_frag_t mca_btl_ib_send_frag_frag_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_send_frag_frag_t); OBJ_CLASS_DECLARATION(mca_btl_ib_send_frag_frag_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_recv_frag_eager_t; typedef struct mca_btl_ib_frag_t mca_btl_ib_recv_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_recv_frag_eager_t); OBJ_CLASS_DECLARATION(mca_btl_ib_recv_frag_eager_t);
typedef struct mca_bmi_ib_frag_t mca_bmi_ib_recv_frag_max_t; typedef struct mca_btl_ib_frag_t mca_btl_ib_recv_frag_max_t;
OBJ_CLASS_DECLARATION(mca_bmi_ib_recv_frag_max_t); OBJ_CLASS_DECLARATION(mca_btl_ib_recv_frag_max_t);
@ -96,52 +96,52 @@ OBJ_CLASS_DECLARATION(mca_bmi_ib_recv_frag_max_t);
* *
*/ */
#define MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc) \ #define MCA_BMI_IB_FRAG_ALLOC_EAGER(btl, frag, rc) \
{ \ { \
\ \
ompi_list_item_t *item; \ ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_bmi_ib_module_t*)bmi)->send_free_eager, item, rc); \ OMPI_FREE_LIST_WAIT(&((mca_btl_ib_module_t*)btl)->send_free_eager, item, rc); \
frag = (mca_bmi_ib_frag_t*) item; \ frag = (mca_btl_ib_frag_t*) item; \
} }
#define MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag) \ #define MCA_BMI_IB_FRAG_RETURN_EAGER(btl, frag) \
{ \ { \
OMPI_FREE_LIST_RETURN(&((mca_bmi_ib_module_t*)bmi)->send_free_eager, (ompi_list_item_t*)(frag)); \ OMPI_FREE_LIST_RETURN(&((mca_btl_ib_module_t*)btl)->send_free_eager, (ompi_list_item_t*)(frag)); \
} }
#define MCA_BMI_IB_FRAG_ALLOC_MAX(bmi, frag, rc) \ #define MCA_BMI_IB_FRAG_ALLOC_MAX(btl, frag, rc) \
{ \ { \
\ \
ompi_list_item_t *item; \ ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_bmi_ib_module_t*)bmi)->send_free_max, item, rc); \ OMPI_FREE_LIST_WAIT(&((mca_btl_ib_module_t*)btl)->send_free_max, item, rc); \
frag = (mca_bmi_ib_frag_t*) item; \ frag = (mca_btl_ib_frag_t*) item; \
} }
#define MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag) \ #define MCA_BMI_IB_FRAG_RETURN_MAX(btl, frag) \
{ \ { \
OMPI_FREE_LIST_RETURN(&((mca_bmi_ib_module_t*)bmi)->send_free_max, (ompi_list_item_t*)(frag)); \ OMPI_FREE_LIST_RETURN(&((mca_btl_ib_module_t*)btl)->send_free_max, (ompi_list_item_t*)(frag)); \
} }
#define MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc) \ #define MCA_BMI_IB_FRAG_ALLOC_FRAG(btl, frag, rc) \
{ \ { \
\ \
ompi_list_item_t *item; \ ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_bmi_ib_module_t*)bmi)->send_free_frag, item, rc); \ OMPI_FREE_LIST_WAIT(&((mca_btl_ib_module_t*)btl)->send_free_frag, item, rc); \
frag = (mca_bmi_ib_frag_t*) item; \ frag = (mca_btl_ib_frag_t*) item; \
} }
#define MCA_BMI_IB_FRAG_RETURN_FRAG(bmi, frag) \ #define MCA_BMI_IB_FRAG_RETURN_FRAG(btl, frag) \
{ \ { \
OMPI_FREE_LIST_RETURN(&((mca_bmi_ib_module_t*)bmi)->send_free_frag, (ompi_list_item_t*)(frag)); \ OMPI_FREE_LIST_RETURN(&((mca_btl_ib_module_t*)btl)->send_free_frag, (ompi_list_item_t*)(frag)); \
} }
struct mca_bmi_ib_module_t; struct mca_btl_ib_module_t;
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)
} }

74
src/mca/btl/mvapi/bmi_ib_proc.c → src/mca/btl/mvapi/btl_ib_proc.c
Просмотреть файл

@ -19,17 +19,17 @@
#include "class/ompi_hash_table.h" #include "class/ompi_hash_table.h"
#include "mca/base/mca_base_module_exchange.h" #include "mca/base/mca_base_module_exchange.h"
#include "bmi_ib.h" #include "btl_ib.h"
#include "bmi_ib_proc.h" #include "btl_ib_proc.h"
static void mca_bmi_ib_proc_construct(mca_bmi_ib_proc_t* proc); static void mca_btl_ib_proc_construct(mca_btl_ib_proc_t* proc);
static void mca_bmi_ib_proc_destruct(mca_bmi_ib_proc_t* proc); static void mca_btl_ib_proc_destruct(mca_btl_ib_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_bmi_ib_proc_t, OBJ_CLASS_INSTANCE(mca_btl_ib_proc_t,
ompi_list_item_t, mca_bmi_ib_proc_construct, ompi_list_item_t, mca_btl_ib_proc_construct,
mca_bmi_ib_proc_destruct); mca_btl_ib_proc_destruct);
void mca_bmi_ib_proc_construct(mca_bmi_ib_proc_t* proc) void mca_btl_ib_proc_construct(mca_btl_ib_proc_t* proc)
{ {
proc->proc_ompi = 0; proc->proc_ompi = 0;
proc->proc_addr_count = 0; proc->proc_addr_count = 0;
@ -37,21 +37,21 @@ void mca_bmi_ib_proc_construct(mca_bmi_ib_proc_t* proc)
proc->proc_endpoint_count = 0; proc->proc_endpoint_count = 0;
OBJ_CONSTRUCT(&proc->proc_lock, ompi_mutex_t); OBJ_CONSTRUCT(&proc->proc_lock, ompi_mutex_t);
/* add to list of all proc instance */ /* add to list of all proc instance */
OMPI_THREAD_LOCK(&mca_bmi_ib_component.ib_lock); OMPI_THREAD_LOCK(&mca_btl_ib_component.ib_lock);
ompi_list_append(&mca_bmi_ib_component.ib_procs, &proc->super); ompi_list_append(&mca_btl_ib_component.ib_procs, &proc->super);
OMPI_THREAD_UNLOCK(&mca_bmi_ib_component.ib_lock); OMPI_THREAD_UNLOCK(&mca_btl_ib_component.ib_lock);
} }
/* /*
* Cleanup ib proc instance * Cleanup ib proc instance
*/ */
void mca_bmi_ib_proc_destruct(mca_bmi_ib_proc_t* proc) void mca_btl_ib_proc_destruct(mca_btl_ib_proc_t* proc)
{ {
/* remove from list of all proc instances */ /* remove from list of all proc instances */
OMPI_THREAD_LOCK(&mca_bmi_ib_component.ib_lock); OMPI_THREAD_LOCK(&mca_btl_ib_component.ib_lock);
ompi_list_remove_item(&mca_bmi_ib_component.ib_procs, &proc->super); ompi_list_remove_item(&mca_btl_ib_component.ib_procs, &proc->super);
OMPI_THREAD_UNLOCK(&mca_bmi_ib_component.ib_lock); OMPI_THREAD_UNLOCK(&mca_btl_ib_component.ib_lock);
/* release resources */ /* release resources */
if(NULL != proc->proc_endpoints) { if(NULL != proc->proc_endpoints) {
@ -64,45 +64,45 @@ void mca_bmi_ib_proc_destruct(mca_bmi_ib_proc_t* proc)
* Look for an existing IB process instances based on the associated * Look for an existing IB process instances based on the associated
* ompi_proc_t instance. * ompi_proc_t instance.
*/ */
static mca_bmi_ib_proc_t* mca_bmi_ib_proc_lookup_ompi(ompi_proc_t* ompi_proc) static mca_btl_ib_proc_t* mca_btl_ib_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{ {
mca_bmi_ib_proc_t* ib_proc; mca_btl_ib_proc_t* ib_proc;
OMPI_THREAD_LOCK(&mca_bmi_ib_component.ib_lock); OMPI_THREAD_LOCK(&mca_btl_ib_component.ib_lock);
for(ib_proc = (mca_bmi_ib_proc_t*) for(ib_proc = (mca_btl_ib_proc_t*)
ompi_list_get_first(&mca_bmi_ib_component.ib_procs); ompi_list_get_first(&mca_btl_ib_component.ib_procs);
ib_proc != (mca_bmi_ib_proc_t*) ib_proc != (mca_btl_ib_proc_t*)
ompi_list_get_end(&mca_bmi_ib_component.ib_procs); ompi_list_get_end(&mca_btl_ib_component.ib_procs);
ib_proc = (mca_bmi_ib_proc_t*)ompi_list_get_next(ib_proc)) { ib_proc = (mca_btl_ib_proc_t*)ompi_list_get_next(ib_proc)) {
if(ib_proc->proc_ompi == ompi_proc) { if(ib_proc->proc_ompi == ompi_proc) {
OMPI_THREAD_UNLOCK(&mca_bmi_ib_component.ib_lock); OMPI_THREAD_UNLOCK(&mca_btl_ib_component.ib_lock);
return ib_proc; return ib_proc;
} }
} }
OMPI_THREAD_UNLOCK(&mca_bmi_ib_component.ib_lock); OMPI_THREAD_UNLOCK(&mca_btl_ib_component.ib_lock);
return NULL; return NULL;
} }
/* /*
* Create a IB process structure. There is a one-to-one correspondence * Create a IB process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_bmi_ib_proc_t instance. We cache * between a ompi_proc_t and a mca_btl_ib_proc_t instance. We cache
* additional data (specifically the list of mca_bmi_ib_endpoint_t instances, * additional data (specifically the list of mca_btl_ib_endpoint_t instances,
* and published addresses) associated w/ a given destination on this * and published addresses) associated w/ a given destination on this
* datastructure. * datastructure.
*/ */
mca_bmi_ib_proc_t* mca_bmi_ib_proc_create(ompi_proc_t* ompi_proc) mca_btl_ib_proc_t* mca_btl_ib_proc_create(ompi_proc_t* ompi_proc)
{ {
mca_bmi_ib_proc_t* module_proc = NULL; mca_btl_ib_proc_t* module_proc = NULL;
/* Check if we have already created a IB proc /* Check if we have already created a IB proc
* structure for this ompi process */ * structure for this ompi process */
module_proc = mca_bmi_ib_proc_lookup_ompi(ompi_proc); module_proc = mca_btl_ib_proc_lookup_ompi(ompi_proc);
if(module_proc != NULL) { if(module_proc != NULL) {
@ -113,7 +113,7 @@ mca_bmi_ib_proc_t* mca_bmi_ib_proc_create(ompi_proc_t* ompi_proc)
/* Oops! First time, gotta create a new IB proc /* Oops! First time, gotta create a new IB proc
* out of the ompi_proc ... */ * out of the ompi_proc ... */
module_proc = OBJ_NEW(mca_bmi_ib_proc_t); module_proc = OBJ_NEW(mca_btl_ib_proc_t);
/* Initialize number of peer */ /* Initialize number of peer */
module_proc->proc_endpoint_count = 0; module_proc->proc_endpoint_count = 0;
@ -131,11 +131,11 @@ mca_bmi_ib_proc_t* mca_bmi_ib_proc_create(ompi_proc_t* ompi_proc)
/* XXX: Right now, there can be only 1 peer associated /* XXX: Right now, there can be only 1 peer associated
* with a proc. Needs a little bit change in * with a proc. Needs a little bit change in
* mca_bmi_ib_proc_t to allow on demand increasing of * mca_btl_ib_proc_t to allow on demand increasing of
* number of endpoints for this proc */ * number of endpoints for this proc */
module_proc->proc_endpoints = (mca_bmi_base_endpoint_t**) module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc(module_proc->proc_addr_count * sizeof(mca_bmi_base_endpoint_t*)); malloc(module_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
if(NULL == module_proc->proc_endpoints) { if(NULL == module_proc->proc_endpoints) {
OBJ_RELEASE(module_proc); OBJ_RELEASE(module_proc);
@ -147,11 +147,11 @@ mca_bmi_ib_proc_t* mca_bmi_ib_proc_create(ompi_proc_t* ompi_proc)
/* /*
* Note that this routine must be called with the lock on the process * Note that this routine must be called with the lock on the process
* already held. Insert a bmi instance into the proc array and assign * already held. Insert a btl instance into the proc array and assign
* it an address. * it an address.
*/ */
int mca_bmi_ib_proc_insert(mca_bmi_ib_proc_t* module_proc, int mca_btl_ib_proc_insert(mca_btl_ib_proc_t* module_proc,
mca_bmi_base_endpoint_t* module_endpoint) mca_btl_base_endpoint_t* module_endpoint)
{ {
/* insert into endpoint array */ /* insert into endpoint array */
module_endpoint->endpoint_proc = module_proc; module_endpoint->endpoint_proc = module_proc;

18
src/mca/btl/mvapi/bmi_ib_proc.h → src/mca/btl/mvapi/btl_ib_proc.h
Просмотреть файл

@ -20,21 +20,21 @@
#include "mca/ns/ns.h" #include "mca/ns/ns.h"
#include "class/ompi_object.h" #include "class/ompi_object.h"
#include "proc/proc.h" #include "proc/proc.h"
#include "bmi_ib.h" #include "btl_ib.h"
#include "bmi_ib_endpoint.h" #include "btl_ib_endpoint.h"
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)
extern "C" { extern "C" {
#endif #endif
OBJ_CLASS_DECLARATION(mca_bmi_ib_proc_t); OBJ_CLASS_DECLARATION(mca_btl_ib_proc_t);
/** /**
* Represents the state of a remote process and the set of addresses * Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_bmi_base_endpoint_t for * that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each * each
* BMI instance that attempts to open a connection to the process. * BMI instance that attempts to open a connection to the process.
*/ */
struct mca_bmi_ib_proc_t { struct mca_btl_ib_proc_t {
ompi_list_item_t super; ompi_list_item_t super;
/**< allow proc to be placed on a list */ /**< allow proc to be placed on a list */
@ -47,7 +47,7 @@ struct mca_bmi_ib_proc_t {
size_t proc_addr_count; size_t proc_addr_count;
/**< number of addresses published by endpoint */ /**< number of addresses published by endpoint */
struct mca_bmi_base_endpoint_t **proc_endpoints; struct mca_btl_base_endpoint_t **proc_endpoints;
/**< array of endpoints that have been created to access this proc */ /**< array of endpoints that have been created to access this proc */
size_t proc_endpoint_count; size_t proc_endpoint_count;
@ -56,10 +56,10 @@ struct mca_bmi_ib_proc_t {
ompi_mutex_t proc_lock; ompi_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */ /**< lock to protect against concurrent access to proc state */
}; };
typedef struct mca_bmi_ib_proc_t mca_bmi_ib_proc_t; typedef struct mca_btl_ib_proc_t mca_btl_ib_proc_t;
mca_bmi_ib_proc_t* mca_bmi_ib_proc_create(ompi_proc_t* ompi_proc); mca_btl_ib_proc_t* mca_btl_ib_proc_create(ompi_proc_t* ompi_proc);
int mca_bmi_ib_proc_insert(mca_bmi_ib_proc_t*, mca_bmi_base_endpoint_t*); int mca_btl_ib_proc_insert(mca_btl_ib_proc_t*, mca_btl_base_endpoint_t*);
#if defined(c_plusplus) || defined(__cplusplus) #if defined(c_plusplus) || defined(__cplusplus)
} }

2
src/mca/btl/mvapi/configure.params
Просмотреть файл

@ -17,6 +17,6 @@
# Specific to this module # Specific to this module
PARAM_INIT_FILE=bmi_ib.c PARAM_INIT_FILE=btl_ib.c
PARAM_CONFIG_HEADER_FILE="ib_config.h" PARAM_CONFIG_HEADER_FILE="ib_config.h"
PARAM_CONFIG_FILES="Makefile" PARAM_CONFIG_FILES="Makefile"

34
src/mca/btl/mvapi/configure.stub
Просмотреть файл

@ -23,29 +23,29 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
# Additional --with flags that can be specified # Additional --with flags that can be specified
AC_ARG_WITH(bmi-ib, AC_ARG_WITH(btl-ib,
AC_HELP_STRING([--with-bmi-ib=IBDIR], AC_HELP_STRING([--with-btl-ib=IBDIR],
[Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)])) [Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)]))
AC_ARG_WITH(bmi-ib-libdir, AC_ARG_WITH(btl-ib-libdir,
AC_HELP_STRING([--with-bmi-ib-libdir=IBLIBDIR], AC_HELP_STRING([--with-btl-ib-libdir=IBLIBDIR],
[directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64])) [directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64]))
# Quick sanity check # Quick sanity check
if test "$with_bmi_ib" = "no"; then if test "$with_btl_ib" = "no"; then
AC_MSG_WARN([*** --without-bmi-ib specified -- aborting]) AC_MSG_WARN([*** --without-btl-ib specified -- aborting])
AC_MSG_ERROR([Will not continue]) AC_MSG_ERROR([Will not continue])
fi fi
# Find the right IBDIR # Find the right IBDIR
if test "$with_bmi_ib" != "" -a "$with_bmi_ib" != "yes" ; then if test "$with_btl_ib" != "" -a "$with_btl_ib" != "yes" ; then
IBDIR="$with_bmi_ib" IBDIR="$with_btl_ib"
IBLIBDIR="$with_bmi_ib" IBLIBDIR="$with_btl_ib"
fi fi
if test "$with_bmi_ib_libdir" != "" -a "$with_bmi_ib_libdir" != "yes" -a \ if test "$with_btl_ib_libdir" != "" -a "$with_btl_ib_libdir" != "yes" -a \
"$with_bmi_ib_libdir" != "no"; then "$with_btl_ib_libdir" != "no"; then
IBLIBDIR="$with_bmi_ib_libdir" IBLIBDIR="$with_btl_ib_libdir"
fi fi
# Add to CPPFLAGS if necessary # Add to CPPFLAGS if necessary
@ -56,13 +56,13 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
EXTRA_CPPFLAGS="-I$IBDIR/include" EXTRA_CPPFLAGS="-I$IBDIR/include"
else else
AC_MSG_WARN([*** Warning: cannot find $IBDIR/include]) AC_MSG_WARN([*** Warning: cannot find $IBDIR/include])
AC_MSG_WARN([*** Will still try to configure ib bmi anyway...]) AC_MSG_WARN([*** Will still try to configure ib btl anyway...])
fi fi
if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then
EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS" EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS"
else else
AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap]) AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap])
AC_MSG_WARN([*** Will still try to configure ib bmi anyway...]) AC_MSG_WARN([*** Will still try to configure ib btl anyway...])
fi fi
fi fi
@ -90,7 +90,7 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`" CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
mca_bmi_ib_try_find_libvapi() { mca_btl_ib_try_find_libvapi() {
func1=[$]1 func1=[$]1
func2=[$]2 func2=[$]2
@ -121,7 +121,7 @@ mca_bmi_ib_try_find_libvapi() {
if test -d "$IBLIBDIR/lib"; then if test -d "$IBLIBDIR/lib"; then
EXTRA_LDFLAGS="-L$IBLIBDIR/lib" EXTRA_LDFLAGS="-L$IBLIBDIR/lib"
LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
mca_bmi_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap mca_btl_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap
if test "$LIBS" != "$LIBS_orig"; then if test "$LIBS" != "$LIBS_orig"; then
echo "--> found libvapi in $IBLIBDIR/lib" echo "--> found libvapi in $IBLIBDIR/lib"
fi fi
@ -130,7 +130,7 @@ mca_bmi_ib_try_find_libvapi() {
if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then
EXTRA_LDFLAGS="-L$IBLIBDIR/lib64" EXTRA_LDFLAGS="-L$IBLIBDIR/lib64"
LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS" LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS"
mca_bmi_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca mca_btl_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca
if test "$LIBS" != "$LIBS_orig"; then if test "$LIBS" != "$LIBS_orig"; then
echo "--> found libvapi in $IBLIBDIR/lib64" echo "--> found libvapi in $IBLIBDIR/lib64"
fi fi