1
1

Added with-bmi-ib instead of with-ptl-ib

Begun adding support for RDMA_WRITE_WITH_IMM

This commit was SVN r6081.
Этот коммит содержится в:
Galen Shipman 2005-06-17 19:15:03 +00:00
родитель 2bc9d9c657
Коммит 507730a2ca
8 изменённых файлов: 62 добавлений и 42 удалений

Просмотреть файл

@ -172,6 +172,7 @@ struct mca_bmi_base_descriptor_t {
mca_bmi_base_completion_fn_t des_cbfunc; mca_bmi_base_completion_fn_t des_cbfunc;
void* des_cbdata; void* des_cbdata;
int32_t des_flags; int32_t des_flags;
ompi_ptr_t user_data;
}; };
typedef struct mca_bmi_base_descriptor_t mca_bmi_base_descriptor_t; typedef struct mca_bmi_base_descriptor_t mca_bmi_base_descriptor_t;

Просмотреть файл

@ -252,7 +252,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
frag->base.des_dst = NULL; frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0; frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0; frag->base.des_flags = 0;
frag->base.user_data.pval = NULL;
return &frag->base; return &frag->base;
}else if( max_data + reserve <= ib_bmi->super.bmi_max_send_size || 1 == ompi_convertor_need_buffers( convertor) ){ }else if( max_data + reserve <= ib_bmi->super.bmi_max_send_size || 1 == ompi_convertor_need_buffers( convertor) ){
@ -281,7 +281,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
frag->base.des_dst = NULL; frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0; frag->base.des_dst_cnt = 0;
frag->base.des_flags=0; frag->base.des_flags=0;
frag->base.user_data.pval = NULL;
return &frag->base; return &frag->base;
} else { } else {
@ -363,7 +363,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
frag->base.des_dst = NULL; frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0; frag->base.des_dst_cnt = 0;
frag->base.des_flags=0; frag->base.des_flags=0;
frag->base.user_data.pval = NULL;
return &frag->base; return &frag->base;
} }
@ -537,7 +537,11 @@ int mca_bmi_ib_put( mca_bmi_base_module_t* bmi,
mca_bmi_ib_module_t* ib_bmi = (mca_bmi_ib_module_t*) bmi; mca_bmi_ib_module_t* ib_bmi = (mca_bmi_ib_module_t*) bmi;
mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*) descriptor; mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*) descriptor;
frag->endpoint = endpoint; frag->endpoint = endpoint;
if(NULL == frag->base.user_data.pval)
frag->sr_desc.opcode = VAPI_RDMA_WRITE; frag->sr_desc.opcode = VAPI_RDMA_WRITE;
else
frag->sr_desc.opcode = VAPI_RDMA_WRITE_WITH_IMM;
frag->sr_desc.remote_qp = endpoint->rem_qp_num_low; frag->sr_desc.remote_qp = endpoint->rem_qp_num_low;
frag->sr_desc.remote_addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->base.des_dst->seg_addr.pval; frag->sr_desc.remote_addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->base.des_dst->seg_addr.pval;
frag->sr_desc.r_key = frag->base.des_dst->seg_key.key32[0]; frag->sr_desc.r_key = frag->base.des_dst->seg_key.key32[0];

Просмотреть файл

@ -479,6 +479,7 @@ int mca_bmi_ib_component_progress()
VAPI_wc_desc_t comp; VAPI_wc_desc_t comp;
mca_bmi_ib_module_t* ib_bmi = &mca_bmi_ib_component.ib_bmis[i]; mca_bmi_ib_module_t* ib_bmi = &mca_bmi_ib_component.ib_bmis[i];
do{
ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_high, &comp); ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_high, &comp);
if(VAPI_OK == ret) { if(VAPI_OK == ret) {
if(comp.status != VAPI_SUCCESS) { if(comp.status != VAPI_SUCCESS) {
@ -490,6 +491,12 @@ int mca_bmi_ib_component_progress()
/* Handle n/w completions */ /* Handle n/w completions */
switch(comp.opcode) { switch(comp.opcode) {
case VAPI_CQE_RQ_RDMA_WITH_IMM:
if(comp.imm_data_valid){
ompi_output(0, "Got an RQ_RDMA_WITH_IMM!\n");
}
break;
case VAPI_CQE_SQ_RDMA_WRITE: case VAPI_CQE_SQ_RDMA_WRITE:
case VAPI_CQE_SQ_SEND_DATA : case VAPI_CQE_SQ_SEND_DATA :
@ -522,6 +529,9 @@ int mca_bmi_ib_component_progress()
break; break;
} }
} }
}
while(VAPI_OK == ret);
ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_low, &comp); ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_low, &comp);
if(VAPI_OK == ret) { if(VAPI_OK == ret) {
if(comp.status != VAPI_SUCCESS) { if(comp.status != VAPI_SUCCESS) {

Просмотреть файл

@ -23,29 +23,29 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
# Additional --with flags that can be specified # Additional --with flags that can be specified
AC_ARG_WITH(ptl-ib, AC_ARG_WITH(bmi-ib,
AC_HELP_STRING([--with-ptl-ib=IBDIR], AC_HELP_STRING([--with-bmi-ib=IBDIR],
[Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)])) [Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)]))
AC_ARG_WITH(ptl-ib-libdir, AC_ARG_WITH(bmi-ib-libdir,
AC_HELP_STRING([--with-ptl-ib-libdir=IBLIBDIR], AC_HELP_STRING([--with-bmi-ib-libdir=IBLIBDIR],
[directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64])) [directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64]))
# Quick sanity check # Quick sanity check
if test "$with_ptl_ib" = "no"; then if test "$with_bmi_ib" = "no"; then
AC_MSG_WARN([*** --without-ptl-ib specified -- aborting]) AC_MSG_WARN([*** --without-bmi-ib specified -- aborting])
AC_MSG_ERROR([Will not continue]) AC_MSG_ERROR([Will not continue])
fi fi
# Find the right IBDIR # Find the right IBDIR
if test "$with_ptl_ib" != "" -a "$with_ptl_ib" != "yes" ; then if test "$with_bmi_ib" != "" -a "$with_bmi_ib" != "yes" ; then
IBDIR="$with_ptl_ib" IBDIR="$with_bmi_ib"
IBLIBDIR="$with_ptl_ib" IBLIBDIR="$with_bmi_ib"
fi fi
if test "$with_ptl_ib_libdir" != "" -a "$with_ptl_ib_libdir" != "yes" -a \ if test "$with_bmi_ib_libdir" != "" -a "$with_bmi_ib_libdir" != "yes" -a \
"$with_ptl_ib_libdir" != "no"; then "$with_bmi_ib_libdir" != "no"; then
IBLIBDIR="$with_ptl_ib_libdir" IBLIBDIR="$with_bmi_ib_libdir"
fi fi
# Add to CPPFLAGS if necessary # Add to CPPFLAGS if necessary
@ -56,13 +56,13 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
EXTRA_CPPFLAGS="-I$IBDIR/include" EXTRA_CPPFLAGS="-I$IBDIR/include"
else else
AC_MSG_WARN([*** Warning: cannot find $IBDIR/include]) AC_MSG_WARN([*** Warning: cannot find $IBDIR/include])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) AC_MSG_WARN([*** Will still try to configure ib bmi anyway...])
fi fi
if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then
EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS" EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS"
else else
AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap]) AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) AC_MSG_WARN([*** Will still try to configure ib bmi anyway...])
fi fi
fi fi
@ -90,7 +90,7 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`" CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
mca_ptl_ib_try_find_libvapi() { mca_bmi_ib_try_find_libvapi() {
func1=[$]1 func1=[$]1
func2=[$]2 func2=[$]2
@ -121,7 +121,7 @@ mca_ptl_ib_try_find_libvapi() {
if test -d "$IBLIBDIR/lib"; then if test -d "$IBLIBDIR/lib"; then
EXTRA_LDFLAGS="-L$IBLIBDIR/lib" EXTRA_LDFLAGS="-L$IBLIBDIR/lib"
LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
mca_ptl_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap mca_bmi_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap
if test "$LIBS" != "$LIBS_orig"; then if test "$LIBS" != "$LIBS_orig"; then
echo "--> found libvapi in $IBLIBDIR/lib" echo "--> found libvapi in $IBLIBDIR/lib"
fi fi
@ -130,7 +130,7 @@ mca_ptl_ib_try_find_libvapi() {
if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then
EXTRA_LDFLAGS="-L$IBLIBDIR/lib64" EXTRA_LDFLAGS="-L$IBLIBDIR/lib64"
LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS" LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS"
mca_ptl_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca mca_bmi_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca
if test "$LIBS" != "$LIBS_orig"; then if test "$LIBS" != "$LIBS_orig"; then
echo "--> found libvapi in $IBLIBDIR/lib64" echo "--> found libvapi in $IBLIBDIR/lib64"
fi fi

Просмотреть файл

@ -25,29 +25,29 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
# Additional --with flags that can be specified # Additional --with flags that can be specified
AC_ARG_WITH(ptl-ib, AC_ARG_WITH(bmi-ib,
AC_HELP_STRING([--with-ptl-ib=IBDIR], AC_HELP_STRING([--with-bmi-ib=IBDIR],
[Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)])) [Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)]))
AC_ARG_WITH(ptl-ib-libdir, AC_ARG_WITH(bmi-ib-libdir,
AC_HELP_STRING([--with-ptl-ib-libdir=IBLIBDIR], AC_HELP_STRING([--with-bmi-ib-libdir=IBLIBDIR],
[directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64])) [directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64]))
# Quick sanity check # Quick sanity check
if test "$with_ptl_ib" = "no"; then if test "$with_bmi_ib" = "no"; then
AC_MSG_WARN([*** --without-ptl-ib specified -- aborting]) AC_MSG_WARN([*** --without-bmi-ib specified -- aborting])
AC_MSG_ERROR([Will not continue]) AC_MSG_ERROR([Will not continue])
fi fi
# Find the right IBDIR # Find the right IBDIR
if test "$with_ptl_ib" != "" -a "$with_ptl_ib" != "yes" ; then if test "$with_bmi_ib" != "" -a "$with_bmi_ib" != "yes" ; then
IBDIR="$with_ptl_ib" IBDIR="$with_bmi_ib"
IBLIBDIR="$with_ptl_ib" IBLIBDIR="$with_bmi_ib"
fi fi
if test "$with_ptl_ib_libdir" != "" -a "$with_ptl_ib_libdir" != "yes" -a \ if test "$with_bmi_ib_libdir" != "" -a "$with_bmi_ib_libdir" != "yes" -a \
"$with_ptl_ib_libdir" != "no"; then "$with_bmi_ib_libdir" != "no"; then
IBLIBDIR="$with_ptl_ib_libdir" IBLIBDIR="$with_bmi_ib_libdir"
fi fi
# Add to CPPFLAGS if necessary # Add to CPPFLAGS if necessary
@ -58,13 +58,13 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
EXTRA_CPPFLAGS="-I$IBDIR/include" EXTRA_CPPFLAGS="-I$IBDIR/include"
else else
AC_MSG_WARN([*** Warning: cannot find $IBDIR/include]) AC_MSG_WARN([*** Warning: cannot find $IBDIR/include])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) AC_MSG_WARN([*** Will still try to configure ib bmi anyway...])
fi fi
if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then
EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS" EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS"
else else
AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap]) AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap])
AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) AC_MSG_WARN([*** Will still try to configure ib bmi anyway...])
fi fi
fi fi
@ -94,7 +94,7 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
mca_ptl_ib_try_find_libvapi() { mca_bmi_ib_try_find_libvapi() {
func1=[$]1 func1=[$]1
func2=[$]2 func2=[$]2
@ -125,7 +125,7 @@ mca_ptl_ib_try_find_libvapi() {
if test -d "$IBLIBDIR/lib"; then if test -d "$IBLIBDIR/lib"; then
EXTRA_LDFLAGS="-L$IBLIBDIR/lib" EXTRA_LDFLAGS="-L$IBLIBDIR/lib"
LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
mca_ptl_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap mca_bmi_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap
if test "$LIBS" != "$LIBS_orig"; then if test "$LIBS" != "$LIBS_orig"; then
echo "--> found libvapi in $IBLIBDIR/lib" echo "--> found libvapi in $IBLIBDIR/lib"
fi fi
@ -134,7 +134,7 @@ mca_ptl_ib_try_find_libvapi() {
if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then
EXTRA_LDFLAGS="-L$IBLIBDIR/lib64" EXTRA_LDFLAGS="-L$IBLIBDIR/lib64"
LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS" LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS"
mca_ptl_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca mca_bmi_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca
if test "$LIBS" != "$LIBS_orig"; then if test "$LIBS" != "$LIBS_orig"; then
echo "--> found libvapi in $IBLIBDIR/lib64" echo "--> found libvapi in $IBLIBDIR/lib64"
fi fi

Просмотреть файл

@ -33,6 +33,7 @@ extern "C" {
struct mca_mpool_vapi_component_t { struct mca_mpool_vapi_component_t {
mca_mpool_base_component_t super; mca_mpool_base_component_t super;
char* vapi_allocator_name; char* vapi_allocator_name;
long page_size;
}; };
typedef struct mca_mpool_vapi_component_t mca_mpool_vapi_component_t; typedef struct mca_mpool_vapi_component_t mca_mpool_vapi_component_t;

Просмотреть файл

@ -22,6 +22,7 @@
#include "mpool_vapi.h" #include "mpool_vapi.h"
#include "util/proc_info.h" #include "util/proc_info.h"
#include "util/sys_info.h" #include "util/sys_info.h"
#include <unistd.h>
/* /*
* Local functions * Local functions
@ -89,6 +90,8 @@ static int mca_mpool_vapi_open(void)
mca_mpool_vapi_component.vapi_allocator_name = mca_mpool_vapi_component.vapi_allocator_name =
mca_mpool_vapi_param_register_string("allocator", "bucket"); mca_mpool_vapi_param_register_string("allocator", "bucket");
mca_mpool_vapi_component.page_size = sysconf(_SC_PAGESIZE);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -175,6 +178,7 @@ static mca_mpool_base_module_t* mca_mpool_vapi_init(void * user_in)
return NULL; return NULL;
} }
return &mpool_module->super; return &mpool_module->super;
} }