From 507730a2ca9b9c1ea6b27d54d575fe9bc4d157b7 Mon Sep 17 00:00:00 2001 From: Galen Shipman Date: Fri, 17 Jun 2005 19:15:03 +0000 Subject: [PATCH] Added with-bmi-ib instead of with-ptl-ib Begun adding support for RDMA_WRITE_WITH_IMM This commit was SVN r6081. --- src/mca/bmi/bmi.h | 1 + src/mca/bmi/ib/bmi_ib.c | 14 ++++++---- src/mca/bmi/ib/bmi_ib_component.c | 10 +++++++ src/mca/bmi/ib/bmi_ib_frag.h | 2 +- src/mca/bmi/ib/configure.stub | 34 +++++++++++------------ src/mca/mpool/vapi/configure.stub | 34 +++++++++++------------ src/mca/mpool/vapi/mpool_vapi.h | 5 ++-- src/mca/mpool/vapi/mpool_vapi_component.c | 4 +++ 8 files changed, 62 insertions(+), 42 deletions(-) diff --git a/src/mca/bmi/bmi.h b/src/mca/bmi/bmi.h index d18b2562be..b39490b98b 100644 --- a/src/mca/bmi/bmi.h +++ b/src/mca/bmi/bmi.h @@ -172,6 +172,7 @@ struct mca_bmi_base_descriptor_t { mca_bmi_base_completion_fn_t des_cbfunc; void* des_cbdata; int32_t des_flags; + ompi_ptr_t user_data; }; typedef struct mca_bmi_base_descriptor_t mca_bmi_base_descriptor_t; diff --git a/src/mca/bmi/ib/bmi_ib.c b/src/mca/bmi/ib/bmi_ib.c index 84e47c5758..d657fdb11d 100644 --- a/src/mca/bmi/ib/bmi_ib.c +++ b/src/mca/bmi/ib/bmi_ib.c @@ -252,7 +252,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src( frag->base.des_dst = NULL; frag->base.des_dst_cnt = 0; frag->base.des_flags = 0; - + frag->base.user_data.pval = NULL; return &frag->base; }else if( max_data + reserve <= ib_bmi->super.bmi_max_send_size || 1 == ompi_convertor_need_buffers( convertor) ){ @@ -281,7 +281,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src( frag->base.des_dst = NULL; frag->base.des_dst_cnt = 0; frag->base.des_flags=0; - + frag->base.user_data.pval = NULL; return &frag->base; } else { @@ -363,7 +363,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src( frag->base.des_dst = NULL; frag->base.des_dst_cnt = 0; frag->base.des_flags=0; - + frag->base.user_data.pval = NULL; return &frag->base; } @@ -536,8 +536,12 @@ int mca_bmi_ib_put( mca_bmi_base_module_t* bmi, { mca_bmi_ib_module_t* ib_bmi = (mca_bmi_ib_module_t*) bmi; mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*) descriptor; - frag->endpoint = endpoint; - frag->sr_desc.opcode = VAPI_RDMA_WRITE; + frag->endpoint = endpoint; + if(NULL == frag->base.user_data.pval) + frag->sr_desc.opcode = VAPI_RDMA_WRITE; + else + frag->sr_desc.opcode = VAPI_RDMA_WRITE_WITH_IMM; + frag->sr_desc.remote_qp = endpoint->rem_qp_num_low; frag->sr_desc.remote_addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->base.des_dst->seg_addr.pval; frag->sr_desc.r_key = frag->base.des_dst->seg_key.key32[0]; diff --git a/src/mca/bmi/ib/bmi_ib_component.c b/src/mca/bmi/ib/bmi_ib_component.c index e23250c255..ba5a120943 100644 --- a/src/mca/bmi/ib/bmi_ib_component.c +++ b/src/mca/bmi/ib/bmi_ib_component.c @@ -479,6 +479,7 @@ int mca_bmi_ib_component_progress() VAPI_wc_desc_t comp; mca_bmi_ib_module_t* ib_bmi = &mca_bmi_ib_component.ib_bmis[i]; + do{ ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_high, &comp); if(VAPI_OK == ret) { if(comp.status != VAPI_SUCCESS) { @@ -490,6 +491,12 @@ int mca_bmi_ib_component_progress() /* Handle n/w completions */ switch(comp.opcode) { + case VAPI_CQE_RQ_RDMA_WITH_IMM: + if(comp.imm_data_valid){ + ompi_output(0, "Got an RQ_RDMA_WITH_IMM!\n"); + + } + break; case VAPI_CQE_SQ_RDMA_WRITE: case VAPI_CQE_SQ_SEND_DATA : @@ -522,6 +529,9 @@ int mca_bmi_ib_component_progress() break; } } + } + while(VAPI_OK == ret); + ret = VAPI_poll_cq(ib_bmi->nic, ib_bmi->cq_hndl_low, &comp); if(VAPI_OK == ret) { if(comp.status != VAPI_SUCCESS) { diff --git a/src/mca/bmi/ib/bmi_ib_frag.h b/src/mca/bmi/ib/bmi_ib_frag.h index 2c8608853b..33431e2741 100644 --- a/src/mca/bmi/ib/bmi_ib_frag.h +++ b/src/mca/bmi/ib/bmi_ib_frag.h @@ -60,7 +60,7 @@ struct mca_bmi_ib_frag_t { VAPI_sg_lst_entry_t sg_entry; VAPI_mr_hndl_t mem_hndl; VAPI_ret_t ret; - mca_bmi_ib_header_t *hdr; + mca_bmi_ib_header_t *hdr; }; typedef struct mca_bmi_ib_frag_t mca_bmi_ib_frag_t; OBJ_CLASS_DECLARATION(mca_bmi_ib_frag_t); diff --git a/src/mca/bmi/ib/configure.stub b/src/mca/bmi/ib/configure.stub index 5b50fb3785..2db5aa9a4d 100644 --- a/src/mca/bmi/ib/configure.stub +++ b/src/mca/bmi/ib/configure.stub @@ -23,29 +23,29 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ # Additional --with flags that can be specified - AC_ARG_WITH(ptl-ib, - AC_HELP_STRING([--with-ptl-ib=IBDIR], + AC_ARG_WITH(bmi-ib, + AC_HELP_STRING([--with-bmi-ib=IBDIR], [Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)])) - AC_ARG_WITH(ptl-ib-libdir, - AC_HELP_STRING([--with-ptl-ib-libdir=IBLIBDIR], + AC_ARG_WITH(bmi-ib-libdir, + AC_HELP_STRING([--with-bmi-ib-libdir=IBLIBDIR], [directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64])) # Quick sanity check - if test "$with_ptl_ib" = "no"; then - AC_MSG_WARN([*** --without-ptl-ib specified -- aborting]) + if test "$with_bmi_ib" = "no"; then + AC_MSG_WARN([*** --without-bmi-ib specified -- aborting]) AC_MSG_ERROR([Will not continue]) fi # Find the right IBDIR - if test "$with_ptl_ib" != "" -a "$with_ptl_ib" != "yes" ; then - IBDIR="$with_ptl_ib" - IBLIBDIR="$with_ptl_ib" + if test "$with_bmi_ib" != "" -a "$with_bmi_ib" != "yes" ; then + IBDIR="$with_bmi_ib" + IBLIBDIR="$with_bmi_ib" fi - if test "$with_ptl_ib_libdir" != "" -a "$with_ptl_ib_libdir" != "yes" -a \ - "$with_ptl_ib_libdir" != "no"; then - IBLIBDIR="$with_ptl_ib_libdir" + if test "$with_bmi_ib_libdir" != "" -a "$with_bmi_ib_libdir" != "yes" -a \ + "$with_bmi_ib_libdir" != "no"; then + IBLIBDIR="$with_bmi_ib_libdir" fi # Add to CPPFLAGS if necessary @@ -56,13 +56,13 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ EXTRA_CPPFLAGS="-I$IBDIR/include" else AC_MSG_WARN([*** Warning: cannot find $IBDIR/include]) - AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) + AC_MSG_WARN([*** Will still try to configure ib bmi anyway...]) fi if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS" else AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap]) - AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) + AC_MSG_WARN([*** Will still try to configure ib bmi anyway...]) fi fi @@ -90,7 +90,7 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`" -mca_ptl_ib_try_find_libvapi() { +mca_bmi_ib_try_find_libvapi() { func1=[$]1 func2=[$]2 @@ -121,7 +121,7 @@ mca_ptl_ib_try_find_libvapi() { if test -d "$IBLIBDIR/lib"; then EXTRA_LDFLAGS="-L$IBLIBDIR/lib" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" - mca_ptl_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap + mca_bmi_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap if test "$LIBS" != "$LIBS_orig"; then echo "--> found libvapi in $IBLIBDIR/lib" fi @@ -130,7 +130,7 @@ mca_ptl_ib_try_find_libvapi() { if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then EXTRA_LDFLAGS="-L$IBLIBDIR/lib64" LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS" - mca_ptl_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca + mca_bmi_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca if test "$LIBS" != "$LIBS_orig"; then echo "--> found libvapi in $IBLIBDIR/lib64" fi diff --git a/src/mca/mpool/vapi/configure.stub b/src/mca/mpool/vapi/configure.stub index ac923d0d0e..2329e743fb 100644 --- a/src/mca/mpool/vapi/configure.stub +++ b/src/mca/mpool/vapi/configure.stub @@ -25,29 +25,29 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ # Additional --with flags that can be specified - AC_ARG_WITH(ptl-ib, - AC_HELP_STRING([--with-ptl-ib=IBDIR], + AC_ARG_WITH(bmi-ib, + AC_HELP_STRING([--with-bmi-ib=IBDIR], [Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)])) - AC_ARG_WITH(ptl-ib-libdir, - AC_HELP_STRING([--with-ptl-ib-libdir=IBLIBDIR], + AC_ARG_WITH(bmi-ib-libdir, + AC_HELP_STRING([--with-bmi-ib-libdir=IBLIBDIR], [directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64])) # Quick sanity check - if test "$with_ptl_ib" = "no"; then - AC_MSG_WARN([*** --without-ptl-ib specified -- aborting]) + if test "$with_bmi_ib" = "no"; then + AC_MSG_WARN([*** --without-bmi-ib specified -- aborting]) AC_MSG_ERROR([Will not continue]) fi # Find the right IBDIR - if test "$with_ptl_ib" != "" -a "$with_ptl_ib" != "yes" ; then - IBDIR="$with_ptl_ib" - IBLIBDIR="$with_ptl_ib" + if test "$with_bmi_ib" != "" -a "$with_bmi_ib" != "yes" ; then + IBDIR="$with_bmi_ib" + IBLIBDIR="$with_bmi_ib" fi - if test "$with_ptl_ib_libdir" != "" -a "$with_ptl_ib_libdir" != "yes" -a \ - "$with_ptl_ib_libdir" != "no"; then - IBLIBDIR="$with_ptl_ib_libdir" + if test "$with_bmi_ib_libdir" != "" -a "$with_bmi_ib_libdir" != "yes" -a \ + "$with_bmi_ib_libdir" != "no"; then + IBLIBDIR="$with_bmi_ib_libdir" fi # Add to CPPFLAGS if necessary @@ -58,13 +58,13 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ EXTRA_CPPFLAGS="-I$IBDIR/include" else AC_MSG_WARN([*** Warning: cannot find $IBDIR/include]) - AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) + AC_MSG_WARN([*** Will still try to configure ib bmi anyway...]) fi if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS" else AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap]) - AC_MSG_WARN([*** Will still try to configure ib ptl anyway...]) + AC_MSG_WARN([*** Will still try to configure ib bmi anyway...]) fi fi @@ -94,7 +94,7 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ -mca_ptl_ib_try_find_libvapi() { +mca_bmi_ib_try_find_libvapi() { func1=[$]1 func2=[$]2 @@ -125,7 +125,7 @@ mca_ptl_ib_try_find_libvapi() { if test -d "$IBLIBDIR/lib"; then EXTRA_LDFLAGS="-L$IBLIBDIR/lib" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" - mca_ptl_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap + mca_bmi_ib_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap if test "$LIBS" != "$LIBS_orig"; then echo "--> found libvapi in $IBLIBDIR/lib" fi @@ -134,7 +134,7 @@ mca_ptl_ib_try_find_libvapi() { if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then EXTRA_LDFLAGS="-L$IBLIBDIR/lib64" LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS" - mca_ptl_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca + mca_bmi_ib_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca if test "$LIBS" != "$LIBS_orig"; then echo "--> found libvapi in $IBLIBDIR/lib64" fi diff --git a/src/mca/mpool/vapi/mpool_vapi.h b/src/mca/mpool/vapi/mpool_vapi.h index 0c1e6b639b..3c4ff66ed4 100644 --- a/src/mca/mpool/vapi/mpool_vapi.h +++ b/src/mca/mpool/vapi/mpool_vapi.h @@ -31,8 +31,9 @@ extern "C" { #endif struct mca_mpool_vapi_component_t { - mca_mpool_base_component_t super; - char* vapi_allocator_name; + mca_mpool_base_component_t super; + char* vapi_allocator_name; + long page_size; }; typedef struct mca_mpool_vapi_component_t mca_mpool_vapi_component_t; diff --git a/src/mca/mpool/vapi/mpool_vapi_component.c b/src/mca/mpool/vapi/mpool_vapi_component.c index 96caab1fcf..1686dab9b7 100644 --- a/src/mca/mpool/vapi/mpool_vapi_component.c +++ b/src/mca/mpool/vapi/mpool_vapi_component.c @@ -22,6 +22,7 @@ #include "mpool_vapi.h" #include "util/proc_info.h" #include "util/sys_info.h" +#include /* * Local functions @@ -89,6 +90,8 @@ static int mca_mpool_vapi_open(void) mca_mpool_vapi_component.vapi_allocator_name = mca_mpool_vapi_param_register_string("allocator", "bucket"); + mca_mpool_vapi_component.page_size = sysconf(_SC_PAGESIZE); + return OMPI_SUCCESS; } @@ -175,6 +178,7 @@ static mca_mpool_base_module_t* mca_mpool_vapi_init(void * user_in) return NULL; } + return &mpool_module->super; }