diff --git a/ompi/mca/btl/udapl/btl_udapl.c b/ompi/mca/btl/udapl/btl_udapl.c index aa93ee3564..8c69d97761 100644 --- a/ompi/mca/btl/udapl/btl_udapl.c +++ b/ompi/mca/btl/udapl/btl_udapl.c @@ -60,6 +60,105 @@ mca_btl_udapl_module_t mca_btl_udapl_module = { }; +/** + * Report a uDAPL error - for debugging + */ + +static void +mca_btl_udapl_error(DAT_RETURN ret, char* str) +{ + char* major; + char* minor; + + /* don't output anything if debug is not set */ + if(0 == mca_btl_udapl_component.udapl_debug) { + return; + } + + if(DAT_SUCCESS != dat_strerror(ret, + (const char**)&major, (const char**)&minor)) + { + printf("dat_strerror failed! ret is %d\n", ret); + exit(-1); + } + + opal_output(0, "ERROR: %s %s %s\n", str, major, minor); +} + + +/** + * Initialize module module resources. + */ + +int +mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t * btl) +{ + DAT_RETURN rc; + + /* open the uDAPL interface */ + rc = dat_ia_open(ia_name, mca_btl_udapl_component.udapl_evd_qlen, + &btl->udapl_evd_dflt, &btl->udapl_ia); + if(DAT_SUCCESS != rc) { + mca_btl_udapl_error(rc, "dat_ia_open"); + return OMPI_ERROR; + } + + /* set up evd's */ + rc = dat_evd_create(btl->udapl_ia, + mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto); + if(DAT_SUCCESS != rc) { + mca_btl_udapl_error(rc, "dat_evd_create (dto)"); + return OMPI_ERROR; + } + + rc = dat_evd_create(btl->udapl_ia, + mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL, + DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_conn); + if(DAT_SUCCESS != rc) { + mca_btl_udapl_error(rc, "dat_evd_create (conn)"); + return OMPI_ERROR; + } + + /* initialize objects */ + OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t); + OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t); + OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t); + OBJ_CONSTRUCT(&btl->udapl_pending, opal_list_t); + OBJ_CONSTRUCT(&btl->udapl_repost, opal_list_t); + OBJ_CONSTRUCT(&btl->udapl_mru_reg, opal_list_t); + OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t); + + return OMPI_SUCCESS; +} + + +/* + * Cleanup/release module resources. + */ + +int mca_btl_udapl_finalize(struct mca_btl_base_module_t* base_btl) +{ + mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) base_btl; + + opal_output(0, "udapl_finalize\n"); + + /* release uDAPL resources */ + dat_evd_free(udapl_btl->udapl_evd_dto); + dat_evd_free(udapl_btl->udapl_evd_conn); + dat_ia_close(udapl_btl->udapl_ia, DAT_CLOSE_GRACEFUL_FLAG); + + /* destroy objects */ + OBJ_DESTRUCT(&udapl_btl->udapl_lock); + OBJ_DESTRUCT(&udapl_btl->udapl_frag_eager); + OBJ_DESTRUCT(&udapl_btl->udapl_frag_max); + OBJ_DESTRUCT(&udapl_btl->udapl_frag_user); + + free(udapl_btl); + return OMPI_SUCCESS; +} + + /** * */ @@ -74,7 +173,7 @@ int mca_btl_udapl_add_procs( mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl; int i, rc; - opal_output(0, "udapl_add_procs\n"); + opal_output(0, "udapl_add_procs\n"); for(i = 0; i < (int) nprocs; i++) { @@ -253,18 +352,20 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src( iov.iov_len = max_data; iov.iov_base = NULL; - ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); - + ompi_convertor_pack(convertor, &iov, + &iov_count, &max_data, &free_after); + frag->segment.seg_len = max_data; frag->segment.seg_addr.pval = iov.iov_base; - reg_len = (unsigned char*)registration->bound - (unsigned char*)iov.iov_base + 1; + reg_len = (unsigned char*)registration->bound - + (unsigned char*)iov.iov_base + 1; /* bump reference count as so that the registration * doesn't go away when the operation completes */ btl->btl_mpool->mpool_retain(btl->btl_mpool, - (mca_mpool_base_registration_t*) registration); + (mca_mpool_base_registration_t*) registration); frag->registration = registration; @@ -285,8 +386,9 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src( iov.iov_len = max_data; iov.iov_base = NULL; - ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); - + ompi_convertor_pack(convertor, &iov, + &iov_count, &max_data, &free_after); + frag->segment.seg_len = max_data; frag->segment.seg_addr.pval = iov.iov_base; @@ -494,21 +596,3 @@ int mca_btl_udapl_get( return OMPI_ERR_NOT_IMPLEMENTED; } - -/* - * Cleanup/release module resources. - */ - -int mca_btl_udapl_finalize(struct mca_btl_base_module_t* btl) -{ - mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; - - opal_output(0, "udapl_finalize\n"); - OBJ_DESTRUCT(&udapl_btl->udapl_lock); - OBJ_DESTRUCT(&udapl_btl->udapl_frag_eager); - OBJ_DESTRUCT(&udapl_btl->udapl_frag_max); - OBJ_DESTRUCT(&udapl_btl->udapl_frag_user); - free(udapl_btl); - return OMPI_SUCCESS; -} - diff --git a/ompi/mca/btl/udapl/btl_udapl.h b/ompi/mca/btl/udapl/btl_udapl.h index cac4dff207..d99a6e1291 100644 --- a/ompi/mca/btl/udapl/btl_udapl.h +++ b/ompi/mca/btl/udapl/btl_udapl.h @@ -146,6 +146,19 @@ extern mca_btl_base_module_t** mca_btl_udapl_component_init( extern int mca_btl_udapl_component_progress(void); +/** + * Initialize resources for a new BTL/uDAPL IA + * + * @param ia_name Name of uDAPL interface adapter + * @param btl BTL instance. + * @return OMPI_SUCCESS or error status on failure. + */ + +extern int mca_btl_udapl_init( + DAT_NAME_PTR ia_name, + struct mca_btl_udapl_module_t* btl +); + /** * Cleanup any resources held by the BTL. diff --git a/ompi/mca/btl/udapl/btl_udapl_component.c b/ompi/mca/btl/udapl/btl_udapl_component.c index 644d152458..9a28dbc4fa 100644 --- a/ompi/mca/btl/udapl/btl_udapl_component.c +++ b/ompi/mca/btl/udapl/btl_udapl_component.c @@ -190,84 +190,11 @@ int mca_btl_udapl_component_close(void) { opal_output(0, "udapl_component_close\n"); - /* TODO - clean up each btl module */ + /* TODO - what needs to be done here? */ return OMPI_SUCCESS; } -/** - * Report a uDAPL error - for debugging - */ - -static void -mca_btl_udapl_error(DAT_RETURN ret, char* str) -{ - char* major; - char* minor; - - /* don't output anything if debug is not set */ - if(0 == mca_btl_udapl_component.udapl_debug) { - return; - } - - if(DAT_SUCCESS != dat_strerror(ret, - (const char**)&major, (const char**)&minor)) - { - printf("dat_strerror failed! ret is %d\n", ret); - exit(-1); - } - - opal_output(0, "ERROR: %s %s %s\n", str, major, minor); -} - - -/** - * Initialize module instance - */ - -static int -mca_btl_udapl_module_init (DAT_NAME_PTR ia_name, - mca_btl_udapl_module_t * btl) -{ - DAT_RETURN rc; - - /* open the uDAPL interface */ - rc = dat_ia_open(ia_name, mca_btl_udapl_component.udapl_evd_qlen, - &btl->udapl_evd_dflt, &btl->udapl_ia); - if(DAT_SUCCESS != rc) { - mca_btl_udapl_error(rc, "dat_ia_open"); - return OMPI_ERROR; - } - - /* set up evd's */ - rc = dat_evd_create(btl->udapl_ia, - mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL, - DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto); - if(DAT_SUCCESS != rc) { - mca_btl_udapl_error(rc, "dat_evd_create (dto)"); - return OMPI_ERROR; - } - - rc = dat_evd_create(btl->udapl_ia, - mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL, - DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_conn); - if(DAT_SUCCESS != rc) { - mca_btl_udapl_error(rc, "dat_evd_create (conn)"); - return OMPI_ERROR; - } - - /* initialize objects */ - OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t); - OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t); - OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t); - OBJ_CONSTRUCT(&btl->udapl_pending, opal_list_t); - OBJ_CONSTRUCT(&btl->udapl_repost, opal_list_t); - OBJ_CONSTRUCT(&btl->udapl_mru_reg, opal_list_t); - OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t); - - return OMPI_SUCCESS; -} - /* * Register uDAPL component addressing information. The MCA framework * will make this available to all peers. @@ -301,7 +228,8 @@ mca_btl_udapl_modex_send(void) } return rc; } - + + /* * Initialize the uDAPL component, * check how many interfaces are available and create a btl module for each. @@ -359,7 +287,7 @@ mca_btl_udapl_component_init (int *num_btl_modules, /* copy default values into the new BTL */ memcpy(btl, &mca_btl_udapl_module, sizeof(mca_btl_udapl_module_t)); - if(OMPI_SUCCESS != mca_btl_udapl_module_init(datinfo[i].ia_name, btl)) { + if(OMPI_SUCCESS != mca_btl_udapl_init(datinfo[i].ia_name, btl)) { opal_output(0, "udapl module init for %s failed\n", datinfo[i].ia_name); /*TODO - how do i correctly handle an error here? */ @@ -390,7 +318,7 @@ mca_btl_udapl_component_init (int *num_btl_modules, *num_btl_modules = mca_btl_udapl_component.udapl_num_btls; return btls; } - + /* * uDAPL component progress.