From c1bf71b1be062823a52cd3ddadb75449fab64a9a Mon Sep 17 00:00:00 2001 From: Tim Woodall Date: Mon, 27 Mar 2006 22:48:12 +0000 Subject: [PATCH] - updated copyrights - removed unused state - starting to add support for btl failover This commit was SVN r9431. --- ompi/mca/bml/base/base.h | 2 +- ompi/mca/bml/base/bml_base_btl.c | 2 +- ompi/mca/bml/base/bml_base_btl.h | 2 +- ompi/mca/bml/base/bml_base_close.c | 2 +- ompi/mca/bml/base/bml_base_endpoint.c | 6 +- ompi/mca/bml/base/bml_base_endpoint.h | 2 +- ompi/mca/bml/base/bml_base_init.c | 2 +- ompi/mca/bml/base/bml_base_lookup.c | 17 ---- ompi/mca/bml/base/bml_base_open.c | 2 +- ompi/mca/bml/bml.h | 32 +++++-- ompi/mca/bml/r2/bml_r2.c | 120 +++++++++++++++++++++++++- ompi/mca/bml/r2/bml_r2.h | 6 +- ompi/mca/bml/r2/bml_r2_btl.c | 2 +- ompi/mca/bml/r2/bml_r2_btl.h | 2 +- ompi/mca/bml/r2/bml_r2_component.c | 2 +- 15 files changed, 158 insertions(+), 43 deletions(-) delete mode 100644 ompi/mca/bml/base/bml_base_lookup.c diff --git a/ompi/mca/bml/base/base.h b/ompi/mca/bml/base/base.h index b734acbec7..7a1487d25a 100644 --- a/ompi/mca/bml/base/base.h +++ b/ompi/mca/bml/base/base.h @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/base/bml_base_btl.c b/ompi/mca/bml/base/bml_base_btl.c index 9fb85b4755..8fb152960b 100644 --- a/ompi/mca/bml/base/bml_base_btl.c +++ b/ompi/mca/bml/base/bml_base_btl.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/base/bml_base_btl.h b/ompi/mca/bml/base/bml_base_btl.h index 10be4408f6..13bf48c078 100644 --- a/ompi/mca/bml/base/bml_base_btl.h +++ b/ompi/mca/bml/base/bml_base_btl.h @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/base/bml_base_close.c b/ompi/mca/bml/base/bml_base_close.c index cf6eb313a8..5114abfb0f 100644 --- a/ompi/mca/bml/base/bml_base_close.c +++ b/ompi/mca/bml/base/bml_base_close.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/base/bml_base_endpoint.c b/ompi/mca/bml/base/bml_base_endpoint.c index 8d9f5ee5d8..7dbf456541 100644 --- a/ompi/mca/bml/base/bml_base_endpoint.c +++ b/ompi/mca/bml/base/bml_base_endpoint.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * @@ -27,14 +27,11 @@ static void mca_bml_base_endpoint_construct(mca_bml_base_endpoint_t* ep) { - ep->btl_proc = NULL; ep->btl_rdma_offset = 0; ep->btl_max_send_size = 0; - ep->btl_flags = 0; ep->btl_rdma_size = 0; ep->btl_rdma_align = 0; - OBJ_CONSTRUCT(&ep->btl_lock, opal_mutex_t); OBJ_CONSTRUCT(&ep->btl_eager, mca_bml_base_btl_array_t); OBJ_CONSTRUCT(&ep->btl_send, mca_bml_base_btl_array_t); OBJ_CONSTRUCT(&ep->btl_rdma, mca_bml_base_btl_array_t); @@ -43,7 +40,6 @@ static void mca_bml_base_endpoint_construct(mca_bml_base_endpoint_t* ep) static void mca_bml_base_endpoint_destruct(mca_bml_base_endpoint_t* ep) { - OBJ_DESTRUCT(&ep->btl_lock); OBJ_DESTRUCT(&ep->btl_eager); OBJ_DESTRUCT(&ep->btl_send); OBJ_DESTRUCT(&ep->btl_rdma); diff --git a/ompi/mca/bml/base/bml_base_endpoint.h b/ompi/mca/bml/base/bml_base_endpoint.h index 80968e8912..c670607de1 100644 --- a/ompi/mca/bml/base/bml_base_endpoint.h +++ b/ompi/mca/bml/base/bml_base_endpoint.h @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/base/bml_base_init.c b/ompi/mca/bml/base/bml_base_init.c index 3f8349c845..27617afb42 100644 --- a/ompi/mca/bml/base/bml_base_init.c +++ b/ompi/mca/bml/base/bml_base_init.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/base/bml_base_lookup.c b/ompi/mca/bml/base/bml_base_lookup.c deleted file mode 100644 index 6f374daf13..0000000000 --- a/ompi/mca/bml/base/bml_base_lookup.c +++ /dev/null @@ -1,17 +0,0 @@ -mca_bml_base_module_t* mca_bml_base_module_lookup(const char* name) -{ - opal_list_item_t* item; - - for (item = opal_list_get_first(&mca_bml_base_modules); - item != opal_list_get_end(&mca_bml_base_modules); - item = opal_list_get_next(item)) { - mca_mpool_base_selected_module_t *mli = - (mca_mpool_base_selected_module_t *) item; - if(0 == strcmp(mli->mpool_component->mpool_version.mca_component_name, - name)) { - return mli->mpool_module; - } - } - - return NULL; -} diff --git a/ompi/mca/bml/base/bml_base_open.c b/ompi/mca/bml/base/bml_base_open.c index 7668507b70..ae5e2efb2b 100644 --- a/ompi/mca/bml/base/bml_base_open.c +++ b/ompi/mca/bml/base/bml_base_open.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/bml.h b/ompi/mca/bml/bml.h index 0a6c8bab23..5605ab6435 100644 --- a/ompi/mca/bml/bml.h +++ b/ompi/mca/bml/bml.h @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * @@ -211,9 +211,6 @@ static inline mca_bml_base_btl_t* mca_bml_base_btl_array_find( */ struct mca_bml_base_endpoint_t { mca_pml_proc_t super; - struct ompi_proc_t *btl_proc; /**< back-pointer to ompi_proc_t */ - opal_mutex_t btl_lock; /**< lock to protect against concurrent access */ - int btl_flags; /**< prefered method of accessing this peer */ size_t btl_rdma_offset; /**< max of min rdma size for available rmda btls */ size_t btl_max_send_size; /**< min of max send size for available send btls */ size_t btl_rdma_size; /**< max of min rdma size for available rmda btls */ @@ -451,7 +448,6 @@ typedef int (*mca_bml_base_module_finalize_fn_t)( void ); /** * PML->BML notification of change in the process list. * - * @param bml (IN) BML module * @param nprocs (IN) Number of processes * @param procs (IN) Set of processes * @param endpoint (OUT) Set of (optional) mca_bml_base_endpoint_t structures by BML. @@ -482,10 +478,8 @@ typedef int (*mca_bml_base_module_add_procs_fn_t)( /** * Notification of change to the process list. * - * @param bml (IN) BML module * @param nprocs (IN) Number of processes * @param proc (IN) Set of processes - * @param peer (IN) Set of peer addressing information. * @return Status indicating if cleanup was successful * * When the process list changes, the PML notifies the BML of the @@ -497,6 +491,28 @@ typedef int (*mca_bml_base_module_del_procs_fn_t)( struct ompi_proc_t** procs ); +/** + * Notification of change to the btl list. + * + * @param bml (IN) BTL module + * @return Status indicating if cleanup was successful + * + * On recovery of a btl, add it to the set of forwarding + * entries used by the BML. + */ +typedef int (*mca_bml_base_module_add_btl_fn_t)( struct mca_btl_base_module_t* ); + +/** + * Notification of change to the btl list. + * + * @param bml (IN) BTL module + * @return Status indicating if cleanup was successful + * + * On failure of a btl, remove it from the set of forwarding + * entries used by the BML. + */ +typedef int (*mca_bml_base_module_del_btl_fn_t)( struct mca_btl_base_module_t* ); + /** * Callback function that is called asynchronously on receipt * of data by the transport layer. @@ -544,6 +560,8 @@ struct mca_bml_base_module_t { /* BML function table */ mca_bml_base_module_add_procs_fn_t bml_add_procs; mca_bml_base_module_del_procs_fn_t bml_del_procs; + mca_bml_base_module_add_btl_fn_t bml_add_btl; + mca_bml_base_module_del_btl_fn_t bml_del_btl; mca_bml_base_module_register_fn_t bml_register; mca_bml_base_module_finalize_fn_t bml_finalize; diff --git a/ompi/mca/bml/r2/bml_r2.c b/ompi/mca/bml/r2/bml_r2.c index 1b2aea089b..0e850e2e00 100644 --- a/ompi/mca/bml/r2/bml_r2.c +++ b/ompi/mca/bml/r2/bml_r2.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * @@ -43,6 +43,8 @@ mca_bml_r2_module_t mca_bml_r2 = { 0, /* max rdma size */ mca_bml_r2_add_procs, mca_bml_r2_del_procs, + mca_bml_r2_add_btl, + mca_bml_r2_del_btl, mca_bml_r2_register, mca_bml_r2_finalize, mca_bml_r2_progress @@ -259,7 +261,7 @@ int mca_bml_r2_add_procs( mca_bml_base_btl_array_reserve(&bml_endpoint->btl_rdma, mca_bml_r2.num_btl_modules); bml_endpoint->btl_max_send_size = -1; bml_endpoint->btl_rdma_size = -1; - bml_endpoint->btl_proc = proc; + bml_endpoint->super.proc_ompi = proc; proc->proc_pml = (struct mca_pml_proc_t*) bml_endpoint; } @@ -302,7 +304,6 @@ int mca_bml_r2_add_procs( bml_btl->btl_put = btl->btl_put; bml_btl->btl_get = btl->btl_get; bml_btl->btl_flags = btl->btl_flags; - bml_btl->btl_mpool = btl->btl_mpool; } } @@ -483,6 +484,118 @@ int mca_bml_r2_finalize( void ) { return OMPI_SUCCESS; /* TODO */ } + +/* + * (1) Remove btl from each bml endpoint + * (2) Remove btl from the global list + */ + +int mca_bml_r2_del_btl(mca_btl_base_module_t* btl) +{ + ompi_proc_t** procs; + size_t p, num_procs; + + procs = ompi_proc_all(&num_procs); + if(NULL == procs) + return OMPI_SUCCESS; + + for(p=0; pproc_pml; + mca_bml_base_endpoint_t* ep_new = OBJ_NEW(mca_bml_base_endpoint_t); + size_t b; + + /* initialize */ + ep_new->super.proc_ompi = proc; + ep_new->btl_max_send_size = -1; + ep_new->btl_rdma_size = -1; + ep_new->btl_rdma_align = 0; + ep_new->btl_rdma_offset = 0; + + /* build new eager list */ + mca_bml_base_btl_array_reserve(&ep_new->btl_eager, mca_bml_base_btl_array_get_size(&ep_old->btl_eager)); + for(b=0; b< mca_bml_base_btl_array_get_size(&ep_old->btl_eager); b++) { + mca_bml_base_btl_t* bml_btl_old = mca_bml_base_btl_array_get_index(&ep_old->btl_eager, b); + if(bml_btl_old->btl != btl) { + mca_bml_base_btl_t* bml_btl_new = mca_bml_base_btl_array_insert(&ep_new->btl_eager); + *bml_btl_new = *bml_btl_old; + } + } + + /* build new send list */ + total_bandwidth = 0; + mca_bml_base_btl_array_reserve(&ep_new->btl_send, mca_bml_base_btl_array_get_size(&ep_old->btl_send)); + for(b=0; b< mca_bml_base_btl_array_get_size(&ep_old->btl_send); b++) { + mca_bml_base_btl_t* bml_btl_old = mca_bml_base_btl_array_get_index(&ep_old->btl_send, b); + if(bml_btl_old->btl != btl) { + mca_bml_base_btl_t* bml_btl_new = mca_bml_base_btl_array_insert(&ep_new->btl_send); + *bml_btl_new = *bml_btl_old; + total_bandwidth += bml_btl_new->btl->btl_bandwidth; + if (bml_btl_new->btl_max_send_size < ep_new->btl_max_send_size) { + ep_new->btl_max_send_size = bml_btl_new->btl->btl_max_send_size; + } + } + } + /* compute weighting factor for this btl */ + for(b=0; b< mca_bml_base_btl_array_get_size(&ep_new->btl_send); b++) { + mca_bml_base_btl_t* bml_btl = mca_bml_base_btl_array_get_index(&ep_old->btl_send, b); + if(bml_btl->btl->btl_bandwidth > 0) { + bml_btl->btl_weight = bml_btl->btl->btl_bandwidth / total_bandwidth; + } else { + bml_btl->btl_weight = 1.0 / mca_bml_base_btl_array_get_size(&ep_new->btl_send); + } + } + + /* build new rdma list */ + total_bandwidth = 0; + mca_bml_base_btl_array_reserve(&ep_new->btl_rdma, mca_bml_base_btl_array_get_size(&ep_old->btl_rdma)); + for(b=0; b< mca_bml_base_btl_array_get_size(&ep_old->btl_rdma); b++) { + mca_bml_base_btl_t* bml_btl_old = mca_bml_base_btl_array_get_index(&ep_old->btl_rdma, b); + if(bml_btl_old->btl != btl) { + mca_bml_base_btl_t* bml_btl_new = mca_bml_base_btl_array_insert(&ep_new->btl_rdma); + *bml_btl_new = *bml_btl_old; + + /* update aggregate endpoint info */ + total_bandwidth += bml_btl_new->btl->btl_bandwidth; + if (ep_new->btl_rdma_offset < bml_btl_new->btl_min_rdma_size) { + ep_new->btl_rdma_offset = bml_btl_new->btl_min_rdma_size; + } + if (ep_new->btl_rdma_size > bml_btl_new->btl_max_rdma_size) { + ep_new->btl_rdma_size = bml_btl_new->btl_max_rdma_size; + ep_new->btl_rdma_align = bml_base_log2(ep_new->btl_rdma_size); + } + } + } + + /* compute weighting factor for this btl */ + for(b=0; b< mca_bml_base_btl_array_get_size(&ep_new->btl_rdma); b++) { + mca_bml_base_btl_t* bml_btl = mca_bml_base_btl_array_get_index(&ep_old->btl_rdma, b); + if(bml_btl->btl->btl_bandwidth > 0) { + bml_btl->btl_weight = bml_btl->btl->btl_bandwidth / total_bandwidth; + } else { + bml_btl->btl_weight = 1.0 / mca_bml_base_btl_array_get_size(&ep_new->btl_rdma); + } + } + + /* save on proc */ + proc->proc_pml = (mca_pml_proc_t*)ep_new; + + /* DONT delete old endpoint structure as it may still be in use... */ + } + return OMPI_SUCCESS; +} + +int mca_bml_r2_add_btl(mca_btl_base_module_t* btl) +{ + return ORTE_ERR_NOT_IMPLEMENTED; +} + + +/* + * Register callback w/ all active btls + */ + int mca_bml_r2_register( mca_btl_base_tag_t tag, mca_bml_base_module_recv_cb_fn_t cbfunc, @@ -510,3 +623,4 @@ int mca_bml_r2_component_fini(void) return OMPI_SUCCESS; } + diff --git a/ompi/mca/bml/r2/bml_r2.h b/ompi/mca/bml/r2/bml_r2.h index 3e08f16fa3..ef1808b2f2 100644 --- a/ompi/mca/bml/r2/bml_r2.h +++ b/ompi/mca/bml/r2/bml_r2.h @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * @@ -83,6 +83,10 @@ int mca_bml_r2_del_procs( struct ompi_proc_t** procs ); +int mca_bml_r2_add_btl( mca_btl_base_module_t* btl ); + +int mca_bml_r2_del_btl( mca_btl_base_module_t* btl ); + int mca_bml_r2_register( mca_btl_base_tag_t tag, mca_bml_base_module_recv_cb_fn_t cbfunc, diff --git a/ompi/mca/bml/r2/bml_r2_btl.c b/ompi/mca/bml/r2/bml_r2_btl.c index b71160cd89..59e0f8bded 100644 --- a/ompi/mca/bml/r2/bml_r2_btl.c +++ b/ompi/mca/bml/r2/bml_r2_btl.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/r2/bml_r2_btl.h b/ompi/mca/bml/r2/bml_r2_btl.h index bb8b2db4d7..384e355995 100644 --- a/ompi/mca/bml/r2/bml_r2_btl.h +++ b/ompi/mca/bml/r2/bml_r2_btl.h @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/bml/r2/bml_r2_component.c b/ompi/mca/bml/r2/bml_r2_component.c index 16164af665..db7ec420b9 100644 --- a/ompi/mca/bml/r2/bml_r2_component.c +++ b/ompi/mca/bml/r2/bml_r2_component.c @@ -7,7 +7,7 @@ * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. + * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ *