From b0b6ddd0787911d74cc097e2b58637df1baa9b0b Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Thu, 28 Jul 2005 16:16:13 +0000 Subject: [PATCH] * add --enable-heterogeneous (default: enabled) to enable heterogeneous support in OMPI. Currently only enables/disables the architecture sharing modex in ob1 pml. * Add sds framework to ompi_info * Figure out table ids to use for Portals BTL at configure time, since we should use 30 & 31 on Red Storm, but the reference implementation only supports 0-8. * Some bug fixes in Portals UTCP sds This commit was SVN r6650. --- config/ompi_configure_options.m4 | 15 +++++++++++ ompi/mca/btl/portals/configure.m4 | 15 ++++++----- ompi/mca/btl/portals/src/btl_portals.h | 2 ++ .../btl/portals/src/btl_portals_compat_utcp.c | 1 + ompi/mca/pml/ob1/pml_ob1.c | 25 +++++++++++-------- ompi/tools/ompi_info/components.cc | 6 +++++ ompi/tools/ompi_info/ompi_info.cc | 1 + .../portals_utcp/sds_portals_utcp_component.c | 2 +- .../portals_utcp/sds_portals_utcp_module.c | 2 +- 9 files changed, 49 insertions(+), 20 deletions(-) diff --git a/config/ompi_configure_options.m4 b/config/ompi_configure_options.m4 index c0da080391..63e884c117 100644 --- a/config/ompi_configure_options.m4 +++ b/config/ompi_configure_options.m4 @@ -420,4 +420,19 @@ fi # this doesn't define anything in the makefiles or ompi_config.h. # Only provides a variable that the MCA and ltdl stuff keys off. +AC_MSG_CHECKING([if heterogeneous support should be enabled]) +AC_ARG_ENABLE([heterogeneous], + [AC_HELP_STRING([--enable-heterogeneous], + [Enable features required for heterogeneous + platform support (default=enabled)])]) +if test "$enable_heterogeneous" = "no" ; then + AC_MSG_RESULT([no]) + ompi_want_heterogeneous=0 +else + AC_MSG_RESULT([yes]) + ompi_want_heterogeneous=1 +fi +AC_DEFINE_UNQUOTED([OMPI_ENABLE_HETEROGENEOUS_SUPPORT], + [$ompi_want_heterogeneous], + [Enable features required for heterogeneous support]) ]) diff --git a/ompi/mca/btl/portals/configure.m4 b/ompi/mca/btl/portals/configure.m4 index 5c674e979f..8d9a66b5dd 100644 --- a/ompi/mca/btl/portals/configure.m4 +++ b/ompi/mca/btl/portals/configure.m4 @@ -45,14 +45,6 @@ AC_DEFUN([MCA_btl_portals_CONFIG_VAL], [ # ------------------------------ AC_DEFUN([MCA_btl_portals_CONFIG_VALS], [ # User configuration options - MCA_btl_portals_CONFIG_VAL([send-table-id], - [OMPI_BTL_PORTALS_SEND_TABLE_ID], [30], - [Portals table id to use for send/recv ]) - - MCA_btl_portals_CONFIG_VAL([rdma-table-id], - [OMPI_BTL_PORTALS_RDMA_TABLE_ID], [31], - [Portals table id to use for RDMA request]) - MCA_btl_portals_CONFIG_VAL([debug-level], [OMPI_BTL_PORTALS_DEFAULT_DEBUG_LEVEL], [0], [debugging level for portals btl]) @@ -111,6 +103,7 @@ AC_DEFUN([MCA_btl_portals_CONFIG_PLATFORM], [ BTL_PORTALS_HAVE_EVENT_UNLINK=0 btl_portals_compat="none" btl_portals_header_prefix= + btl_portals_starting_table_id=0 AC_ARG_WITH([btl-portals-config], AC_HELP_STRING([--with-btl-portals-config], [configuration to use for Portals support. @@ -126,6 +119,7 @@ AC_DEFUN([MCA_btl_portals_CONFIG_PLATFORM], [ btl_portals_LIBS="-lutcpapi -lutcplib -lp3api -lp3lib -lp3rt" btl_portals_compat="utcp" btl_portals_header_prefix= + btl_portals_starting_table_id=0 AC_MSG_RESULT([utcp]) ;; "redstorm") @@ -134,6 +128,7 @@ AC_DEFUN([MCA_btl_portals_CONFIG_PLATFORM], [ btl_portals_LIBS= btl_portals_compat="redstorm" btl_portals_header_prefix="portals/" + btl_portals_starting_table_id=30 AC_MSG_RESULT([red storm]) ;; *) @@ -163,6 +158,10 @@ AC_DEFUN([MCA_btl_portals_CONFIG_PLATFORM], [ AC_DEFINE_UNQUOTED([OMPI_BTL_PORTALS_REDSTORM], [$BTL_PORTALS_REDSTORM], [Use the Red Storm implementation or Portals]) + AC_DEFINE_UNQUOTED([OMPI_BTL_PORTALS_STARTING_TABLE_ID], + [$btl_portals_starting_table_id], + [first table id to use for portals btl]) + AC_CONFIG_LINKS([ompi/mca/btl/portals/src/btl_portals_compat.c:ompi/mca/btl/portals/src/btl_portals_compat_${btl_portals_compat}.c]) ]) diff --git a/ompi/mca/btl/portals/src/btl_portals.h b/ompi/mca/btl/portals/src/btl_portals.h index 23ccd1e116..2dd4d98fa0 100644 --- a/ompi/mca/btl/portals/src/btl_portals.h +++ b/ompi/mca/btl/portals/src/btl_portals.h @@ -30,6 +30,8 @@ #include "btl_portals_endpoint.h" +#define OMPI_BTL_PORTALS_SEND_TABLE_ID (OMPI_BTL_PORTALS_STARTING_TABLE_ID + 0) +#define OMPI_BTL_PORTALS_RDMA_TABLE_ID (OMPI_BTL_PORTALS_STARTING_TABLE_ID + 1) /* * Portals BTL component. diff --git a/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c b/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c index b4c1b8a5f7..6238aec012 100644 --- a/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c +++ b/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c @@ -136,6 +136,7 @@ mca_btl_portals_init_compat(mca_btl_portals_component_t *comp) "PtlNIInit failed, returning %d\n", ret); return OMPI_ERR_FATAL; } + printf("max index: %d\n", btl->portals_ni_limits.max_pt_index); } return OMPI_SUCCESS; diff --git a/ompi/mca/pml/ob1/pml_ob1.c b/ompi/mca/pml/ob1/pml_ob1.c index ed777b0563..e0a709b68d 100644 --- a/ompi/mca/pml/ob1/pml_ob1.c +++ b/ompi/mca/pml/ob1/pml_ob1.c @@ -58,15 +58,19 @@ static int mca_pml_ob1_add_btls( void ); int mca_pml_ob1_enable(bool enable) { int rc; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT uint32_t proc_arch; +#endif if( false == enable ) return OMPI_SUCCESS; +#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT /* post this processes datatype */ proc_arch = ompi_proc_local()->proc_arch; proc_arch = htonl(proc_arch); - (void) mca_base_modex_send(&mca_pml_ob1_component.pmlm_version, &proc_arch, sizeof(proc_arch)); - + rc = mca_base_modex_send(&mca_pml_ob1_component.pmlm_version, &proc_arch, sizeof(proc_arch)); + if (OMPI_SUCCESS != rc) return rc; +#endif OBJ_CONSTRUCT(&mca_pml_ob1.lock, opal_mutex_t); /* requests */ @@ -261,18 +265,19 @@ int mca_pml_ob1_add_procs(ompi_proc_t** procs, size_t nprocs) /* iterate through each of the procs and set the peers architecture */ for(p=0; pproc_arch = ntohl(*proc_arch); - free(proc_arch); - } else { - procs[p]->proc_arch = ompi_proc_local()->proc_arch; - } + if(rc == OMPI_SUCCESS) return rc; + if(size != sizeof(uint32_t)) + return OMPI_ERROR; + procs[p]->proc_arch = ntohl(*proc_arch); + free(proc_arch); +#else + procs[p]->proc_arch = ompi_proc_local()->proc_arch; +#endif } /* attempt to add all procs to each ob1 */ diff --git a/ompi/tools/ompi_info/components.cc b/ompi/tools/ompi_info/components.cc index db9a079dfc..e3b7844850 100644 --- a/ompi/tools/ompi_info/components.cc +++ b/ompi/tools/ompi_info/components.cc @@ -68,6 +68,8 @@ #include "mca/pls/base/base.h" #include "mca/soh/soh.h" #include "mca/soh/base/base.h" +#include "mca/sds/sds.h" +#include "mca/sds/base/base.h" using namespace std; using namespace ompi_info; @@ -170,6 +172,9 @@ void ompi_info::open_components() orte_pls_base_open(); component_map["pls"] = &orte_pls_base.pls_opened; + orte_sds_base_open(); + component_map["sds"] = &orte_sds_base_components_available; + orte_soh_base_open(); component_map["soh"] = &orte_soh_base.soh_components; @@ -230,6 +235,7 @@ void ompi_info::close_components() mca_allocator_base_close(); orte_iof_base_close(); + orte_sds_base_close(); orte_soh_base_close(); orte_pls_base_close(); orte_rmgr_base_close(); diff --git a/ompi/tools/ompi_info/ompi_info.cc b/ompi/tools/ompi_info/ompi_info.cc index 11d78a2c9b..893886d498 100644 --- a/ompi/tools/ompi_info/ompi_info.cc +++ b/ompi/tools/ompi_info/ompi_info.cc @@ -180,6 +180,7 @@ int main(int argc, char *argv[]) ompi_info::mca_types.push_back("rmgr"); ompi_info::mca_types.push_back("rml"); ompi_info::mca_types.push_back("pls"); + ompi_info::mca_types.push_back("sds"); ompi_info::mca_types.push_back("soh"); // Execute the desired action(s) diff --git a/orte/mca/sds/portals_utcp/sds_portals_utcp_component.c b/orte/mca/sds/portals_utcp/sds_portals_utcp_component.c index e276cd5a6a..588fcf913c 100644 --- a/orte/mca/sds/portals_utcp/sds_portals_utcp_component.c +++ b/orte/mca/sds/portals_utcp/sds_portals_utcp_component.c @@ -82,7 +82,7 @@ orte_sds_portals_utcp_component_init(int *priority) /* if mode isn't NULL, then we have an ORTE starter. Don't use this component */ if (NULL != mode) return NULL; - if (NULL == getenv("PTL_MY_RID") return NULL; + if (NULL == getenv("PTL_MY_RID")) return NULL; *priority = 60; return &sds_portals_utcp_module; diff --git a/orte/mca/sds/portals_utcp/sds_portals_utcp_module.c b/orte/mca/sds/portals_utcp/sds_portals_utcp_module.c index e077588dc4..7962f50032 100644 --- a/orte/mca/sds/portals_utcp/sds_portals_utcp_module.c +++ b/orte/mca/sds/portals_utcp/sds_portals_utcp_module.c @@ -26,7 +26,7 @@ #include "mca/errmgr/base/base.h" orte_sds_base_module_t sds_portals_utcp_module = { - orte_sds_base_contact_universe, + orte_sds_portals_utcp_contact_universe, orte_sds_portals_utcp_set_name, orte_sds_portals_utcp_finalize, };