From 165d4f88415f9ae70b1f3afee481024ba4d9a624 Mon Sep 17 00:00:00 2001 From: Tim Woodall Date: Thu, 1 Jul 2004 14:49:54 +0000 Subject: [PATCH] merge of oob/rte changes from rte branch This commit was SVN r1527. --- autogen.sh | 4 +- config/mca_configure.ac | 7 +- config/ompi_mca.m4 | 115 +++--- configure.ac | 4 +- src/communicator/comm.c | 15 +- src/communicator/comm_init.c | 2 +- src/dynamic-mca/allocator/Makefile.am | 2 +- src/dynamic-mca/coll/Makefile.am | 2 +- src/dynamic-mca/common/Makefile.am | 2 +- src/dynamic-mca/gpr/Makefile.am | 2 +- src/dynamic-mca/io/Makefile.am | 2 +- src/dynamic-mca/mpool/Makefile.am | 2 +- src/dynamic-mca/one/Makefile.am | 2 +- src/dynamic-mca/oob/Makefile.am | 2 +- src/dynamic-mca/pcm/Makefile.am | 2 +- src/dynamic-mca/pml/Makefile.am | 2 +- src/dynamic-mca/ptl/Makefile.am | 2 +- src/dynamic-mca/topo/Makefile.am | 2 +- src/mca/allocator/bucket/Makefile.am | 23 +- src/mca/base/mca_base_module_exchange.c | 37 +- src/mca/coll/base/coll_base_close.c | 4 - src/mca/coll/basic/Makefile.am | 29 +- src/mca/coll/basic/src/coll_basic_reduce.c | 44 +-- src/mca/io/romio/Makefile.am | 28 +- src/mca/mpool/sm/Makefile.am | 23 +- src/mca/ns/ns.h | 7 +- src/mca/oob/base/Makefile.am | 8 +- src/mca/oob/base/base.h | 76 ++-- src/mca/oob/base/oob_base_close.c | 5 +- src/mca/oob/base/oob_base_init.c | 85 +++++ src/mca/oob/base/oob_base_open.c | 10 +- src/mca/oob/base/oob_base_pack.c | 54 +-- src/mca/oob/base/oob_base_recv.c | 94 +++++ src/mca/oob/base/oob_base_recv_nb.c | 131 +++++++ src/mca/oob/base/oob_base_select.c | 127 ------- src/mca/oob/base/oob_base_send.c | 82 +++++ src/mca/oob/base/oob_base_send_nb.c | 124 +++++++ src/mca/oob/cofs/Makefile.am | 29 +- src/mca/oob/cofs/src/oob_cofs.c | 175 ++++----- src/mca/oob/cofs/src/oob_cofs.h | 98 ++++- src/mca/oob/cofs/src/oob_cofs_module.c | 67 ++-- src/mca/oob/oob.h | 406 +++++++++++++++++---- src/mca/oob/tcp/Makefile.am | 31 ++ src/mca/oob/tcp/VERSION | 6 + src/mca/oob/tcp/configure.params | 9 + src/mca/oob/tcp/oob_tcp.c | 73 ++++ src/mca/oob/tcp/oob_tcp.h | 109 ++++++ src/mca/oob/tcp/oob_tcp_message.h | 31 ++ src/mca/oob/tcp/oob_tcp_peer.c | 26 ++ src/mca/oob/tcp/oob_tcp_peer.h | 86 +++++ src/mca/oob/tcp/oob_tcp_recv.c | 39 ++ src/mca/oob/tcp/oob_tcp_send.c | 36 ++ src/mca/pcm/cofs/Makefile.am | 29 +- src/mca/pcm/cofs/src/pcm_cofs.c | 178 +-------- src/mca/pcm/cofs/src/pcm_cofs.h | 43 +-- src/mca/pcm/cofs/src/pcm_cofs_module.c | 165 ++++----- src/mca/pcm/pcm.h | 344 +---------------- src/mca/pcm/rsh/Makefile.am | 29 +- src/mca/pcm/rsh/src/pcm_rsh.c | 156 -------- src/mca/pcm/rsh/src/pcm_rsh.h | 58 --- src/mca/pcm/rsh/src/pcm_rsh_module.c | 59 --- src/mca/pml/teg/Makefile.am | 28 +- src/mca/ptl/elan/Makefile.am | 33 +- src/mca/ptl/gm/Makefile.am | 28 +- src/mca/ptl/gm/configure.stub | 156 ++------ src/mca/ptl/self/Makefile.am | 28 +- src/mca/ptl/sm/Makefile.am | 28 +- src/mca/ptl/tcp/Makefile.am | 28 +- src/mca/ptl/tcp/src/ptl_tcp_module.c | 32 +- src/mca/ptl/tcp/src/ptl_tcp_peer.c | 26 +- src/mca/ptl/tcp/src/ptl_tcp_proc.c | 20 +- src/mca/ptl/tcp/src/ptl_tcp_proc.h | 12 +- src/mca/topo/unity/Makefile.am | 29 +- src/proc/proc.c | 55 +-- src/proc/proc.h | 14 +- src/runtime/ompi_mpi_init.c | 2 + src/runtime/ompi_rte_init.c | 3 +- src/tools/mpirun/mpiboot | 25 +- src/tools/mpirun/mpirun | 6 +- src/tools/ompi_info/modules.cc | 2 +- src/util/proc_info.c | 6 - 81 files changed, 2065 insertions(+), 1940 deletions(-) create mode 100644 src/mca/oob/base/oob_base_init.c create mode 100644 src/mca/oob/base/oob_base_recv.c create mode 100644 src/mca/oob/base/oob_base_recv_nb.c delete mode 100644 src/mca/oob/base/oob_base_select.c create mode 100644 src/mca/oob/base/oob_base_send.c create mode 100644 src/mca/oob/base/oob_base_send_nb.c create mode 100644 src/mca/oob/tcp/Makefile.am create mode 100644 src/mca/oob/tcp/VERSION create mode 100644 src/mca/oob/tcp/configure.params create mode 100644 src/mca/oob/tcp/oob_tcp.c create mode 100644 src/mca/oob/tcp/oob_tcp.h create mode 100644 src/mca/oob/tcp/oob_tcp_message.h create mode 100644 src/mca/oob/tcp/oob_tcp_peer.c create mode 100644 src/mca/oob/tcp/oob_tcp_peer.h create mode 100644 src/mca/oob/tcp/oob_tcp_recv.c create mode 100644 src/mca/oob/tcp/oob_tcp_send.c diff --git a/autogen.sh b/autogen.sh index 185e115ae0..3c4cd8575b 100755 --- a/autogen.sh +++ b/autogen.sh @@ -541,8 +541,8 @@ AC_DEFINE_UNQUOTED(MCA_${pd_module_type}_${pd_module_name}_FULL_VERSION, EOF cat >> $pd_amc_file <cr, # crompi->cr). @@ -138,7 +125,7 @@ for type in $found_types; do # Remove any possible sym link in the mca-dynamic tree - rm -f src/dynamic-mca/$type/$m + rm -f src/mca/dyanmic/$type/$m # Now process the module @@ -152,12 +139,12 @@ for type in $found_types; do # generated the AM_CONDITIONAL directly. Here, we fill in # the variable that is used in that AM_CONDITIONAL. - if test "$compile_mode" = "dso"; then + if test "$compile_mode" = "dynamic"; then value=1 else value=0 fi - foo="BUILD_${type}_${m}_DSO=$value" + foo="BUILD_${type}_${m}_LOADABLE_MODULE=$value" eval $foo fi done @@ -179,7 +166,7 @@ for type in $found_types; do # Remove any possible sym link in the mca-dynamic tree - rm -f src/dyanmic-mca/$type/$m + rm -f src/mca/dyanmic/$type/$m # Configure the module subdirectory @@ -200,9 +187,9 @@ for type in $found_types; do all_modules="`echo $all_modules`" static_modules="`sort $outfile.static`" static_modules="`echo $static_modules`" - dso_modules="`sort $outfile.dso`" - dso_modules="`echo $dso_modules`" - rm -f $outfile $outfile.all $outfile.static $outfile.dso + dynamic_modules="`sort $outfile.dynamic`" + dynamic_modules="`echo $dynamic_modules`" + rm -f $outfile $outfile.all $outfile.static $outfile.dynamic # Create the final .h file that will be included in the type's # top-level glue. This lists all the static modules. @@ -229,13 +216,13 @@ EOF eval "$foo" foo="MCA_${type}_STATIC_SUBDIRS"'="$static_modules"' eval "$foo" - foo="MCA_${type}_DSO_SUBDIRS"'="$dso_modules"' + foo="MCA_${type}_DYNAMIC_SUBDIRS"'="$dynamic_modules"' eval "$foo" foo="MCA_${type}_STATIC_LTLIBS"'="$static_ltlibs"' eval "$foo" done unset foo type m modules structs outfile outdir total_dir file \ - all_modules static_modules dso_modules static_ltlibs + all_modules static_modules dynamic_modules static_ltlibs # Grumble. It seems that AC_SUBST and AC_DEFINE don't let you # substitue on a variable name that contains a variable (e.g., @@ -245,66 +232,66 @@ unset foo type m modules structs outfile outdir total_dir file \ AC_SUBST(MCA_common_ALL_SUBDIRS) AC_SUBST(MCA_common_STATIC_SUBDIRS) -AC_SUBST(MCA_common_DSO_SUBDIRS) +AC_SUBST(MCA_common_DYNAMIC_SUBDIRS) AC_SUBST(MCA_common_STATIC_LTLIBS) # OMPI types AC_SUBST(MCA_oob_ALL_SUBDIRS) AC_SUBST(MCA_oob_STATIC_SUBDIRS) -AC_SUBST(MCA_oob_DSO_SUBDIRS) +AC_SUBST(MCA_oob_DYNAMIC_SUBDIRS) AC_SUBST(MCA_oob_STATIC_LTLIBS) AC_SUBST(MCA_pcm_ALL_SUBDIRS) AC_SUBST(MCA_pcm_STATIC_SUBDIRS) -AC_SUBST(MCA_pcm_DSO_SUBDIRS) +AC_SUBST(MCA_pcm_DYNAMIC_SUBDIRS) AC_SUBST(MCA_pcm_STATIC_LTLIBS) AC_SUBST(MCA_gpr_ALL_SUBDIRS) AC_SUBST(MCA_gpr_STATIC_SUBDIRS) -AC_SUBST(MCA_gpr_DSO_SUBDIRS) +AC_SUBST(MCA_gpr_DYNAMIC_SUBDIRS) AC_SUBST(MCA_gpr_STATIC_LTLIBS) # MPI types AC_SUBST(MCA_allocator_ALL_SUBDIRS) AC_SUBST(MCA_allocator_STATIC_SUBDIRS) -AC_SUBST(MCA_allocator_DSO_SUBDIRS) +AC_SUBST(MCA_allocator_DYNAMIC_SUBDIRS) AC_SUBST(MCA_allocator_STATIC_LTLIBS) AC_SUBST(MCA_coll_ALL_SUBDIRS) AC_SUBST(MCA_coll_STATIC_SUBDIRS) -AC_SUBST(MCA_coll_DSO_SUBDIRS) +AC_SUBST(MCA_coll_DYNAMIC_SUBDIRS) AC_SUBST(MCA_coll_STATIC_LTLIBS) AC_SUBST(MCA_io_ALL_SUBDIRS) AC_SUBST(MCA_io_STATIC_SUBDIRS) -AC_SUBST(MCA_io_DSO_SUBDIRS) +AC_SUBST(MCA_io_DYNAMIC_SUBDIRS) AC_SUBST(MCA_io_STATIC_LTLIBS) AC_SUBST(MCA_mpool_ALL_SUBDIRS) AC_SUBST(MCA_mpool_STATIC_SUBDIRS) -AC_SUBST(MCA_mpool_DSO_SUBDIRS) +AC_SUBST(MCA_mpool_DYNAMIC_SUBDIRS) AC_SUBST(MCA_mpool_STATIC_LTLIBS) AC_SUBST(MCA_one_ALL_SUBDIRS) AC_SUBST(MCA_one_STATIC_SUBDIRS) -AC_SUBST(MCA_one_DSO_SUBDIRS) +AC_SUBST(MCA_one_DYNAMIC_SUBDIRS) AC_SUBST(MCA_one_STATIC_LTLIBS) AC_SUBST(MCA_pml_ALL_SUBDIRS) AC_SUBST(MCA_pml_STATIC_SUBDIRS) -AC_SUBST(MCA_pml_DSO_SUBDIRS) +AC_SUBST(MCA_pml_DYNAMIC_SUBDIRS) AC_SUBST(MCA_pml_STATIC_LTLIBS) AC_SUBST(MCA_ptl_ALL_SUBDIRS) AC_SUBST(MCA_ptl_STATIC_SUBDIRS) -AC_SUBST(MCA_ptl_DSO_SUBDIRS) +AC_SUBST(MCA_ptl_DYNAMIC_SUBDIRS) AC_SUBST(MCA_ptl_STATIC_LTLIBS) AC_SUBST(MCA_topo_ALL_SUBDIRS) AC_SUBST(MCA_topo_STATIC_SUBDIRS) -AC_SUBST(MCA_topo_DSO_SUBDIRS) +AC_SUBST(MCA_topo_DYNAMIC_SUBDIRS) AC_SUBST(MCA_topo_STATIC_LTLIBS) # Finally, now that we've filled in all the test variables, get all @@ -340,24 +327,24 @@ if test "$HAPPY" = "1"; then # Is this module going to built staic or shared? - str="SHARED_TYPE=\$DSO_$type" + str="SHARED_TYPE=\$LOADABLE_MODULE_$type" eval $str - str="SHARED_GENERIC_TYPE=\$DSO_$generic_type" + str="SHARED_GENERIC_TYPE=\$LOADABLE_MODULE_$generic_type" eval $str - str="SHARED_MODULE=\$DSO_${type}_$m" + str="SHARED_MODULE=\$LOADABLE_MODULE_${type}_$m" eval $str shared_mode_override=static # Setup for either shared or static - if test "$shared_mode_override" = "dso" -o \ + if test "$shared_mode_override" = "dynamic" -o \ "$SHARED_TYPE" = "1" -o \ "$SHARED_GENERIC_TYPE" = "1" -o \ "$SHARED_MODULE" = "1" -o \ - "$DSO_all" = "1"; then - compile_mode="dso" - echo $m >> $outfile.dso + "$LOADABLE_MODULE_all" = "1"; then + compile_mode="dynamic" + echo $m >> $outfile.dynamic rm -f "src/dynamic-mca/$type/$m" $LN_S "$OMPI_TOP_BUILDDIR/src/mca/$type/$m" \ "src/dynamic-mca/$type/$m" diff --git a/configure.ac b/configure.ac index 9973d0c31f..864636abf3 100644 --- a/configure.ac +++ b/configure.ac @@ -118,8 +118,8 @@ OMPI_CONFIGURE_OPTIONS ompi_enable_shared="$enable_shared" ompi_enable_static="$enable_static" -AM_ENABLE_SHARED -AM_DISABLE_STATIC +AM_DISABLE_SHARED +AM_ENABLE_STATIC ############################################################################ diff --git a/src/communicator/comm.c b/src/communicator/comm.c index ef4eecec85..06b14d1f77 100644 --- a/src/communicator/comm.c +++ b/src/communicator/comm.c @@ -608,9 +608,10 @@ ompi_proc_t **ompi_comm_get_rprocs ( ompi_communicator_t *local_comm, int tag, int rsize) { + ompi_proc_t **rprocs = NULL; +#if 0 /* TSW - fix this */ int local_rank, local_size; - ompi_proc_t **rprocs; - ompi_job_handle_t job; + ompi_process_id_t jobid; uint32_t *rvpids=NULL, *vpids=NULL; int rc, i; @@ -629,7 +630,7 @@ ompi_proc_t **ompi_comm_get_rprocs ( ompi_communicator_t *local_comm, MPI_Status status; /* generate vpid list */ for ( i = 0; i < local_size; i++ ){ - vpids[i] = (uint32_t) local_comm->c_local_group->grp_proc_pointers[i]->proc_vpid; + vpids[i] = (uint32_t) local_comm->c_local_group->grp_proc_pointers[i]->proc_name; } @@ -656,11 +657,6 @@ ompi_proc_t **ompi_comm_get_rprocs ( ompi_communicator_t *local_comm, goto err_exit; } - /* determine according proc-list */ - if(NULL == (job = mca_pcm.pcm_handle_get())) { - return NULL; - } - for ( i = 0; i < rsize; i++ ) { rprocs[i] = ompi_proc_find ( job, rvpids[i] ); } @@ -674,6 +670,7 @@ ompi_proc_t **ompi_comm_get_rprocs ( ompi_communicator_t *local_comm, } /* rprocs has to be freed in the level above (i.e. intercomm_create ) */ +#endif return rprocs; } /**********************************************************************/ @@ -692,7 +689,7 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm, int high ) /* * determine maximal high value over the intercomm */ - if ( lvpid->proc_vpid > rvpid->proc_vpid ) { + if ( lvpid->proc_name.procid > rvpid->proc_name.procid ) { if ( 0 == local_rank ) { rc = intercomm->c_coll.coll_bcast(&high, 1, MPI_INT, MPI_ROOT, intercomm ); diff --git a/src/communicator/comm_init.c b/src/communicator/comm_init.c index 98f5afc207..2fa888453d 100644 --- a/src/communicator/comm_init.c +++ b/src/communicator/comm_init.c @@ -51,7 +51,7 @@ int ompi_comm_init(void) OBJ_CONSTRUCT(&ompi_mpi_comm_world, ompi_communicator_t); group = OBJ_NEW(ompi_group_t); group->grp_proc_pointers = ompi_proc_world(&size); - group->grp_my_rank = ompi_proc_local()->proc_vpid ; + group->grp_my_rank = ompi_proc_local()->proc_name.procid; group->grp_proc_count = size; group->grp_flags |= OMPI_GROUP_INTRINSIC; OBJ_RETAIN(group); /* bump reference count for remote reference */ diff --git a/src/dynamic-mca/allocator/Makefile.am b/src/dynamic-mca/allocator/Makefile.am index 2a01cd52e3..b409415bb1 100644 --- a/src/dynamic-mca/allocator/Makefile.am +++ b/src/dynamic-mca/allocator/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_allocator_DSO_SUBDIRS) +SUBDIRS = $(MCA_allocator_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/coll/Makefile.am b/src/dynamic-mca/coll/Makefile.am index 86d95cce0d..d31bcadf60 100644 --- a/src/dynamic-mca/coll/Makefile.am +++ b/src/dynamic-mca/coll/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_coll_DSO_SUBDIRS) +SUBDIRS = $(MCA_coll_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/common/Makefile.am b/src/dynamic-mca/common/Makefile.am index 74caa78778..ce3b490e32 100644 --- a/src/dynamic-mca/common/Makefile.am +++ b/src/dynamic-mca/common/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_common_DSO_SUBDIRS) +SUBDIRS = $(MCA_common_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/gpr/Makefile.am b/src/dynamic-mca/gpr/Makefile.am index fbaf88e981..bafae94546 100644 --- a/src/dynamic-mca/gpr/Makefile.am +++ b/src/dynamic-mca/gpr/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_gpr_DSO_SUBDIRS) +SUBDIRS = $(MCA_gpr_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/io/Makefile.am b/src/dynamic-mca/io/Makefile.am index f6100496e9..1c96800371 100644 --- a/src/dynamic-mca/io/Makefile.am +++ b/src/dynamic-mca/io/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_io_DSO_SUBDIRS) +SUBDIRS = $(MCA_io_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/mpool/Makefile.am b/src/dynamic-mca/mpool/Makefile.am index 18da179d29..7c0f2ed53f 100644 --- a/src/dynamic-mca/mpool/Makefile.am +++ b/src/dynamic-mca/mpool/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_mpool_DSO_SUBDIRS) +SUBDIRS = $(MCA_mpool_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/one/Makefile.am b/src/dynamic-mca/one/Makefile.am index a6a0498421..59aecebf14 100644 --- a/src/dynamic-mca/one/Makefile.am +++ b/src/dynamic-mca/one/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_one_DSO_SUBDIRS) +SUBDIRS = $(MCA_one_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/oob/Makefile.am b/src/dynamic-mca/oob/Makefile.am index acf45eaa17..c522e110ac 100644 --- a/src/dynamic-mca/oob/Makefile.am +++ b/src/dynamic-mca/oob/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_oob_DSO_SUBDIRS) +SUBDIRS = $(MCA_oob_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/pcm/Makefile.am b/src/dynamic-mca/pcm/Makefile.am index 3d7837dadf..3e7884d3f6 100644 --- a/src/dynamic-mca/pcm/Makefile.am +++ b/src/dynamic-mca/pcm/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_pcm_DSO_SUBDIRS) +SUBDIRS = $(MCA_pcm_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/pml/Makefile.am b/src/dynamic-mca/pml/Makefile.am index 6c806c9632..74c2025e15 100644 --- a/src/dynamic-mca/pml/Makefile.am +++ b/src/dynamic-mca/pml/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_pml_DSO_SUBDIRS) +SUBDIRS = $(MCA_pml_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/ptl/Makefile.am b/src/dynamic-mca/ptl/Makefile.am index def3ba9f85..03a795c468 100644 --- a/src/dynamic-mca/ptl/Makefile.am +++ b/src/dynamic-mca/ptl/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_ptl_DSO_SUBDIRS) +SUBDIRS = $(MCA_ptl_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/dynamic-mca/topo/Makefile.am b/src/dynamic-mca/topo/Makefile.am index 1020bda9c3..793c2116ac 100644 --- a/src/dynamic-mca/topo/Makefile.am +++ b/src/dynamic-mca/topo/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/config/Makefile.options -SUBDIRS = $(MCA_topo_DSO_SUBDIRS) +SUBDIRS = $(MCA_topo_DYNAMIC_SUBDIRS) DISTCLEANFILES = $(SUBDIRS) # Every directory under here is a sym link to something in the main diff --git a/src/mca/allocator/bucket/Makefile.am b/src/mca/allocator/bucket/Makefile.am index ab1eba3cf9..e0d8aeb6da 100644 --- a/src/mca/allocator/bucket/Makefile.am +++ b/src/mca/allocator/bucket/Makefile.am @@ -1,9 +1,8 @@ +# -*- makefile -*- # # $HEADER$ # -# Use the top-level Makefile.options - include $(top_ompi_srcdir)/config/Makefile.options sources = \ @@ -11,25 +10,21 @@ sources = \ allocator_bucket_alloc.c \ allocator_bucket_alloc.h -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if OMPI_BUILD_allocator_bucket_DSO -component_noinst = -component_install = mca_allocator_bucket.la +if OMPI_BUILD_allocator_bucket_LOADABLE_MODULE +module_noinst = +module_install = mca_allocator_bucket.la else -component_noinst = libmca_allocator_bucket.la -component_install = +module_noinst = libmca_allocator_bucket.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_allocator_bucket_la_SOURCES = $(sources) mca_allocator_bucket_la_LIBADD = $(LIBOMPI_LA) mca_allocator_bucket_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_allocator_bucket_la_SOURCES = $(sources) libmca_allocator_bucket_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/base/mca_base_module_exchange.c b/src/mca/base/mca_base_module_exchange.c index a99ca75233..3abe3cd850 100644 --- a/src/mca/base/mca_base_module_exchange.c +++ b/src/mca/base/mca_base_module_exchange.c @@ -228,18 +228,16 @@ int mca_base_modex_exchange(void) size_t i; for(i=0; iproc_job, - proc->proc_vpid, - 0, - self_module->module_data, - self_module->module_data_size); - if(rc != OMPI_SUCCESS) { + iov.iov_base = self_module->module_data; + iov.iov_len = self_module->module_data_size; + rc = mca_oob_send(&proc->proc_name, &iov, 1, 0); + if(rc != iov.iov_len) { free(procs); OMPI_THREAD_UNLOCK(&self->proc_lock); return rc; @@ -255,8 +253,9 @@ int mca_base_modex_exchange(void) for(i=0; iproc_job, - proc->proc_vpid, - &tag, - &proc_module->module_data, - &proc_module->module_data_size); - if(rc != OMPI_SUCCESS) { + size = mca_oob_recv(&proc->proc_name, 0, 0, MCA_OOB_TRUNC|MCA_OOB_PEEK); + if(size <= 0) { + free(procs); + OMPI_THREAD_UNLOCK(&proc->proc_lock); + OMPI_THREAD_UNLOCK(&self->proc_lock); + return rc; + } + + proc_module->module_data = malloc(size); + proc_module->module_data_size = size; + iov.iov_base = proc_module->module_data; + iov.iov_len = size; + + rc = mca_oob_recv(&proc->proc_name, &iov, 1, 0); + if(rc != size) { free(procs); OMPI_THREAD_UNLOCK(&proc->proc_lock); OMPI_THREAD_UNLOCK(&self->proc_lock); diff --git a/src/mca/coll/base/coll_base_close.c b/src/mca/coll/base/coll_base_close.c index 7ccfc39eb9..0fbb8cd986 100644 --- a/src/mca/coll/base/coll_base_close.c +++ b/src/mca/coll/base/coll_base_close.c @@ -6,7 +6,6 @@ #include -#include "util/output.h" #include "include/constants.h" #include "mca/mca.h" #include "mca/base/base.h" @@ -30,9 +29,6 @@ int mca_coll_base_close(void) mca_coll_base_components_available_valid = false; } - /* Close the output stream for this framework */ - ompi_output_close(mca_coll_base_output); - /* All done */ return OMPI_SUCCESS; diff --git a/src/mca/coll/basic/Makefile.am b/src/mca/coll/basic/Makefile.am index a2ee539d08..2f1b9f5990 100644 --- a/src/mca/coll/basic/Makefile.am +++ b/src/mca/coll/basic/Makefile.am @@ -1,8 +1,9 @@ +# -*- makefile -*- # # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +11,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_coll_basic_DSO -component_noinst = -component_install = mca_coll_basic.la +if OMPI_BUILD_coll_basic_LOADABLE_MODULE +module_noinst = +module_install = mca_coll_basic.la else -component_noinst = libmca_coll_basic.la -component_install = +module_noinst = libmca_coll_basic.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_coll_basic_la_SOURCES = -mca_coll_basic_la_LIBADD = src/libmca_coll_basic.la $(LIBMPI_LA) +mca_coll_basic_la_LIBADD = \ + src/libmca_coll_basic.la \ + $(LIBMPI_LA) mca_coll_basic_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_coll_basic_la_SOURCES = libmca_coll_basic_la_LIBADD = src/libmca_coll_basic.la libmca_coll_basic_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/coll/basic/src/coll_basic_reduce.c b/src/mca/coll/basic/src/coll_basic_reduce.c index 6d26430667..ee42149153 100644 --- a/src/mca/coll/basic/src/coll_basic_reduce.c +++ b/src/mca/coll/basic/src/coll_basic_reduce.c @@ -263,13 +263,6 @@ int mca_coll_basic_reduce_log_intra(void *sbuf, void *rbuf, int count, void *inmsg; void *resmsg; - /* Some variables */ - - size = ompi_comm_size(comm); - rank = ompi_comm_rank(comm); - vrank = ompi_op_is_commute(op) ? (rank - root + size) % size : rank; - dim = comm->c_cube_dim; - /* Allocate the incoming and resulting message buffers. See lengthy rationale above. */ @@ -286,6 +279,13 @@ int mca_coll_basic_reduce_log_intra(void *sbuf, void *rbuf, int count, pml_buffer2 = free_buffer - lb; } + /* Some variables */ + + size = ompi_comm_size(comm); + rank = ompi_comm_rank(comm); + vrank = ompi_op_is_commute(op) ? (rank - root + size) % size : rank; + dim = comm->c_cube_dim; + /* Loop over cube dimensions. High processes send to low ones in the dimension. */ @@ -299,17 +299,17 @@ int mca_coll_basic_reduce_log_intra(void *sbuf, void *rbuf, int count, if (vrank & mask) { peer = vrank & ~mask; if (ompi_op_is_commute(op)) { - peer = (peer + root) % size; + peer = (peer + root) % size; } err = mca_pml.pml_send((fl_recv) ? resmsg : sbuf, count, dtype, peer, MCA_COLL_BASE_TAG_REDUCE, MCA_PML_BASE_SEND_STANDARD, comm); if (MPI_SUCCESS != err) { - if (NULL != free_buffer) { - free(free_buffer); - } - return err; + if (NULL != free_buffer) { + free(free_buffer); + } + return err; } break; @@ -321,10 +321,10 @@ int mca_coll_basic_reduce_log_intra(void *sbuf, void *rbuf, int count, else { peer = vrank | mask; if (peer >= size) { - continue; + continue; } if (ompi_op_is_commute(op)) { - peer = (peer + root) % size; + peer = (peer + root) % size; } fl_recv = 1; @@ -332,10 +332,10 @@ int mca_coll_basic_reduce_log_intra(void *sbuf, void *rbuf, int count, MCA_COLL_BASE_TAG_REDUCE, comm, MPI_STATUS_IGNORE); if (MPI_SUCCESS != err) { - if (NULL != free_buffer) { - free(free_buffer); - } - return err; + if (NULL != free_buffer) { + free(free_buffer); + } + return err; } /* Perform the operation */ @@ -343,11 +343,11 @@ int mca_coll_basic_reduce_log_intra(void *sbuf, void *rbuf, int count, ompi_op_reduce(op, (i > 0) ? resmsg : sbuf, inmsg, count, dtype); if (inmsg == pml_buffer1) { - resmsg = pml_buffer1; - inmsg = pml_buffer2; + resmsg = pml_buffer1; + inmsg = pml_buffer2; } else { - resmsg = pml_buffer2; - inmsg = pml_buffer1; + resmsg = pml_buffer2; + inmsg = pml_buffer1; } } } diff --git a/src/mca/io/romio/Makefile.am b/src/mca/io/romio/Makefile.am index ebd3ff42bd..5819f3f644 100644 --- a/src/mca/io/romio/Makefile.am +++ b/src/mca/io/romio/Makefile.am @@ -2,7 +2,7 @@ # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +10,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_io_romio_DSO -component_noinst = -component_install = mca_io_romio.la +if OMPI_BUILD_io_romio_LOADABLE_MODULE +module_noinst = +module_install = mca_io_romio.la else -component_noinst = libmca_mpi_io_romio.la -component_install = +module_noinst = libmca_mpi_io_romio.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_io_romio_la_SOURCES = -mca_io_romio_la_LIBADD = src/libmca_io_romio.la $(LIBOMPI_LA) +mca_io_romio_la_LIBADD = \ + src/libmca_io_romio.la \ + $(LIBOMPI_LA) mca_io_romio_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_mpi_io_romio_la_SOURCES = libmca_mpi_io_romio_la_LIBADD = src/libmca_io_romio.la libmca_mpi_io_romio_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/mpool/sm/Makefile.am b/src/mca/mpool/sm/Makefile.am index 1814be7be5..191dcbde15 100644 --- a/src/mca/mpool/sm/Makefile.am +++ b/src/mca/mpool/sm/Makefile.am @@ -1,9 +1,8 @@ +# -*- makefile -*- # # $HEADER$ # -# Use the top-level Makefile.options - include $(top_ompi_srcdir)/config/Makefile.options sources = \ @@ -13,24 +12,20 @@ sources = \ mpool_sm_mmap.h \ mpool_sm_component.c -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if OMPI_BUILD_mpool_sm_DSO -component_noinst = -component_install = mca_mpool_sm.la +if OMPI_BUILD_mpool_sm_LOADABLE_MODULE +module_noinst = +module_install = mca_mpool_sm.la else -component_noinst = libmca_mpool_sm.la -component_install = +module_noinst = libmca_mpool_sm.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_mpool_sm_la_SOURCES = $(sources) mca_mpool_sm_la_LIBADD = $(LIBOMPI_LA) mca_mpool_sm_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_mpool_sm_la_SOURCES = $(sources) libmca_mpool_sm_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/ns/ns.h b/src/mca/ns/ns.h index 88d61cac70..f329bd1320 100644 --- a/src/mca/ns/ns.h +++ b/src/mca/ns/ns.h @@ -11,6 +11,9 @@ * */ +#ifndef MCA_NS_H +#define MCA_NS_H + /* * includes */ @@ -26,8 +29,6 @@ typedef uint32_t ompi_process_id_t; /**< Set the allowed range for id's in each space */ struct ompi_process_name_t { - bool daemon; /**< Boolean indicating if process is a daemon (true) or application process (false) */ - char *name; /**< String representation of the process name, expressed in %x.%x.%x format */ ompi_process_id_t cellid; /**< Cell number */ ompi_process_id_t jobid; /**< Job number */ ompi_process_id_t procid; /**< Process number */ @@ -128,3 +129,5 @@ char *ompi_get_cellid_string(ompi_process_name_t *name); ompi_process_id_t ompi_get_cellid(ompi_process_name_t *name); +#endif + diff --git a/src/mca/oob/base/Makefile.am b/src/mca/oob/base/Makefile.am index cd7dbf89d3..579607ba41 100644 --- a/src/mca/oob/base/Makefile.am +++ b/src/mca/oob/base/Makefile.am @@ -21,8 +21,12 @@ libmca_oob_base_la_SOURCES = \ $(headers) \ oob_base_close.c \ oob_base_open.c \ - oob_base_select.c \ - oob_base_pack.c + oob_base_init.c \ + oob_base_send.c \ + oob_base_send_nb.c \ + oob_base_recv.c \ + oob_base_recv_nb.c \ + oob_base_pack.c # Conditionally install the header files diff --git a/src/mca/oob/base/base.h b/src/mca/oob/base/base.h index 0923efc1d8..4b5022c2e2 100644 --- a/src/mca/oob/base/base.h +++ b/src/mca/oob/base/base.h @@ -1,7 +1,10 @@ -/* -*- C -*- - * +/* * $HEADER$ */ +/** @file: + * + * Contains packing functions provided by the oob + */ #ifndef MCA_OOB_BASE_H_ #define MCA_OOB_BASE_H_ @@ -12,16 +15,27 @@ #include "mca/mca.h" #include "mca/oob/oob.h" -/** - * the different types one can use to describe the data +/* + * This is the first module on the list. This is here temporarily + * to make things work */ -typedef enum { - MCA_OOB_BASE_BYTE, /**< a byte of data */ - MCA_OOB_BASE_INT16, /**< a 16 bit integer */ - MCA_OOB_BASE_INT32, /**< a 32 bit integer */ - MCA_OOB_BASE_PACKED,/**< already packed data. */ - MCA_OOB_BASE_BYTE_BY_REF /**< indicates to try to use the data directly without copying */ -} mca_oob_base_types_t; +mca_oob_t mca_oob; + +/** + * the module data structure + */ +struct mca_oob_base_module_t { + ompi_list_item_t super; + mca_oob_base_component_t *oob_component; + mca_oob_t *oob_module; +}; +typedef struct mca_oob_base_module_t mca_oob_base_module_t; + +/** + * declare the module structure as a class + */ +OBJ_CLASS_DECLARATION(mca_oob_base_module_t); + /* * Global functions for MCA overall collective open and close @@ -30,16 +44,9 @@ typedef enum { extern "C" { #endif int mca_oob_base_open(void); - int mca_oob_base_select(bool *allow_multi_user_threads, - bool *have_hidden_threads); + int mca_oob_base_init(bool *allow_multi_user_threads, bool *have_hidden_threads); int mca_oob_base_close(void); - bool mca_oob_base_is_checkpointable(void); - - int mca_oob_base_checkpoint(void); - int mca_oob_base_continue(void); - int mca_oob_base_restart(void); - /* * functions for pack and unpack routines */ @@ -54,7 +61,7 @@ extern "C" { * @retval OMPI_SUCCESS * @retval OMPI_ERROR */ - int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_types_t type); + int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_type_t type); /** * This function unpacks the passed data according to the type enum. @@ -67,30 +74,8 @@ extern "C" { * @retval OMPI_SUCCESS * @retval OMPI_ERROR */ - int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_types_t type); + int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_type_t type); -/** - * This function packs null terminated strings - * - * @param dest the destination for the packed information - * @param src the source of the information - * @param maxlen the maximum length available in dest. the string will - * be truncated and a NULL added if it is longer then maxlen. - * - * @retval OMPI_SUCCESS - */ - int mca_oob_base_pack_string(void * dest, void * src, size_t maxlen); -/** - * This function unpacks strings - * - * @param dest the destination for the packed information - * @param src the source of the information - * @param maxlen the maximum length available in dest. the string will - * be truncated and a NULL added if it is longer then maxlen. - * - * @retval OMPI_SUCCESS - */ - int mca_oob_base_unpack_string(void * dest, void * src, size_t maxlen); #if defined(c_plusplus) || defined(__cplusplus) } @@ -101,8 +86,7 @@ extern "C" { * Global struct holding the selected module's function pointers */ extern int mca_oob_base_output; -extern ompi_list_t mca_oob_base_modules_available; -extern mca_oob_base_module_t mca_oob_base_selected_module; -extern mca_oob_t mca_oob; +extern ompi_list_t mca_oob_base_components; +extern ompi_list_t mca_oob_base_modules; #endif diff --git a/src/mca/oob/base/oob_base_close.c b/src/mca/oob/base/oob_base_close.c index 140f0f1b5d..3ff43ac4df 100644 --- a/src/mca/oob/base/oob_base_close.c +++ b/src/mca/oob/base/oob_base_close.c @@ -18,9 +18,8 @@ int mca_oob_base_close(void) /* Close all remaining available modules (may be one if this is a OMPI RTE program, or [possibly] multiple if this is ompi_info) */ - mca_base_modules_close(mca_oob_base_output, - &mca_oob_base_modules_available, NULL); - + OBJ_DESTRUCT(&mca_oob_base_modules); + mca_base_modules_close(mca_oob_base_output, &mca_oob_base_components, NULL); /* All done */ return OMPI_SUCCESS; diff --git a/src/mca/oob/base/oob_base_init.c b/src/mca/oob/base/oob_base_init.c new file mode 100644 index 0000000000..dc4b5fd343 --- /dev/null +++ b/src/mca/oob/base/oob_base_init.c @@ -0,0 +1,85 @@ +/* + * $HEADER$ + */ + +#include "ompi_config.h" + +#include + +#include "runtime/runtime.h" +#include "util/output.h" +#include "mca/mca.h" +#include "mca/base/base.h" +#include "mca/pcm/pcm.h" +#include "mca/oob/oob.h" +#include "mca/oob/base/base.h" + + +OBJ_CLASS_INSTANCE( + mca_oob_base_module_t, + ompi_list_item_t, + NULL, + NULL +); + +ompi_process_name_t mca_oob_base_self; +ompi_process_name_t mca_oob_base_any; + +/** + * Function for selecting one module from all those that are + * available. + * + * Call the init function on all available modules. + */ +int mca_oob_base_init(bool *user_threads, bool *hidden_threads) +{ + ompi_list_item_t *item; + mca_base_module_list_item_t *mli; + mca_oob_base_module_t * first; + mca_oob_base_component_t *component; + mca_oob_t *module; + extern ompi_list_t mca_oob_base_modules; + ompi_process_name_t *self; + + /* setup local name */ + self = mca_pcm.pcm_self(); + if(NULL == self) { + return OMPI_ERROR; + } + mca_oob_base_self = *self; + + /* setup wildcard name */ + mca_oob_base_any.cellid = -1; + mca_oob_base_any.jobid = -1; + mca_oob_base_any.procid = -1; + + /* Traverse the list of available modules; call their init functions. */ + for (item = ompi_list_get_first(&mca_oob_base_components); + item != ompi_list_get_end(&mca_oob_base_components); + item = ompi_list_get_next(item)) { + mca_oob_base_module_t *inited; + + mli = (mca_base_module_list_item_t *) item; + component = (mca_oob_base_component_t *) mli->mli_module; + + if (NULL == component->oob_init) { + ompi_output_verbose(10, mca_oob_base_output, "mca_oob_base_init: no init function; ignoring component"); + } else { + module = component->oob_init(user_threads, hidden_threads); + if (NULL == module) { + ompi_output_verbose(10, mca_oob_base_output, "mca_oob_base_init: oob_init returned failure"); + } + } + + inited = OBJ_NEW(mca_oob_base_module_t); + inited->oob_component = component; + inited->oob_module = module; + ompi_list_append(&mca_oob_base_modules, &inited->super); + } + /* set the global variable to point to the first initialize module */ + first = (mca_oob_base_module_t *) ompi_list_get_first(&mca_oob_base_modules); + if(NULL != first) + mca_oob = *first->oob_module; + return OMPI_SUCCESS; +} + diff --git a/src/mca/oob/base/oob_base_open.c b/src/mca/oob/base/oob_base_open.c index a92304b2b9..d874a02086 100644 --- a/src/mca/oob/base/oob_base_open.c +++ b/src/mca/oob/base/oob_base_open.c @@ -23,9 +23,8 @@ * Global variables */ int mca_oob_base_output = -1; -mca_oob_t mca_oob; -ompi_list_t mca_oob_base_modules_available; -mca_oob_base_module_t mca_oob_base_selected_module; +ompi_list_t mca_oob_base_components; +ompi_list_t mca_oob_base_modules; /** @@ -38,11 +37,12 @@ int mca_oob_base_open(void) if (OMPI_SUCCESS != mca_base_modules_open("oob", 0, mca_oob_base_static_modules, - &mca_oob_base_modules_available)) { + &mca_oob_base_components)) { return OMPI_ERROR; } /* All done */ - + OBJ_CONSTRUCT(&mca_oob_base_modules, ompi_list_t); return OMPI_SUCCESS; } + diff --git a/src/mca/oob/base/oob_base_pack.c b/src/mca/oob/base/oob_base_pack.c index 9880021635..0d512f7891 100644 --- a/src/mca/oob/base/oob_base_pack.c +++ b/src/mca/oob/base/oob_base_pack.c @@ -21,7 +21,7 @@ * @retval OMPI_SUCCESS * @retval OMPI_ERROR */ -int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_types_t type) +int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_type_t type) { int i; uint16_t * d16 = (uint16_t *) dest; @@ -32,7 +32,6 @@ int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_types_t ty switch(type) { case MCA_OOB_BASE_BYTE: case MCA_OOB_BASE_PACKED: - case MCA_OOB_BASE_BYTE_BY_REF: memcpy(dest, src, n); break; case MCA_OOB_BASE_INT16: @@ -47,13 +46,17 @@ int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_types_t ty d32[i] = htonl(s32[i]); } break; + case MCA_OOB_BASE_STRING: + strncpy(dest, src, n); + *((char *) dest + n - 1) = '\0'; + break; default: return OMPI_ERROR; } return OMPI_SUCCESS; } -/** +/* * This function unpacks the passed data according to the type enum. * * @param dest the destination for the unpacked data @@ -64,7 +67,7 @@ int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_types_t ty * @retval OMPI_SUCCESS * @retval OMPI_ERROR */ -int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_types_t type) +int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_type_t type) { int i; uint16_t * d16 = (uint16_t *) dest; @@ -75,7 +78,6 @@ int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_types_t switch(type) { case MCA_OOB_BASE_BYTE: case MCA_OOB_BASE_PACKED: - case MCA_OOB_BASE_BYTE_BY_REF: memcpy(dest, src, n); break; case MCA_OOB_BASE_INT16: @@ -90,47 +92,13 @@ int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_types_t d32[i] = ntohl(s32[i]); } break; + case MCA_OOB_BASE_STRING: + strncpy(dest, src, n); + *((char *) dest + n - 1) = '\0'; + break; default: return OMPI_ERROR; } return OMPI_SUCCESS; } -/* - * This function packs null terminated strings - * - * @param dest the destination for the packed information - * @param src the source of the information - * @param maxlen the maximum length available in dest. the string will - * be truncated and a NULL added if it is longer then maxlen. - * - * @retval OMPI_SUCCESS - */ -int mca_oob_base_pack_string(void * dest, void * src, size_t maxlen) -{ - strncpy(dest, src, maxlen); - /* add a null terminator at the end of dest since strncpy doesn't do it for us - * if the src is longer than then dest */ - *((char *) dest + maxlen - 1) = '\0'; - return OMPI_SUCCESS; -} - -/* - * This function unpacks strings - * - * @param dest the destination for the packed information - * @param src the source of the information - * @param maxlen the maximum length available in dest. the string will - * be truncated and a NULL added if it is longer then maxlen. - * - * @retval OMPI_SUCCESS - */ -int mca_oob_base_unpack_string(void * dest, void * src, size_t maxlen) -{ - strncpy(dest, src, maxlen); - /* add a null terminator at the end of dest since strncpy doesn't do it for us - * if the src is longer than then dest */ - *((char *) dest + maxlen - 1) = '\0'; - return OMPI_SUCCESS; -} - diff --git a/src/mca/oob/base/oob_base_recv.c b/src/mca/oob/base/oob_base_recv.c new file mode 100644 index 0000000000..46097ed5db --- /dev/null +++ b/src/mca/oob/base/oob_base_recv.c @@ -0,0 +1,94 @@ +#include "mca/oob/oob.h" +#include "mca/oob/base/base.h" +#include +#include + +/* +* Similiar to unix recv(2) +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the +* iovec array without removing the message from the queue. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +int mca_oob_recv(ompi_process_name_t* peer, const struct iovec *msg, int count, int flags) +{ + return(mca_oob.oob_recv(peer, msg, count, flags)); +} + +/* + * Receive data and convert (if required) to host byte order. + * + * @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param types (IN) Parallel array to iovecs describing data type of each iovec element. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the + * iovec array without removing the message from the queue. + * @return OMPI error code (<0) on error or number of bytes actually received. + */ + +int mca_oob_recv_ntoh(ompi_process_name_t* peer, const struct iovec *msg, + const mca_oob_base_type_t *types, int count, int flags) +{ + int rc, num, i = 0; + struct iovec * orig; + bool convert = false; + /* see if we actually have to convert anything */ + /* first check to see if we are already in network byte order */ + if(1 != htons(1)) { + /* if we aren't, see if there is any data types that need to be converted */ + while(!convert && (i < count)) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + convert = true; + } + i++; + } + } + + if(convert) { + /* now if we need to convert anything we neeg to create a new iovec + * to recieve into */ + orig = malloc(sizeof(struct iovec) * count); + /* copy their array into ours */ + memcpy(orig, msg, sizeof(struct iovec) * count); + /* now we need to go through the iovects, and any ints we need to + * allocate our own space to recieve into, so we can convert into + * their space later */ + for(i = 0; i < count; i++) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + orig[i].iov_base = malloc(orig[i].iov_len); + } + } + /* now the new buffers are ready. do the recieve */ + rc = mca_oob.oob_recv(peer, orig, count, flags); + /* now we have to do the conversions */ + for(i = 0; i < count; i++) { + if(types[i] == MCA_OOB_BASE_INT16) { + /* figure out how many integers we have */ + num = orig[i].iov_len / 2; + /* unpack the data */ + mca_oob_base_unpack(msg[i].iov_base, orig[i].iov_base, num, MCA_OOB_BASE_INT16); + /* free the old buffer */ + free(orig[i].iov_base); + } else if(types[i] == MCA_OOB_BASE_INT32) { + /* figure out how many integers we have */ + num = orig[i].iov_len / 4; + /* unpack the data */ + mca_oob_base_unpack(msg[i].iov_base, orig[i].iov_base, num, MCA_OOB_BASE_INT32); + /* free the old buffer */ + free(orig[i].iov_base); + } + } + /* free the iovecs we allocated */ + free(orig); + } else { + rc = mca_oob.oob_recv(peer, msg, count, flags); + } + return rc; +} + diff --git a/src/mca/oob/base/oob_base_recv_nb.c b/src/mca/oob/base/oob_base_recv_nb.c new file mode 100644 index 0000000000..8ae8fe2646 --- /dev/null +++ b/src/mca/oob/base/oob_base_recv_nb.c @@ -0,0 +1,131 @@ +#include "mca/oob/oob.h" +#include "mca/oob/base/base.h" +#include + +struct mca_oob_base_cb_data_t { + mca_oob_callback_fn_t user_callback; + void * user_data; + const mca_oob_base_type_t * types; + const struct iovec * user_iovec; +}; + +/* this is the callback function we will register when we have to do any conversion */ +static void mca_oob_base_recv_cb(int status, const ompi_process_name_t* peer, + const struct iovec* msg, + size_t count, void* cbdata); + +static void mca_oob_base_recv_cb(int status, const ompi_process_name_t* peer, + const struct iovec* msg, + size_t count, void* cbdata) +{ + int i, num; + struct mca_oob_base_cb_data_t * cb_struct = (struct mca_oob_base_cb_data_t *) cbdata; + const struct iovec * user_iovec = cb_struct->user_iovec; + mca_oob_base_type_t * types = cb_struct->types; + + for(i = 0; i < count; i++) { + if(types[i] == MCA_OOB_BASE_INT16) { + /* figure out how many integers we have */ + num = msg[i].iov_len / 2; + /* unpack the data */ + mca_oob_base_unpack(user_iovec[i].iov_base, msg[i].iov_base, num, MCA_OOB_BASE_INT16); + /* free the old buffer */ + free(msg[i].iov_base); + } else if(types[i] == MCA_OOB_BASE_INT32) { + /* figure out how many integers we have */ + num = msg[i].iov_len / 4; + /* unpack the data */ + mca_oob_base_unpack(user_iovec[i].iov_base, msg[i].iov_base, num, MCA_OOB_BASE_INT32); + /* free the old buffer */ + free(user_iovec[i].iov_base); + } + } + /* free the iovecs we allocated */ + free((void *)msg); + /* call the user callback function */ + cb_struct->user_callback(status, peer, user_iovec, count, cb_struct->user_data); + /* free the cb structure */ + free(cb_struct); + return; +} + +/* + * Non-blocking version of mca_oob_recv_nb(). + * + * @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, + * @param cbfunc (IN) Callback function on recv completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error or number of bytes actually received. + */ + +int mca_oob_recv_nb(ompi_process_name_t* peer, const struct iovec* msg, int count, int flags, + mca_oob_callback_fn_t cbfunc, void* cbdata) +{ + return(mca_oob.oob_recv_nb(peer, msg, count, flags, cbfunc, cbdata)); +} + +/* + * Non-blocking version of mca_oob_recv_ntoh(). + * + * @param peer (IN/OUT) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param types (IN) Parallel array to iovecs describing data type of each iovec element. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, + * @param cbfunc (IN) Callback function on recv completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error or number of bytes actually received. + */ + +int mca_oob_recv_ntoh_nb(ompi_process_name_t* peer, const struct iovec* msg, + const mca_oob_base_type_t* types, int count, int flags, + mca_oob_callback_fn_t cbfunc, void* cbdata) +{ + int rc, i = 0; + struct iovec * orig; + bool convert = false; + struct mca_oob_base_cb_data_t * cb_struct; + /* see if we actually have to convert anything */ + /* first check to see if we are already in network byte order */ + if(1 != htons(1)) { + /* if we aren't, see if there is any data types that need to be converted */ + while(!convert && (i < count)) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + convert = true; + } + i++; + } + } + if(convert) { + /* now if we need to convert anything we neeg to create a new iovec + * to recieve into */ + orig = malloc(sizeof(struct iovec) * count); + /* copy their iovecs */ + memcpy(orig, msg, sizeof(struct iovec) * count); + cb_struct = malloc(sizeof(struct mca_oob_base_cb_data_t)); + cb_struct->user_data = cbdata; + cb_struct->user_callback = cbfunc; + cb_struct->user_iovec = msg; + cb_struct->types = types; + /* copy their array into ours */ + memcpy(orig, msg, sizeof(struct iovec) * count); + /* now we need to go through the iovects, and any ints we need to + * allocate our own space to recieve into, so we can convert into + * their space later */ + for(i = 0; i < count; i++) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + orig[i].iov_base = malloc(orig[i].iov_len); + } + } + /* now the new buffers are ready. do the recieve */ + rc = mca_oob.oob_recv_nb(peer, orig, count, flags, &mca_oob_base_recv_cb, + (void *) cb_struct); + } else { + rc = mca_oob.oob_recv_nb(peer, msg, count, flags, cbfunc, cbdata); + } + return rc; +} + diff --git a/src/mca/oob/base/oob_base_select.c b/src/mca/oob/base/oob_base_select.c deleted file mode 100644 index ecb54adb4e..0000000000 --- a/src/mca/oob/base/oob_base_select.c +++ /dev/null @@ -1,127 +0,0 @@ -/* - * $HEADER$ - */ - -#include "ompi_config.h" - -#include - -#include "runtime/runtime.h" -#include "util/output.h" -#include "mca/mca.h" -#include "mca/base/base.h" -#include "mca/oob/oob.h" -#include "mca/oob/base/base.h" - - -/** - * Function for selecting one module from all those that are - * available. - * - * Call the init function on all available modules and get their - * priorities. Select the module with the highest priority. All - * other modules will be closed and unloaded. - */ -int mca_oob_base_select(bool *allow_multi_user_threads, - bool *have_hidden_threads) -{ - int priority, best_priority; - bool user_threads, hidden_threads; - bool best_user_threads, best_hidden_threads; - ompi_list_item_t *item; - mca_base_module_list_item_t *mli; - mca_oob_base_module_t *module, *best_module; - mca_oob_t *actions, *best_actions; - extern ompi_list_t mca_oob_base_modules_available; - - /* Traverse the list of available modules; call their init - functions. */ - - best_priority = -1; - best_module = NULL; - for (item = ompi_list_get_first(&mca_oob_base_modules_available); - ompi_list_get_end(&mca_oob_base_modules_available) != item; - item = ompi_list_get_next(item)) { - mli = (mca_base_module_list_item_t *) item; - module = (mca_oob_base_module_t *) mli->mli_module; - - ompi_output_verbose(10, mca_oob_base_output, - "select: initializing %s module %s", - module->oobm_version.mca_type_name, - module->oobm_version.mca_module_name); - if (NULL == module->oobm_init) { - ompi_output_verbose(10, mca_oob_base_output, - "select: no init function; ignoring module"); - } else { - actions = module->oobm_init(&priority, &user_threads, &hidden_threads); - if (NULL == actions) { - ompi_output_verbose(10, mca_oob_base_output, - "select: init returned failure"); - } else { - ompi_output_verbose(10, mca_oob_base_output, - "select: init returned priority %d", priority); - if (priority > best_priority) { - best_priority = priority; - best_user_threads = user_threads; - best_hidden_threads = hidden_threads; - best_module = module; - best_actions = actions; - } - } - } - } - - /* Finished querying all modules. Check for the bozo case. */ - - if (NULL == best_module) { - /* JMS Replace with show_help */ - ompi_abort(1, "No OOB module available. This shouldn't happen."); - } - - /* Finalize all non-selected modules */ - - for (item = ompi_list_get_first(&mca_oob_base_modules_available); - ompi_list_get_end(&mca_oob_base_modules_available) != item; - item = ompi_list_get_next(item)) { - mli = (mca_base_module_list_item_t *) item; - module = (mca_oob_base_module_t *) mli->mli_module; - - if (module != best_module) { - - /* Finalize */ - - if (NULL != module->oobm_finalize) { - - /* Blatently ignore the return code (what would we do to - recover, anyway? This module is going away, so errors - don't matter anymore) */ - - module->oobm_finalize(); - ompi_output_verbose(10, mca_oob_base_output, - "select: module %s finalized", - module->oobm_version.mca_module_name); - } - } - } - - /* This base function closes, unloads, and removes from the - available list all unselected modules. The available list will - contain only the selected module. */ - - mca_base_modules_close(mca_oob_base_output, &mca_oob_base_modules_available, - (mca_base_module_t *) best_module); - - /* Save the winner */ - - mca_oob_base_selected_module = *best_module; - mca_oob = *best_actions; - *allow_multi_user_threads = best_user_threads; - *have_hidden_threads = best_hidden_threads; - ompi_output_verbose(10, mca_oob_base_output, - "select: module %s initialized", - module->oobm_version.mca_module_name); - - /* All done */ - - return OMPI_SUCCESS; -} diff --git a/src/mca/oob/base/oob_base_send.c b/src/mca/oob/base/oob_base_send.c new file mode 100644 index 0000000000..1c35601454 --- /dev/null +++ b/src/mca/oob/base/oob_base_send.c @@ -0,0 +1,82 @@ +#include "mca/oob/oob.h" +#include "mca/oob/base/base.h" +#include +/* +* Similiar to unix send(2). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +*/ + +int mca_oob_send(const ompi_process_name_t* peer, const struct iovec *msg, int count, int flags) +{ + return(mca_oob.oob_send(peer, msg, count, flags)); +} + +/* + * Convert data (if required) to network byte order prior to sending to peer. + * + * @param peer (IN) Opaque name of peer process. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param types (IN) Parallel array to iovecs describing data type of each iovec element. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) Currently unused. + * @return OMPI error code (<0) on error number of bytes actually sent. + */ + +int mca_oob_send_hton(const ompi_process_name_t* peer, const struct iovec *msg, + const mca_oob_base_type_t *types, int count, int flags) +{ + int rc, i = 0; + struct iovec * converted; + bool convert = false; + /* see if we actually have to convert anything */ + /* first check to see if we are already in network byte order */ + if(1 != htons(1)) { + /* if we aren't, see if there is any data types that need to be converted */ + while(!convert && (i < count)) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + convert = true; + } + i++; + } + } + + if(convert) { + converted = malloc(sizeof(struct iovec) * count); + memcpy(converted, msg, sizeof(struct iovec) * count); + for(i = 0; i < count; i++) { + if(types[i] == MCA_OOB_BASE_INT16) { + /* figure out how many integers we have */ + rc = msg[i].iov_len / 2; + /* allocate a buffer for the converted data */ + converted[i].iov_base = malloc(msg[i].iov_len); + /* pack the data */ + mca_oob_base_pack(converted[i].iov_base, msg[i].iov_base, rc, MCA_OOB_BASE_INT16); + } else if(types[i] == MCA_OOB_BASE_INT32) { + /* figure out how many integers we have */ + rc = msg[i].iov_len / 4; + /* allocate a buffer for the converted data */ + converted[i].iov_base = malloc(msg[i].iov_len); + /* pack the data */ + mca_oob_base_pack(converted[i].iov_base, msg[i].iov_base, rc, MCA_OOB_BASE_INT32); + } + } + rc = mca_oob.oob_send(peer, converted, count, flags); + /* clean up any space we allocated */ + for(i = 0; i < count; i++) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + free(converted[i].iov_base); + } + } + free(converted); + } else { + rc = mca_oob.oob_send(peer, msg, count, flags); + } + return rc; + +} + diff --git a/src/mca/oob/base/oob_base_send_nb.c b/src/mca/oob/base/oob_base_send_nb.c new file mode 100644 index 0000000000..3f8fc8ddaf --- /dev/null +++ b/src/mca/oob/base/oob_base_send_nb.c @@ -0,0 +1,124 @@ +#include "mca/oob/oob.h" +#include "mca/oob/base/base.h" +#include + +struct mca_oob_base_cb_data_t { + mca_oob_callback_fn_t user_callback; + void * user_data; + const mca_oob_base_type_t * types; + const struct iovec * user_iovec; +}; + +/* this is the callback function we will register when we have to do any conversion */ +static void mca_oob_base_send_cb(int status, const ompi_process_name_t* peer, + const struct iovec* msg, + size_t count, void* cbdata); + +static void mca_oob_base_send_cb(int status, const ompi_process_name_t* peer, + const struct iovec* msg, + size_t count, void* cbdata) +{ + int i; + struct mca_oob_base_cb_data_t * cb_struct = (struct mca_oob_base_cb_data_t *) cbdata; + mca_oob_base_type_t * types = cb_struct->types; + for(i = 0; i < count; i++) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + free(msg[i].iov_base); + } + } + free((void *)msg); + /* call the user callback function */ + cb_struct->user_callback(status, peer, cb_struct->user_iovec, count, cb_struct->user_data); + free(cb_struct); + return; +} + + + + +/* + * Non-blocking version of mca_oob_send(). + * + * @param peer (IN) Opaque name of peer process. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) Currently unused. + * @param cbfunc (IN) Callback function on send completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error number of bytes actually sent. + * + */ + +int mca_oob_send_nb(const ompi_process_name_t* peer, const struct iovec* msg, int count, + int flags, mca_oob_callback_fn_t cbfunc, void* cbdata) +{ + return(mca_oob.oob_send_nb(peer, msg, count, flags, cbfunc, cbdata)); +} + +/* + * Non-blocking version of mca_oob_send_hton(). + * + * @param peer (IN) Opaque name of peer process. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param types (IN) Parallel array to iovecs describing data type of each iovec element. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) Currently unused. + * @param cbfunc (IN) Callback function on send completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error number of bytes actually sent. + */ + +int mca_oob_send_hton_nb(const ompi_process_name_t* peer, const struct iovec* msg, + const mca_oob_base_type_t* types, int count, int flags, + mca_oob_callback_fn_t cbfunc, void* cbdata) +{ + int rc, i = 0; + struct iovec * converted; + struct mca_oob_base_cb_data_t * cb_struct; + bool convert = false; + /* see if we actually have to convert anything */ + /* first check to see if we are already in network byte order */ + if(1 != htons(1)) { + /* if we aren't, see if there is any data types that need to be converted */ + while(!convert && (i < count)) { + if((types[i] == MCA_OOB_BASE_INT16) || (types[i] == MCA_OOB_BASE_INT32)) { + convert = true; + } + i++; + } + } + if(convert) { + converted = malloc(sizeof(struct iovec) * count); + /* copy the passed iovect into the new one */ + memcpy(converted, msg, sizeof(struct iovec) * count); + cb_struct = malloc(sizeof(struct mca_oob_base_cb_data_t)); + cb_struct->user_data = cbdata; + cb_struct->user_callback = cbfunc; + cb_struct->user_iovec = msg; + cb_struct->types = types; + for(i = 0; i < count; i++) { + if(types[i] == MCA_OOB_BASE_INT16) { + /* figure out how many integers we have */ + rc = msg[i].iov_len / 2; + /* allocate a buffer for the converted data */ + converted[i].iov_base = malloc(msg[i].iov_len); + /* pack the data */ + mca_oob_base_pack(converted[i].iov_base, msg[i].iov_base, rc, MCA_OOB_BASE_INT16); + } else if(types[i] == MCA_OOB_BASE_INT32) { + /* figure out how many integers we have */ + rc = msg[i].iov_len / 4; + /* allocate a buffer for the converted data */ + converted[i].iov_base = malloc(msg[i].iov_len); + /* pack the data */ + mca_oob_base_pack(converted[i].iov_base, msg[i].iov_base, rc, MCA_OOB_BASE_INT32); + } + } + rc = mca_oob.oob_send_nb(peer, converted, count, flags, + &mca_oob_base_send_cb, + (void *) cb_struct); + } else { + rc = mca_oob.oob_send_nb(peer, msg, count, flags, cbfunc, cbdata); + } + return rc; +} + diff --git a/src/mca/oob/cofs/Makefile.am b/src/mca/oob/cofs/Makefile.am index afaaa4189c..ec2fd17085 100644 --- a/src/mca/oob/cofs/Makefile.am +++ b/src/mca/oob/cofs/Makefile.am @@ -1,8 +1,9 @@ +# -*- makefile -*- # # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +11,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_oob_cofs_DSO -component_noinst = -component_install = mca_oob_cofs.la +if OMPI_BUILD_oob_cofs_LOADABLE_MODULE +module_noinst = +module_install = mca_oob_cofs.la else -component_noinst = libmca_oob_cofs.la -component_install = +module_noinst = libmca_oob_cofs.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_oob_cofs_la_SOURCES = -mca_oob_cofs_la_LIBADD = src/libmca_oob_cofs.la $(LIBOMPI_LA) +mca_oob_cofs_la_LIBADD = \ + src/libmca_oob_cofs.la \ + $(LIBOMPI_LA) mca_oob_cofs_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_oob_cofs_la_SOURCES = libmca_oob_cofs_la_LIBADD = src/libmca_oob_cofs.la libmca_oob_cofs_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/oob/cofs/src/oob_cofs.c b/src/mca/oob/cofs/src/oob_cofs.c index 486d20c3a0..37017e87dd 100644 --- a/src/mca/oob/cofs/src/oob_cofs.c +++ b/src/mca/oob/cofs/src/oob_cofs.c @@ -4,6 +4,9 @@ * */ +#include +#include +#include #include "ompi_config.h" #include "mca/oob/oob.h" @@ -16,104 +19,121 @@ #include #include -static int blocking_recv_posted = 0; -static int do_recv(ompi_job_handle_t job_handle, int vpid, int* tag, - void** data, size_t* data_len); +static int do_recv(ompi_process_id_t jobid, ompi_process_id_t procid, const struct iovec* iov, int count, int flags); -int -mca_oob_cofs_send(ompi_job_handle_t job_handle, int vpid, int tag, - void* data, size_t data_len) +/** +* Similiar to unix send(2). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +*/ + +int mca_oob_cofs_send( + const ompi_process_name_t* peer, + const struct iovec *iov, + int count, + int flags) { FILE *fp; - size_t wlen; + int i, wlen; + size_t size = 0; char msg_file[OMPI_PATH_MAX]; char msg_file_tmp[OMPI_PATH_MAX]; /* create the file and open it... */ - snprintf(msg_file, OMPI_PATH_MAX, "%s/%s_%d_%d_%d_%lld.msg", mca_oob_cofs_comm_loc, - job_handle, mca_oob_cofs_my_vpid, vpid, tag, mca_oob_cofs_serial); - snprintf(msg_file_tmp, OMPI_PATH_MAX, "%s/.%s_%d_%d_%d_%lld.msg", mca_oob_cofs_comm_loc, - job_handle, mca_oob_cofs_my_vpid, vpid, tag, mca_oob_cofs_serial); + snprintf(msg_file, OMPI_PATH_MAX, "%s/%d_%d_%d_%lld.msg", mca_oob_cofs_comm_loc, + mca_oob_base_self.jobid, mca_oob_base_self.procid, peer->procid, mca_oob_cofs_serial); + snprintf(msg_file_tmp, OMPI_PATH_MAX, "%s/.%d_%d_%d_%lld.msg", mca_oob_cofs_comm_loc, + mca_oob_base_self.jobid, mca_oob_base_self.procid, peer->procid, mca_oob_cofs_serial); fp = fopen(msg_file_tmp, "w"); if (fp == NULL) { return OMPI_ERR_OUT_OF_RESOURCE; } - /* BWB - do network byte ordering... */ /* write size */ - wlen = fwrite(&data_len, sizeof(size_t), 1, fp); + for(i=0; ijobid, peer->procid, iov, count, flags); sleep(1); } - blocking_recv_posted = 0; return ret; } int -mca_oob_cofs_recv_nb(ompi_job_handle_t job_handle, int vpid, int* tag, - void** data, size_t* data_len) +mca_oob_cofs_recv_nb( + ompi_process_name_t* peer, + const struct iovec* iov, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata) { - if (blocking_recv_posted != 0) { - return OMPI_ERR_WOULD_BLOCK; - } - - return do_recv(job_handle, vpid, tag, data, data_len); -} - - -int -mca_oob_cofs_recv_cb(ompi_job_handle_t job_handle, int vpid, int tag, - mca_oob_base_recv_cb_t callback) -{ - return OMPI_ERR_NOT_SUPPORTED; + int status = mca_oob_cofs_recv(peer, iov, count, flags); + if(NULL != cbfunc) + cbfunc(status, peer, iov, count, cbdata); + return status; } static char* -find_match(ompi_job_handle_t job_handle, int vpid, int* tag) +find_match(ompi_process_id_t jobid, ompi_process_id_t procid) { DIR* dir; struct dirent *ent; - char tmp_handle[OMPI_PATH_MAX]; uint64_t tmp_serial; - int tmp_tag, tmp_vpid, tmp_myvpid; + int tmp_jobid, tmp_procid, tmp_myprocid; int ret; bool found = false; char best_name[OMPI_PATH_MAX]; - int best_tag; uint64_t best_serial = ((1ULL << 63) - 1); dir = opendir(mca_oob_cofs_comm_loc); @@ -124,22 +144,19 @@ find_match(ompi_job_handle_t job_handle, int vpid, int* tag) while ((ent = readdir(dir)) != NULL) { if (ent->d_name[0] == '.') continue; - ret = sscanf(ent->d_name, "%[^_]_%d_%d_%d_%llu.msg", tmp_handle, &tmp_vpid, - &tmp_myvpid, &tmp_tag, &tmp_serial); - if (ret != 5) { + ret = sscanf(ent->d_name, "%d_%d_%d_%llu.msg", &tmp_jobid, &tmp_procid, + &tmp_myprocid, &tmp_serial); + if (ret != 4) { continue; } - if (strcmp(tmp_handle, job_handle)) { + if (tmp_jobid != jobid) { continue; } - if (tmp_myvpid != mca_oob_cofs_my_vpid) { + if (tmp_myprocid != mca_oob_base_self.procid) { continue; } - if (*tag != MCA_OOB_ANY_TAG && tmp_tag != *tag) { - continue; - } - if (tmp_vpid != vpid) { + if (tmp_procid != procid) { continue; } @@ -147,14 +164,12 @@ find_match(ompi_job_handle_t job_handle, int vpid, int* tag) found = true; if (tmp_serial < best_serial) { strcpy(best_name, ent->d_name); - best_tag = tmp_tag; best_serial = tmp_serial; } } closedir(dir); if (found) { - *tag = best_tag; return strdup(best_name); } else { return NULL; @@ -162,51 +177,39 @@ find_match(ompi_job_handle_t job_handle, int vpid, int* tag) } -static int -do_recv(ompi_job_handle_t job_handle, int vpid, int* tag, - void** data, size_t* data_len) +static int +do_recv(ompi_process_id_t jobid, ompi_process_id_t procid, const struct iovec* iov, int count, int flags) { char *fname; char full_fname[OMPI_PATH_MAX]; - FILE *fp; + int fd; size_t rlen; + size_t size; - fname = find_match(job_handle, vpid, tag); + fname = find_match(jobid, procid); if (fname == NULL) { return OMPI_ERR_WOULD_BLOCK; } snprintf(full_fname, OMPI_PATH_MAX, "%s/%s", mca_oob_cofs_comm_loc, fname); free(fname); - fp = fopen(full_fname, "r"); - if (fp == NULL) { + fd = open(full_fname, O_RDONLY); + if (fd < 0) { + return OMPI_ERROR; + } + if((flags & MCA_OOB_PEEK) == 0) + unlink(full_fname); + + rlen = read(fd, &size, sizeof(size)); + if (rlen != sizeof(size)) { + close(fd); return OMPI_ERROR; } - unlink(full_fname); - - rlen = fread(data_len, sizeof(size_t), 1, fp); - if (rlen != 1) { - fclose(fp); - return OMPI_ERROR; + if(iov != NULL && count > 0) { + rlen = readv(fd, iov, count); } - - *data = (void*) malloc(*data_len); - if (*data == NULL) { - fclose(fp); - *data_len = 0; - return OMPI_ERROR; - } - - rlen = fread(*data, 1, *data_len, fp); - if (rlen != *data_len) { - fclose(fp); - free(*data); - *data_len = 0; - return OMPI_ERROR; - } - - fclose(fp); - - return OMPI_SUCCESS; + close(fd); + return (flags & MCA_OOB_TRUNC) ? size : rlen; } + diff --git a/src/mca/oob/cofs/src/oob_cofs.h b/src/mca/oob/cofs/src/oob_cofs.h index bf40550d5e..54801558d4 100644 --- a/src/mca/oob/cofs/src/oob_cofs.h +++ b/src/mca/oob/cofs/src/oob_cofs.h @@ -3,6 +3,8 @@ * $HEADER$ * */ +#ifndef OOB_COFS_H +#define OOB_COFS_H #include "ompi_config.h" #include "mca/oob/oob.h" @@ -18,25 +20,91 @@ int mca_oob_cofs_close(void); /* * Startup / Shutdown */ -struct mca_oob_1_0_0_t* mca_oob_cofs_init(int *priority, - bool *allow_multi_user_threads, - bool *have_hidden_threads); +mca_oob_t* mca_oob_cofs_init(bool *allow_multi_user_threads, bool *have_hidden_threads); int mca_oob_cofs_finalize(void); -/* - * "Action" functions - */ -int mca_oob_cofs_send(ompi_job_handle_t job_handle, int vpid, int tag, - void* data, size_t data_len); -int mca_oob_cofs_recv(ompi_job_handle_t job_handle, int vpid, int* tag, - void** data, size_t* data_len); -int mca_oob_cofs_recv_nb(ompi_job_handle_t job_handle, int vpid, int* tag, - void** data, size_t* data_len); -int mca_oob_cofs_recv_cb(ompi_job_handle_t job_handle, int vpid, int tag, - mca_oob_base_recv_cb_t callback); + +/** +* Implementation of mca_oob_send(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +*/ + +int mca_oob_cofs_send( + const ompi_process_name_t*, + const struct iovec* msg, + int count, + int flags); +/** +* Implementation of mca_oob_recv(). +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the +* iovec array without removing the message from the queue. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +int mca_oob_cofs_recv( + ompi_process_name_t* peer, + const struct iovec *msg, + int count, + int flags); + + +/** +* Implementation of mca_oob_send_nb(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @param cbfunc (IN) Callback function on send completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +*/ + +int mca_oob_cofs_send_nb( + const ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + + +/** +* Implementation of mca_oob_recv_nb(). +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, +* @param cbfunc (IN) Callback function on recv completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +int mca_oob_cofs_recv_nb( + ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + extern char mca_oob_cofs_comm_loc[OMPI_PATH_MAX]; /* location for file drop-off */ -extern int mca_oob_cofs_my_vpid; extern uint64_t mca_oob_cofs_serial; + +#endif + diff --git a/src/mca/oob/cofs/src/oob_cofs_module.c b/src/mca/oob/cofs/src/oob_cofs_module.c index 0cc4884196..f41a358b82 100644 --- a/src/mca/oob/cofs/src/oob_cofs_module.c +++ b/src/mca/oob/cofs/src/oob_cofs_module.c @@ -9,18 +9,23 @@ #include "include/constants.h" #include "mca/mca.h" #include "mca/oob/oob.h" -#include "mca/oob/cofs/src/oob_cofs.h" +#include "oob_cofs.h" #include "include/types.h" +#include "util/proc_info.h" +#include "util/output.h" #include #include #include #include + + + /* * Struct of function pointers and all that to let us be initialized */ -mca_oob_base_module_1_0_0_t mca_oob_cofs_module = { +mca_oob_base_component_1_0_0_t mca_oob_cofs_module = { { MCA_OOB_BASE_VERSION_1_0_0, @@ -38,15 +43,16 @@ mca_oob_base_module_1_0_0_t mca_oob_cofs_module = { mca_oob_cofs_finalize }; -struct mca_oob_1_0_0_t mca_oob_cofs_1_0_0 = { +mca_oob_t mca_oob_cofs = { mca_oob_cofs_send, mca_oob_cofs_recv, - mca_oob_cofs_recv_nb, - mca_oob_cofs_recv_cb + mca_oob_cofs_send_nb, + mca_oob_cofs_recv_nb }; char mca_oob_cofs_comm_loc[OMPI_PATH_MAX]; -int mca_oob_cofs_my_vpid; +int mca_oob_cofs_my_jobid; +int mca_oob_cofs_my_procid; uint64_t mca_oob_cofs_serial; int @@ -64,54 +70,30 @@ mca_oob_cofs_close(void) struct mca_oob_1_0_0_t* -mca_oob_cofs_init(int *priority, bool *allow_multi_user_threads, - bool *have_hidden_threads) +mca_oob_cofs_init(bool *allow_multi_user_threads, bool *have_hidden_threads) { int len; char *tmp; FILE *fp; - *priority = 0; - *allow_multi_user_threads = true; - *have_hidden_threads = true; - - /* - * BWB - fix me, make register the "right" way... - */ - tmp = getenv("MCA_common_ompi_cofs_comm_dir"); - if (tmp == NULL) { - /* make it $HOME */ - tmp = getenv("HOME"); - if (tmp == NULL) { - printf("oob_cofs can not find communication dir\n"); - return NULL; - } - snprintf(mca_oob_cofs_comm_loc, OMPI_PATH_MAX, "%s/cofs", tmp); - } else { - snprintf(mca_oob_cofs_comm_loc, OMPI_PATH_MAX, "%s", tmp); - } - - /* - * BWB - fix me, make register the "right" way... - */ - /* find our vpid */ - tmp = getenv("MCA_common_ompi_cofs_my_vpid"); - if (tmp == NULL) { - printf("oob_cofs can not find vpid\n"); - return NULL; - } - mca_oob_cofs_my_vpid = atoi(tmp); + *allow_multi_user_threads &= true; /* * See if we can write in our directory... */ - len = strlen(mca_oob_cofs_comm_loc) + 32; + if((tmp = getenv("OMPI_MCA_oob_cofs_dir")) == NULL) { + ompi_output(0, "mca_oob_cofs_init: invalid/missing OMPI_MCA_oob_cofs_dir\n"); + return NULL; + } + strncpy(mca_oob_cofs_comm_loc, tmp, sizeof(mca_oob_cofs_comm_loc)); + + len = strlen(tmp) + 32; tmp = malloc(len); if (tmp == NULL) return NULL; - snprintf(tmp, len, "%s/oob.%d", mca_oob_cofs_comm_loc, mca_oob_cofs_my_vpid); + snprintf(tmp, len, "%s/oob.%d", mca_oob_cofs_comm_loc, mca_oob_cofs_my_procid); fp = fopen(tmp, "w"); if (fp == NULL) { - printf("oob_cofs can not write in communication dir\n"); + printf("oob_cofs: can not write in communication dir\n"); free(tmp); return NULL; } @@ -120,8 +102,7 @@ mca_oob_cofs_init(int *priority, bool *allow_multi_user_threads, free(tmp); mca_oob_cofs_serial = 0; - - return &mca_oob_cofs_1_0_0; + return &mca_oob_cofs; } diff --git a/src/mca/oob/oob.h b/src/mca/oob/oob.h index a22f0c1975..bb32a351f4 100644 --- a/src/mca/oob/oob.h +++ b/src/mca/oob/oob.h @@ -1,108 +1,362 @@ -/* -*- C -*- - * +/* * $HEADER$ */ -/** @file **/ - -/** - * \brief Out of Band Messaging Interface +/** @file: * - * OMPI/MPI provides a simple point-to-point tagged messaging system - * intended for out-of-band communication. This interface should be - * used minimally in general OMPI code and should not be used - * explicitly in the MPI layer. Not all run-time environments provide - * a sufficient out-of-band messaging system, so some environments may - * choose not to implement this interface, at the cost of reduced - * functionality. - * - * This interface can be brought up as soon as the process control - * interface is initiated. The process control interface is not - * strictly required, but it is unclear how one could determine the - * processes' parallel_job_id and vpid without asking the process - * control interface. It should not be a requirement of this - * interface that MPI exist or be properly initialized for a send to - * complete. One can possibly envision using a ptl progression engine - * for out-of-band messaging, but it must not put any MPI requirements - * on the interface. - * - * The out-of-band messaging interface is actually implemented through - * the oob mca module - details of a particular implementation - * will be found there. + * the oob framework */ -#ifndef MCA_OOB_H_ -#define MCA_OOB_H_ - -#include "ompi_config.h" - -#include "include/types.h" +#ifndef _MCA_OOB_H_ +#define _MCA_OOB_H_ #include "mca/mca.h" - +#include "mca/base/base.h" +#include "mca/ns/ns.h" +#include /* - * Global constants / types + * Address for wildcard receives. */ - /* "Special" tags */ -#define MCA_OOB_ANY_TAG -1 -#define MCA_OOB_REGISTRY_TAG -2 +extern ompi_process_name_t mca_oob_base_any; +extern ompi_process_name_t mca_oob_base_self; - /* "Special" vpids */ -#define MCA_OOB_MPIRUN -1 - -typedef void (*mca_oob_base_recv_cb_t)(ompi_job_handle_t job_handle, int tag, - int vpid, void* data, size_t data_len, int status); +#define MCA_OOB_BASE_ANY &mca_oob_base_any +#define MCA_OOB_BASE_SELF &mca_oob_base_self /* - * Functions every module instance will have to provide + * OOB API */ -typedef struct mca_oob_1_0_0_t* - (*mca_oob_base_init_fn_t)(int *priority, bool *allow_multi_user_threads, - bool *have_hidden_threads); -typedef int (*mca_oob_base_send_fn_t)(ompi_job_handle_t job_handle, int vpid, int tag, - void* data, size_t data_len); -typedef int (*mca_oob_base_recv_fn_t)(ompi_job_handle_t job_handle, int vpid, int* tag, - void** data, size_t* data_len); -typedef int (*mca_oob_base_recv_nb_fn_t)(ompi_job_handle_t job_handle, int vpid, int* tag, - void** data, size_t* data_len); -typedef int (*mca_oob_base_recv_cb_fn_t)(ompi_job_handle_t job_handle, int vpid, int tag, - mca_oob_base_recv_cb_t callback); -typedef int (*mca_oob_base_finalize_fn_t)(void); +/** + * Supported datatypes for conversion operations. + */ +typedef enum { + MCA_OOB_BASE_BYTE, /**< a byte of data */ + MCA_OOB_BASE_INT16, /**< a 16 bit integer */ + MCA_OOB_BASE_INT32, /**< a 32 bit integer */ + MCA_OOB_BASE_STRING, /**< a NULL terminated string */ + MCA_OOB_BASE_PACKED /**< already packed data. */ +} mca_oob_base_type_t; + +/** +* General flags for send/recv +* +* An example of usage - to determine the size of the next available message w/out receiving it: +* +* int size = mca_oob_recv(name, 0, 0, MSG_OOB_PEEK|MSG_OOB_TRUNC); +*/ + +#define MCA_OOB_PEEK 0x01 /* flag to oob_recv to allow caller to peek a portion of the next available + * message w/out removing the message from the queue. */ +#define MCA_OOB_TRUNC 0x02 /* flag to oob_recv to return the actual size of the message even if the receive + buffer is smaller than the number of bytes available */ + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +/** +* Similiar to unix send(2). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +*/ + +int mca_oob_send( + const ompi_process_name_t* peer, + const struct iovec *msg, + int count, + int flags); + +/** +* Convert data (if required) to network byte order prior to sending to peer. +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +*/ + +int mca_oob_send_hton( + const ompi_process_name_t* peer, + const struct iovec *msg, + const mca_oob_base_type_t *types, + int count, + int flags); + + +/** +* Similiar to unix recv(2) +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the +* iovec array without removing the message from the queue. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +int mca_oob_recv(ompi_process_name_t* peer, const struct iovec *msg, int count, int flags); + +/** +* Receive data and convert (if required) to host byte order. +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the +* iovec array without removing the message from the queue. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +int mca_oob_recv_ntoh( + ompi_process_name_t* peer, + const struct iovec *msg, + const mca_oob_base_type_t *types, + int count, + int flags); /* - * Ver 1.0.0 + * Non-blocking versions of send/recv. +*/ + + +/** +* Callback function on send/recv completion. +* +* @param status (IN) Completion status - equivalent to the return value from blocking send/recv. +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param cbdata (IN) User data. +*/ + +typedef void (*mca_oob_callback_fn_t)( + int status, + const ompi_process_name_t* peer, + const struct iovec* msg, + size_t count, + void* cbdata); + +/** +* Non-blocking version of mca_oob_send(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @param cbfunc (IN) Callback function on send completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +*/ + +int mca_oob_send_nb( + const ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/** +* Non-blocking version of mca_oob_send_hton(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @param cbfunc (IN) Callback function on send completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +*/ + +int mca_oob_send_hton_nb( + const ompi_process_name_t* peer, + const struct iovec* msg, + const mca_oob_base_type_t* types, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/** +* Non-blocking version of mca_oob_recv(). +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, +* @param cbfunc (IN) Callback function on recv completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +int mca_oob_recv_nb( + ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/** +* Non-blocking version of mca_oob_recv_ntoh(). +* +* @param peer (IN/OUT) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, +* @param cbfunc (IN) Callback function on recv completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +int mca_oob_recv_ntoh_nb( + ompi_process_name_t* peer, + const struct iovec* msg, + const mca_oob_base_type_t* types, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif +/* + * OOB Component/Module function prototypes. */ -struct mca_oob_base_module_1_0_0_t { - mca_base_module_t oobm_version; - mca_base_module_data_1_0_0_t oobm_data; - mca_oob_base_init_fn_t oobm_init; - mca_oob_base_finalize_fn_t oobm_finalize; -}; -typedef struct mca_oob_base_module_1_0_0_t mca_oob_base_module_1_0_0_t; +/** +* Implementation of mca_oob_send(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +*/ +typedef int (*mca_oob_base_send_fn_t)( + const ompi_process_name_t* peer, + const struct iovec *msg, + int count, + int flags); + + +/** +* Implementation of mca_oob_recv(). +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the +* iovec array without removing the message from the queue. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +typedef int (*mca_oob_base_recv_fn_t)( + ompi_process_name_t* peer, + const struct iovec *msg, + int count, + int flags); + +/** +* Implementation of mca_oob_send_nb(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) Currently unused. +* @param cbfunc (IN) Callback function on send completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +*/ + +typedef int (*mca_oob_base_send_nb_fn_t)( + const ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/** +* Implementation of mca_oob_recv_nb(). +* +* @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, +* @param cbfunc (IN) Callback function on recv completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error or number of bytes actually received. +*/ + +typedef int (*mca_oob_base_recv_nb_fn_t)( + ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/* + * OOB Module + */ + struct mca_oob_1_0_0_t { - mca_oob_base_send_fn_t oob_send; - mca_oob_base_recv_fn_t oob_recv; - mca_oob_base_recv_nb_fn_t oob_recv_nb; - mca_oob_base_recv_cb_fn_t oob_recv_cb; + mca_oob_base_send_fn_t oob_send; + mca_oob_base_recv_fn_t oob_recv; + mca_oob_base_send_nb_fn_t oob_send_nb; + mca_oob_base_recv_nb_fn_t oob_recv_nb; }; typedef struct mca_oob_1_0_0_t mca_oob_1_0_0_t; +typedef struct mca_oob_1_0_0_t mca_oob_t; + +/** + * OOB Component + */ + +typedef mca_oob_t* (*mca_oob_base_init_fn_t)( + bool *allow_multi_user_threads, + bool *have_hidden_threads); + +typedef int (*mca_oob_base_finalize_fn_t)(void); + +/** + * the standard component data structure + */ +struct mca_oob_base_component_1_0_0_t { + mca_base_module_t oob_version; + mca_base_module_data_1_0_0_t oob_data; + mca_oob_base_init_fn_t oob_init; + mca_oob_base_finalize_fn_t oob_finalize; +}; +typedef struct mca_oob_base_component_1_0_0_t mca_oob_base_component_1_0_0_t; +typedef mca_oob_base_component_1_0_0_t mca_oob_base_component_t; -typedef mca_oob_base_module_1_0_0_t mca_oob_base_module_t; -typedef mca_oob_1_0_0_t mca_oob_t; /* - * Macro for use in modules that are of type coll v1.0.0 + * Macro for use in components that are of type oob v1.0.0 */ #define MCA_OOB_BASE_VERSION_1_0_0 \ - /* oob v1.0 is chained to MCA v1.0 */ \ + /* pml v1.0 is chained to MCA v1.0 */ \ MCA_BASE_VERSION_1_0_0, \ - /* oob v1.0 */ \ + /* pml v1.0 */ \ "oob", 1, 0, 0 -extern mca_oob_t mca_oob; - - #endif + diff --git a/src/mca/oob/tcp/Makefile.am b/src/mca/oob/tcp/Makefile.am new file mode 100644 index 0000000000..8e04d27bdb --- /dev/null +++ b/src/mca/oob/tcp/Makefile.am @@ -0,0 +1,31 @@ +# -*- makefile -*- +# +# $HEADER$ +# + +include $(top_ompi_srcdir)/config/Makefile.options + +sources = \ + oob_tcp.c \ + oob_tcp_recv.c \ + oob_tcp_send.c \ + oob_tcp_peer.c + +if OMPI_BUILD_allocator_bucket_LOADABLE_MODULE +module_noinst = +module_install = mca_oob_tcp.la +else +module_noinst = libmca_oob_tcp.la +module_install = +endif + +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) +mca_oob_tcp_la_SOURCES = $(sources) +mca_oob_tcp_la_LIBADD = $(LIBOMPI_LA) +mca_oob_tcp_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(module_noinst) +libmca_oob_tcp_la_SOURCES = $(sources) +libmca_oob_tcp_la_LDFLAGS = -module -avoid-version + diff --git a/src/mca/oob/tcp/VERSION b/src/mca/oob/tcp/VERSION new file mode 100644 index 0000000000..dadc1f42ab --- /dev/null +++ b/src/mca/oob/tcp/VERSION @@ -0,0 +1,6 @@ +major=1 +minor=0 +release=0 +alpha=0 +beta=0 +svn=1 diff --git a/src/mca/oob/tcp/configure.params b/src/mca/oob/tcp/configure.params new file mode 100644 index 0000000000..1845450fd7 --- /dev/null +++ b/src/mca/oob/tcp/configure.params @@ -0,0 +1,9 @@ +# -*- shell-script -*- +# +# $HEADER$ +# + +# Specific to this module + +PARAM_INIT_FILE=oob_tcp.c +PARAM_CONFIG_FILES="Makefile" diff --git a/src/mca/oob/tcp/oob_tcp.c b/src/mca/oob/tcp/oob_tcp.c new file mode 100644 index 0000000000..720c678a8c --- /dev/null +++ b/src/mca/oob/tcp/oob_tcp.c @@ -0,0 +1,73 @@ +/* -*- C -*- + * + * $HEADER$ + * + */ + +#include "mca/oob/tcp/oob_tcp.h" + +/* + * Struct of function pointers and all that to let us be initialized + */ +mca_oob_base_component_1_0_0_t mca_oob_tcp_module = { + { + MCA_OOB_BASE_VERSION_1_0_0, + + "tcp", /* MCA module name */ + 1, /* MCA module major version */ + 0, /* MCA module minor version */ + 0, /* MCA module release version */ + mca_oob_tcp_open, /* module open */ + mca_oob_tcp_close /* module close */ + }, + { + false /* checkpoint / restart */ + }, + mca_oob_tcp_init, /* module init */ + mca_oob_tcp_finalize +}; + +struct mca_oob_1_0_0_t mca_oob_tcp_1_0_0 = { + mca_oob_tcp_send, + mca_oob_tcp_recv, + mca_oob_tcp_send_nb, + mca_oob_tcp_recv_nb +}; + +/* + * for now these 2 functions simply return an error so we won't + * use this module + */ +int mca_oob_tcp_open(void) +{ + return OMPI_ERROR; +} + + +int mca_oob_tcp_close(void) +{ + return OMPI_ERROR; +} + +/* + * this function will temporarily return NULL so we don't use it + */ +struct mca_oob_1_0_0_t* mca_oob_tcp_init(bool *allow_multi_user_threads, + bool *have_hidden_threads) +{ + /* set up the list for the cache of peer processes */ + OBJ_CONSTRUCT(&mca_oob_tcp_peer_list, ompi_list_t); + /* set up the rb tree for the cache of peer processes */ + OBJ_CONSTRUCT(&mca_oob_tcp_peer_tree, ompi_rb_tree_t); + ompi_rb_tree_init(&mca_oob_tcp_peer_tree, &mca_oob_tcp_peer_comp); + /* return &mca_oob_tcp_1_0_0; */ + return NULL; +} + + +int mca_oob_tcp_finalize(void) +{ + OBJ_DESTRUCT(&mca_oob_tcp_peer_list); + OBJ_DESTRUCT(&mca_oob_tcp_peer_tree); + return OMPI_SUCCESS; +} diff --git a/src/mca/oob/tcp/oob_tcp.h b/src/mca/oob/tcp/oob_tcp.h new file mode 100644 index 0000000000..fe1340a8c9 --- /dev/null +++ b/src/mca/oob/tcp/oob_tcp.h @@ -0,0 +1,109 @@ +/* + * $HEADER$ + */ +/** @file: + * + * Defines the functions for the tcp module. + */ + +#ifndef _MCA_OOB_TCP_H_ +#define _MCA_OOB_TCP_H_ + +#include "mca/oob/oob.h" +#include "mca/oob/base/base.h" +#include "class/ompi_rb_tree.h" +#include "mca/oob/tcp/oob_tcp_peer.h" +#include "mca/oob/tcp/oob_tcp_message.h" + +/* + * the list of peers + */ +ompi_list_t mca_oob_tcp_peer_list; +/* + * the tree of peers + */ +ompi_rb_tree_t mca_oob_tcp_peer_tree; + + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +/* + * standard module functions + */ +int mca_oob_tcp_open(void); +int mca_oob_tcp_close(void); +struct mca_oob_1_0_0_t* mca_oob_tcp_init(bool *allow_multi_user_threads, + bool *have_hidden_threads); +int mca_oob_tcp_finalize(void); + + +/** + * Similiar to unix send(2). + * + * @param peer (IN) Opaque name of peer process. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) Currently unused. + * @return OMPI error code (<0) on error number of bytes actually sent. + */ + +int mca_oob_tcp_send(const ompi_process_name_t* peer, const struct iovec *msg, int count, int flags); + +/** + * Similiar to unix recv(2) + * + * @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the + * iovec array without removing the message from the queue. + * @return OMPI error code (<0) on error or number of bytes actually received. + */ + +int mca_oob_tcp_recv(ompi_process_name_t* peer, const struct iovec *msg, int count, int flags); + + +/* + * Non-blocking versions of send/recv. + */ + +/** + * Non-blocking version of mca_oob_send(). + * + * @param peer (IN) Opaque name of peer process. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) Currently unused. + * @param cbfunc (IN) Callback function on send completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error number of bytes actually sent. + * + */ + +int mca_oob_tcp_send_nb(const ompi_process_name_t* peer, const struct iovec* msg, int count, + int flags, mca_oob_callback_fn_t cbfunc, void* cbdata); + +/** + * Non-blocking version of mca_oob_recv(). + * + * @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, + * @param cbfunc (IN) Callback function on recv completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error or number of bytes actually received. + */ + +int mca_oob_tcp_recv_nb(ompi_process_name_t* peer, const struct iovec* msg, int count, int flags, + mca_oob_callback_fn_t cbfunc, void* cbdata); + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif /* MCA_OOB_TCP_H_ */ + diff --git a/src/mca/oob/tcp/oob_tcp_message.h b/src/mca/oob/tcp/oob_tcp_message.h new file mode 100644 index 0000000000..a22392e396 --- /dev/null +++ b/src/mca/oob/tcp/oob_tcp_message.h @@ -0,0 +1,31 @@ +/* + * $HEADER$ + */ +/** @file: + * + * contains the data structure we will use to describe a message + */ + + +#ifndef _MCA_OOB_TCP_MESSAGE_H_ +#define _MCA_OOB_TCP_MESSAGE_H_ + +#include "class/ompi_list.h" +#include "mca/oob/tcp/oob_tcp_peer.h" +#include "mca/oob/oob.h" + +/** + * the describes each message + */ +struct mca_oob_tcp_message_t { + ompi_list_item_t super; /**< make it so we can put this on a list */ + size_t message_state; /**< the amount sent or recieved */ + struct iovec * message_data; /**< the data of the message */ + struct iovec * message_tmp; /**< in case we have to make a copy of the iovecs */ + int message_count; /**< the number of items in the iovect array */ + mca_oob_callback_fn_t cbfunc;/**< the callback function for the send/recieve */ + void * cbdata; /**< the data for the callback fnuction */ +}; + + +#endif /* _MCA_OOB_TCP_MESSAGE_H_ */ diff --git a/src/mca/oob/tcp/oob_tcp_peer.c b/src/mca/oob/tcp/oob_tcp_peer.c new file mode 100644 index 0000000000..ef3562785a --- /dev/null +++ b/src/mca/oob/tcp/oob_tcp_peer.c @@ -0,0 +1,26 @@ +#include "mca/oob/tcp/oob_tcp_peer.h" + +/* + * The function to compare 2 peers. Used for the rb tree + * + * @param peer1 the first peer + * @param peer2 the second peer + * + * @retval <0 if peer1 < peer2 + * @retval >0 if peer1 > peer2 + * @retval 0 if peer1 == peer2 + */ +/*****NEED TO MAKE WORK *****/ +int mca_oob_tcp_peer_comp(void * key1, void * key2) +{ +/* mca_oob_tcp_peer_t * p1 = (mca_oob_tcp_peer_t *) key1; + mca_oob_tcp_peer_t * p2 = (mca_oob_tcp_peer_t *) key2; + + if(p1->peer_name < p2->peer_name) { + return(-1); + } else if(p1->peer_name > p2->peer_name) { + return(1); + }*/ + return(0); +} + diff --git a/src/mca/oob/tcp/oob_tcp_peer.h b/src/mca/oob/tcp/oob_tcp_peer.h new file mode 100644 index 0000000000..d7cb1881ae --- /dev/null +++ b/src/mca/oob/tcp/oob_tcp_peer.h @@ -0,0 +1,86 @@ +/* + * $HEADER$ + */ +/** @file: + * + * Contains the data structure which describes each connection + */ + +#ifndef _MCA_OOB_TCP_PEER_H_ +#define _MCA_OOB_TCP_PEER_H_ + +#include "mca/ns/ns.h" +#include "class/ompi_list.h" +#include +#include "threads/mutex.h" +/** + * the state of the connection + */ +typedef enum { + MCA_OOB_TCP_CLOSED, + MCA_OOB_TCP_CONNECTING, + MCA_OOB_TCP_CONNECT_ACK, + MCA_OOB_TCP_CONNECTED, + MCA_OOB_TCP_FAILED +} mca_oob_tcp_state_t; + + +/** + * tcp interface + */ +struct mca_oob_tcp_addr_t { + int ifindex; /**< oob interface index */ + struct sockaddr_in ifaddr; /**< oob interface address */ + struct sockaddr_in ifmask; /**< oob interface netmask */ +}; +typedef struct mca_oob_tcp_addr_t mca_oob_tcp_addr_t; + +/** + * This structire describes a peer + */ +struct mca_oob_tcp_peer_t { + ompi_list_item_t super; /**< allow this to be on a list */ + ompi_process_name_t peer_name; /**< the name of the peer */ + mca_oob_tcp_state_t peer_state; /**< the state of the connection */ + int peer_sd; /**< socket descriptor of the connection */ + mca_oob_tcp_addr_t peer_addr; /**< the address of the peer process */ + ompi_mutex_t peer_lock; /**< make sure only one thread accesses it at a time */ + ompi_list_t peer_send; /**< list of items to send */ + ompi_list_t peer_recieved; /**< list of items to recieve */ +}; +typedef struct mca_oob_tcp_peer_t mca_oob_tcp_peer_t; + +/** + * The function to compare 2 peers. Used for the rb tree + * + * @param peer1 the first peer + * @param peer2 the second peer + * + * @retval <0 if peer1 < peer2 + * @retval >0 if peer1 > peer2 + * @retval 0 if peer1 == peer2 + */ +int mca_oob_tcp_peer_comp(void * key1, void * key2); + +/** + * Creates a peer structure and adds to the tree and list. + * + * @param peer_name the name of the peer + * + * @retval pointer to the newly created struture + * @retval NULL if there was a problem + */ +mca_oob_tcp_peer_t * mca_oob_tcp_add_peer(ompi_process_name_t peer_name); + +/** + * Deletes a peer structure from the tree and lists and frees its memory + * + * @param peer_name the name of the peer + * + * @retval OMPI_SUCCESS + * @retval OMPI_ERROR + */ +int mca_oob_tcp_del_peer(ompi_process_name_t peer_name); + +#endif /* _MCA_OOB_TCP_PEER_H */ + diff --git a/src/mca/oob/tcp/oob_tcp_recv.c b/src/mca/oob/tcp/oob_tcp_recv.c new file mode 100644 index 0000000000..085f7e5f12 --- /dev/null +++ b/src/mca/oob/tcp/oob_tcp_recv.c @@ -0,0 +1,39 @@ +#include "mca/oob/tcp/oob_tcp.h" + +/* + * Similiar to unix recv(2) + * + * @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param types (IN) Parallel array to iovecs describing data type of each iovec element. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) May be OOB_PEEK to return up to the number of bytes provided in the + * iovec array without removing the message from the queue. + * @return OMPI error code (<0) on error or number of bytes actually received. + */ + +int mca_oob_tcp_recv(ompi_process_name_t* peer, const struct iovec *msg, int count, int flags) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +} + +/* + * Non-blocking version of mca_oob_recv(). + * + * @param peer (IN) Opaque name of peer process or OOB_NAME_ANY for wildcard receive. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) May be OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, + * @param cbfunc (IN) Callback function on recv completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error or number of bytes actually received. + */ + +int mca_oob_tcp_recv_nb(ompi_process_name_t* peer, const struct iovec* msg, int count, int flags, + mca_oob_callback_fn_t cbfunc, void* cbdata) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +} + + + diff --git a/src/mca/oob/tcp/oob_tcp_send.c b/src/mca/oob/tcp/oob_tcp_send.c new file mode 100644 index 0000000000..112f9f632d --- /dev/null +++ b/src/mca/oob/tcp/oob_tcp_send.c @@ -0,0 +1,36 @@ +#include "mca/oob/tcp/oob_tcp.h" + +/* + * Similiar to unix send(2). + * + * @param peer (IN) Opaque name of peer process. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) Currently unused. + * @return OMPI error code (<0) on error number of bytes actually sent. + */ + +int mca_oob_tcp_send(const ompi_process_name_t* peer, const struct iovec *msg, int count, int flags) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +} + +/* + * Non-blocking version of mca_oob_send(). + * + * @param peer (IN) Opaque name of peer process. + * @param msg (IN) Array of iovecs describing user buffers and lengths. + * @param count (IN) Number of elements in iovec array. + * @param flags (IN) Currently unused. + * @param cbfunc (IN) Callback function on send completion. + * @param cbdata (IN) User data that is passed to callback function. + * @return OMPI error code (<0) on error number of bytes actually sent. + * + */ + +int mca_oob_tcp_send_nb(const ompi_process_name_t* peer, const struct iovec* msg, int count, + int flags, mca_oob_callback_fn_t cbfunc, void* cbdata) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +} + diff --git a/src/mca/pcm/cofs/Makefile.am b/src/mca/pcm/cofs/Makefile.am index 92ec30ac33..b1532c9b80 100644 --- a/src/mca/pcm/cofs/Makefile.am +++ b/src/mca/pcm/cofs/Makefile.am @@ -1,8 +1,9 @@ +# -*- makefile -*- # # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +11,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_pcm_cofs_DSO -component_noinst = -component_install = mca_pcm_cofs.la +if OMPI_BUILD_pcm_cofs_LOADABLE_MODULE +module_noinst = +module_install = mca_pcm_cofs.la else -component_noinst = libmca_pcm_cofs.la -component_install = +module_noinst = libmca_pcm_cofs.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_pcm_cofs_la_SOURCES = -mca_pcm_cofs_la_LIBADD = src/libmca_pcm_cofs.la $(LIBOMPI_LA) +mca_pcm_cofs_la_LIBADD = \ + src/libmca_pcm_cofs.la \ + $(LIBOMPI_LA) mca_pcm_cofs_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_pcm_cofs_la_SOURCES = libmca_pcm_cofs_la_LIBADD = src/libmca_pcm_cofs.la libmca_pcm_cofs_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/pcm/cofs/src/pcm_cofs.c b/src/mca/pcm/cofs/src/pcm_cofs.c index db7d4cc149..f7dc50877b 100644 --- a/src/mca/pcm/cofs/src/pcm_cofs.c +++ b/src/mca/pcm/cofs/src/pcm_cofs.c @@ -12,191 +12,21 @@ #include #include -#include #include #include -#define HANDLE_FILE_NAME "pcm_cofs_handle_list" - -static int handle_new_count = 0; - int -mca_pcm_cofs_query_get_nodes(mca_pcm_rte_node_t **nodes, size_t * nodes_len, - int *available_procs) +mca_pcm_cofs_get_peers(ompi_process_name_t **procs, size_t *num_procs) { - *nodes = NULL; - *nodes_len = 0; - *available_procs = 0; - - return OMPI_ERR_NOT_SUPPORTED; -} - - -ompi_job_handle_t -mca_pcm_cofs_handle_new(ompi_job_handle_t parent) -{ - pid_t pid; - char *ret; - size_t ret_len; - - /* should really make this a file lookup kind of thing */ - pid = getpid(); - - ret_len = sizeof(pid_t) * 8 + strlen("pcm_cofs_job_handle") + sizeof(int) * 8 + 5; - ret = malloc(ret_len); - if (ret == NULL) { - return NULL; - } - - snprintf(ret, ret_len, "pcm_cofs_job_handle_%d_%d", (int) pid, handle_new_count); - handle_new_count++; - - return ret; -} - - -ompi_job_handle_t -mca_pcm_cofs_handle_get(void) -{ - return mca_pcm_cofs_my_handle; -} - - -void -mca_pcm_cofs_handle_free(ompi_job_handle_t * job_handle) -{ - if (*job_handle == mca_pcm_cofs_my_handle) { - printf("WARNING: attempting to free static internal job handle!\n"); - printf(" Did you perhaps try to free the return from handle_get()?\n"); - } else if (*job_handle != NULL) { - free(*job_handle); - *job_handle = NULL; - } -} - - -int -mca_pcm_cofs_job_can_spawn(ompi_job_handle_t job_handle) -{ -#if 1 - /* Currently, have not coded up spawning support. Need to do - so soon */ - return OMPI_ERR_NOT_SUPPORTED; -#else - if (job_handle != NULL) { - return OMPI_ERR_NOT_SUPPORTED; - } else { - return OMPI_SUCCESS; - } -#endif -} - - -int -mca_pcm_cofs_job_set_arguments(ompi_job_handle_t job_handle, - mca_pcm_control_args_t * opts, - size_t opts_len) -{ - /* need to implement, but not needed to get INIT going */ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_cofs_job_launch_procs(ompi_job_handle_t job_handle, - mca_pcm_rte_node_t *nodes, - size_t nodes_len, const char *file, - int argc, const char *argv[], - const char *env[]) -{ - /* need to implement, but not needed to get INIT going */ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_cofs_job_rendezvous(ompi_job_handle_t job_handle) -{ - /* need to implement, but not needed to get INIT going */ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_cofs_job_wait(ompi_job_handle_t job_handle) -{ - /* need to implement, but not needed to get INIT going */ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_cofs_job_running(ompi_job_handle_t job_handle, - int *running) -{ - /* need to implement, but not needed to get INIT going */ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_cofs_job_list_running(ompi_job_handle_t ** handles, - size_t handles_len) -{ - /* need to implement, but not needed to get INIT going */ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_cofs_proc_startup(void) -{ - int i; - - if (mca_pcm_cofs_nprocs == 0) { - /* well, this really shouldn't happen - we know we have at least ourselves */ - return OMPI_ERR_FATAL; - } - - mca_pcm_cofs_procs = malloc(sizeof(mca_pcm_proc_t) * mca_pcm_cofs_nprocs); - if (mca_pcm_cofs_procs == NULL) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - for (i = 0 ; i < mca_pcm_cofs_nprocs ; ++i) { - /* for now, assume everyone in the same job :( */ - mca_pcm_cofs_procs[i].job_handle = mca_pcm_cofs_handle_get(); - mca_pcm_cofs_procs[i].vpid = i; - } - - return OMPI_SUCCESS; -} - - -int -mca_pcm_cofs_proc_get_peers(mca_pcm_proc_t **procs, size_t *nprocs) -{ - if (mca_pcm_cofs_procs == NULL) { - return OMPI_ERROR; - } - + *num_procs = mca_pcm_cofs_num_procs; *procs = mca_pcm_cofs_procs; - *nprocs = mca_pcm_cofs_nprocs; - return OMPI_SUCCESS; } -mca_pcm_proc_t* -mca_pcm_cofs_proc_get_me(void) +ompi_process_name_t* mca_pcm_cofs_get_self(void) { - return &(mca_pcm_cofs_procs[mca_pcm_cofs_my_vpid]); + return &mca_pcm_cofs_procs[mca_pcm_cofs_procid]; } - -int -mca_pcm_cofs_proc_get_parent(void) -{ - return OMPI_ERR_NOT_IMPLEMENTED; -} diff --git a/src/mca/pcm/cofs/src/pcm_cofs.h b/src/mca/pcm/cofs/src/pcm_cofs.h index cfbb766c8d..5472ff9b39 100644 --- a/src/mca/pcm/cofs/src/pcm_cofs.h +++ b/src/mca/pcm/cofs/src/pcm_cofs.h @@ -17,48 +17,17 @@ int mca_pcm_cofs_close(void); /* * Startup / Shutdown */ -struct mca_pcm_1_0_0_t* mca_pcm_cofs_init(int *priority, - bool *allow_multi_user_threads, - bool *have_hidden_threads); +struct mca_pcm_1_0_0_t* mca_pcm_cofs_init(int *priority, bool *allow_multi_user_threads, bool *have_hidden_threads); int mca_pcm_cofs_finalize(void); /* * "Action" functions */ -int mca_pcm_cofs_query_get_nodes(mca_pcm_rte_node_t **nodes, size_t *nodes_len, - int *available_procs); +int mca_pcm_cofs_get_peers(ompi_process_name_t **peers, size_t *npeers); +ompi_process_name_t* mca_pcm_cofs_get_self(void); -ompi_job_handle_t mca_pcm_cofs_handle_new(ompi_job_handle_t parent); -ompi_job_handle_t mca_pcm_cofs_handle_get(void); -void mca_pcm_cofs_handle_free(ompi_job_handle_t *job_handle); - -int mca_pcm_cofs_job_can_spawn(ompi_job_handle_t job_handle); -int mca_pcm_cofs_job_set_arguments(ompi_job_handle_t job_handle, - mca_pcm_control_args_t* opts, - size_t opts_len); -int mca_pcm_cofs_job_launch_procs(ompi_job_handle_t job_handle, - mca_pcm_rte_node_t *nodes, - size_t nodes_len, const char* file, - int argc, const char* argv[], - const char *env[]); -int mca_pcm_cofs_job_rendezvous(ompi_job_handle_t job_handle); -int mca_pcm_cofs_job_wait(ompi_job_handle_t job_handle); -int mca_pcm_cofs_job_running(ompi_job_handle_t job_handle, - int* running); -int mca_pcm_cofs_job_list_running(ompi_job_handle_t **handles, - size_t handles_len); - -int mca_pcm_cofs_proc_startup(void); -int mca_pcm_cofs_proc_get_peers(mca_pcm_proc_t **procs, size_t *nprocs); -mca_pcm_proc_t* mca_pcm_cofs_proc_get_me(void); -int mca_pcm_cofs_proc_get_parent(void); - -extern char mca_pcm_cofs_comm_loc[OMPI_PATH_MAX]; /* location for file drop-off */ - -extern int mca_pcm_cofs_my_vpid; -extern char *mca_pcm_cofs_my_handle; - -extern mca_pcm_proc_t *mca_pcm_cofs_procs; -extern size_t mca_pcm_cofs_nprocs; +extern ompi_process_name_t *mca_pcm_cofs_procs; +extern size_t mca_pcm_cofs_num_procs; +extern size_t mca_pcm_cofs_procid; diff --git a/src/mca/pcm/cofs/src/pcm_cofs_module.c b/src/mca/pcm/cofs/src/pcm_cofs_module.c index 0db6273ff9..f019eb515c 100644 --- a/src/mca/pcm/cofs/src/pcm_cofs_module.c +++ b/src/mca/pcm/cofs/src/pcm_cofs_module.c @@ -8,9 +8,11 @@ #include "include/constants.h" #include "include/types.h" +#include "util/output.h" #include "mca/mca.h" #include "mca/pcm/pcm.h" #include "mca/pcm/cofs/src/pcm_cofs.h" +#include "mca/base/mca_base_param.h" #include #include @@ -40,39 +42,29 @@ mca_pcm_base_module_1_0_0_t mca_pcm_cofs_module = { struct mca_pcm_1_0_0_t mca_pcm_cofs_1_0_0 = { - mca_pcm_cofs_query_get_nodes, - - mca_pcm_cofs_handle_new, - mca_pcm_cofs_handle_get, - mca_pcm_cofs_handle_free, - - mca_pcm_cofs_job_can_spawn, - mca_pcm_cofs_job_set_arguments, - mca_pcm_cofs_job_launch_procs, - mca_pcm_cofs_job_rendezvous, - mca_pcm_cofs_job_wait, - mca_pcm_cofs_job_running, - mca_pcm_cofs_job_list_running, - - mca_pcm_cofs_proc_startup, - mca_pcm_cofs_proc_get_peers, - mca_pcm_cofs_proc_get_me, - mca_pcm_cofs_proc_get_parent + mca_pcm_cofs_get_peers, + mca_pcm_cofs_get_self }; char mca_pcm_cofs_comm_loc[OMPI_PATH_MAX]; -int mca_pcm_cofs_my_vpid = -1; char *mca_pcm_cofs_my_handle = NULL; -mca_pcm_proc_t *mca_pcm_cofs_procs = NULL; -size_t mca_pcm_cofs_nprocs = 0; +ompi_process_name_t *mca_pcm_cofs_procs = NULL; +size_t mca_pcm_cofs_num_procs = 0; +size_t mca_pcm_cofs_procid = 0; +static int mca_pcm_cofs_num_procs_param; +static int mca_pcm_cofs_cellid_param; +static int mca_pcm_cofs_jobid_param; +static int mca_pcm_cofs_procid_param; int mca_pcm_cofs_open(void) { - /* JMS/BWB: Register MCA params in here -- see - src/mca/base/mca_base_param.h */ - return OMPI_SUCCESS; + mca_pcm_cofs_num_procs_param = mca_base_param_register_int("pcm","cofs","num_procs",NULL,-1); + mca_pcm_cofs_cellid_param = mca_base_param_register_int("pcm","cofs","cellid",NULL,-1); + mca_pcm_cofs_jobid_param = mca_base_param_register_int("pcm","cofs","jobid",NULL,-1); + mca_pcm_cofs_procid_param = mca_base_param_register_int("pcm","cofs","procid",NULL,-1); + return OMPI_SUCCESS; } @@ -83,94 +75,59 @@ mca_pcm_cofs_close(void) } -struct mca_pcm_1_0_0_t* +struct mca_pcm_1_0_0_t * mca_pcm_cofs_init(int *priority, bool *allow_multi_user_threads, bool *have_hidden_threads) { - char *tmp; - FILE *fp; + int i,value; + ompi_process_id_t cellid; + ompi_process_id_t jobid; - char *test_ret; + *priority = 1; + *allow_multi_user_threads = true; + *have_hidden_threads = false; - *priority = 1; - *allow_multi_user_threads = true; - *have_hidden_threads = false; - - /* BWB - remove printfs once things settle down some... */ - /* JMS: Look in src/mca/base/mca_base_param.h */ - test_ret = getenv("MCA_common_ompi_cofs_my_vpid"); - if (test_ret == NULL) { - printf("COFS PCM will not be running because MCA_common_ompi_cofs_my_vpid not set\n"); - return NULL; - } - - test_ret = getenv("MCA_common_ompi_cofs_job_handle"); - if (test_ret == NULL) { - printf("COFS PCM will not be running because MCA_common_ompi_cofs_job_handle not set\n"); - return NULL; - } - - test_ret = getenv("MCA_common_ompi_cofs_num_procs"); - if (test_ret == NULL) { - printf("COFS PCM will not be running because MCA_common_ompi_cofs_num_procs not set\n"); - return NULL; - } - - /* - * BWB - fix me, make register the "right" way... - */ - tmp = getenv("MCA_common_ompi_cofs_comm_dir"); - if (tmp == NULL) { - /* make it $HOME */ - tmp = getenv("HOME"); - if (tmp == NULL) { - printf("pcm_cofs can not find communication dir (MCA_common_ompi_cofs_comm_dir)\n"); - return NULL; + /* lookup parameters for local name */ + mca_base_param_lookup_int(mca_pcm_cofs_num_procs_param, &value); + if(value <= 0) { + ompi_output(0, "mca_pcm_cofs_init: missing/invalid value for OMPI_MCA_pcm_cofs_num_procs\n"); + return NULL; } - snprintf(mca_pcm_cofs_comm_loc, OMPI_PATH_MAX, "%s/cofs", tmp); - } else { - snprintf(mca_pcm_cofs_comm_loc, OMPI_PATH_MAX, "%s", tmp); - } + mca_pcm_cofs_num_procs = value; - /* - * BWB - fix me, make register the "right" way... - */ - /* find our vpid */ - tmp = getenv("MCA_common_ompi_cofs_my_vpid"); - if (tmp == NULL) { - printf("pcm_cofs can not find vpid\n"); - return NULL; - } - mca_pcm_cofs_my_vpid = atoi(tmp); + mca_base_param_lookup_int(mca_pcm_cofs_cellid_param, &value); + if(value < 0) { + ompi_output(0, "mca_pcm_cofs_init: missing/invalid value for OMPI_MCA_pcm_cofs_cellid\n"); + return NULL; + } + cellid = value; + + mca_base_param_lookup_int(mca_pcm_cofs_jobid_param, &value); + if(value < 0) { + ompi_output(0, "mca_pcm_cofs_init: missing/invalid value for OMPI_MCA_pcm_cofs_jobid\n"); + return NULL; + } + jobid = value; + + mca_base_param_lookup_int(mca_pcm_cofs_procid_param, &value); + if(value < 0) { + ompi_output(0, "mca_pcm_cofs_init: missing value for OMPI_MCA_pcm_cofs_procid\n"); + return NULL; + } + mca_pcm_cofs_procid = value; - /* - * See if we can write in our directory... - */ - tmp = malloc(strlen(mca_pcm_cofs_comm_loc) + 32); - if (tmp == NULL) return NULL; - sprintf(tmp, "%s/pcm.%d", mca_pcm_cofs_comm_loc, mca_pcm_cofs_my_vpid); - fp = fopen(tmp, "w"); - if (fp == NULL) { - printf("pcm_cofs can not write in communication dir\n"); - free(tmp); - return NULL; - } - fclose(fp); - unlink(tmp); - free(tmp); + mca_pcm_cofs_procs = (ompi_process_name_t*)malloc(sizeof(ompi_process_name_t) * mca_pcm_cofs_num_procs); + if(NULL == mca_pcm_cofs_procs) { + ompi_output(0, "mca_pcm_cofs_init: missing value for OMPI_MCA_pcm_cofs_num_procs\n"); + return NULL; + } - mca_pcm_cofs_my_handle = getenv("MCA_common_ompi_cofs_job_handle"); - - mca_pcm_cofs_procs = NULL; - - tmp = getenv("MCA_common_ompi_cofs_num_procs"); - if (tmp == NULL) { - printf("pcm_cofs can not find nprocs\n"); - return NULL; - } - mca_pcm_cofs_nprocs = atoi(tmp); - - return &mca_pcm_cofs_1_0_0; + for(i=0; i -/* - * "PCM" global types - */ -#define OMPI_PCM_PROC_MPIRUN 0 -#define OMPI_PCM_PROC_MPIAPP 1 -#define OMPI_PCM_PROC_OTHER 2 +typedef int (*mca_pcm_base_get_peers_fn_t)(ompi_process_name_t **peers, size_t *npeers); -struct mca_pcm_rte_node_t { - char name[MAXHOSTNAMELEN]; - int32_t node_num; - int32_t num_procs; -}; -typedef struct mca_pcm_rte_node_t mca_pcm_rte_node_t; +typedef ompi_process_name_t* (*mca_pcm_base_get_self_fn_t)(void); -struct mca_pcm_control_args_t { - char* request; - char* value; -}; -typedef struct mca_pcm_control_args_t mca_pcm_control_args_t; - -struct mca_pcm_proc_t { - ompi_job_handle_t job_handle; - int vpid; -}; -typedef struct mca_pcm_proc_t mca_pcm_proc_t; - -/* - * functions every module must provide - */ - -typedef struct mca_pcm_1_0_0_t* - (*mca_pcm_base_init_fn_t)(int *priority, bool *allow_multi_user_threads, +typedef struct mca_pcm_1_0_0_t * + (*mca_pcm_base_init_fn_t)(int *priority, bool *allow_multi_user_threads, bool *have_hidden_threads); - /** - * \func mca_pcm_query_get_nodes - * - * Get list of nodes available for execution - * - * @param nodes Pointer that will contain array of nodes available - * @param nodes_len Length of nodes array - * @param available_procs Number of available processors in the RTE - * - * @retval OMPI_SUCCESS success - * @retval OMPI_NOT_SUPPORTED Not available - * - * Obtain a list of nodes available for execution. No promises are - * made that such information is available - for some environments - * (Quadrics/RMS, etc) nothing is really known about the environment - * until you ask for it. If a node/host mapping is unavailable, - * *nodes will be NULL and nodes_len will be 0. If the total number - * of available processors for MPI applications is not available, it - * will be set to -1 and the function will return "Not available". - * In the case where both are available, available_procs will be - * equal to the sum of nodes[0...n].num_procs. - */ -typedef int (*mca_pcm_base_query_get_nodes_fn_t)(mca_pcm_rte_node_t **nodes, - size_t *nodes_len, - int *available_procs); - - - /** - * Get new parallel job handle - * - * @param parent Parent job handle (NULL if not in parallel job) - * - * @retval NULL failure - * @retval non-NULL sucess - * - * The run-time environment tracks MPI applications through a - * parallel job handle, which is a char* string. This string can be - * used to request information about the status of a currently - * running job, kill the job, etc. - * - * The parent parameter allows the run-time system to provide a - * process tree (spawn, etc.) if the user really wants such - * information. For mpirun, it should just be NULL. - * - * \warning The handle must be released using mca_pcm_handle_free - */ -typedef ompi_job_handle_t (*mca_pcm_base_handle_new_fn_t)(ompi_job_handle_t parent); - - - /** - * Get my parallel job handle - * - * @retval NULL failure - environment not properly initialized - * @retval non-NULL success - * - * Return the parallel job handle for the currently running process - * - * \warning The handle must be released using mca_pcm_handle_free - */ -typedef ompi_job_handle_t (*mca_pcm_base_handle_get_fn_t)(void); - - - /** - * Free a job handle - * - * @param job_handle Poiner to a ompi_job_handle_t - * - * Free a job handle returned by mca_pcm_handle_new or - * mca_pcm_handle_get. - */ -typedef void (*mca_pcm_base_handle_free_fn_t)(ompi_job_handle_t *job_handle); - - - /** - * Ask if mca module can spawn processes - * - * @param job_handle Parallel job handle of running process - * - * @retval OMPI_SUCCESS OMPI can spawn more jobs - * @retval OMPI_NOT_SUPPORTED OMPI can not spawn more jobs - * - * Ask the currently running mca module for the runtime environment - * if it supports spawning more processes. This question should - * always return OMPI_SUCCESS (yes) if called from mpirun. Useful - * for asking if MPI_SPAWN and friends can run. - */ -typedef int (*mca_pcm_base_job_can_spawn_fn_t)(ompi_job_handle_t job_handle); - - - /** - * Configure arguments for the parallel job to be started - * - * @param job_handle Parallel job handle to configure - * @param opts Array of key=value structures requesting job behaviour - * @param opts_len Length of opts array - * - * @retval OMPI_SUCCESS Sucess - * @retval OMPI_ERROR Unkonwn failure - * - * Configure the job using key=value arguments. The meanings of the - * arguments are up to the specific mca module providing run-time support. - * - * Common key values will be provided here once MCAs begin to use - * this function. The only existing module no-ops this entire - * function. - * - * \Warning It is an error to call this function more than once on a single - * job handle. - */ -typedef int (*mca_pcm_base_job_set_arguments_fn_t)(ompi_job_handle_t job_handle, - mca_pcm_control_args_t* opts, - size_t opts_len); - - - /** - * Launch processes across parallel job - * - * @param job_handle Parallel job handle within which to launch processes - * @param nodes Array of nodes structures describing targets of launch - * @param nodes_len Length of nodes - * @param file Process to laucnh (does not have to be equal to argv[0]) - * @param argc Length of argv - * @param argv Argv array for launched processes - * @param env Environment array for launched process. See note below - * - * @retval OMPI_SUCCESS Success - * @retval OMPI_ERR_RESOURCE_BUSY Try again real soon now - * @retval OMPI_ERR_NOT_SUPPORTED non-MPIRUN process can not spawn jobs - * @retval OMPI_FAILURE Unkonwn failure - * - * Launch num_procs nodes[?].processes on nodes[?].node_num for each - * nodes entry, as part of job_handle's job. The env array should - * contain any environment variables that should be pushed to the - * remote processes. The mca may provide a more detailed - * environment if necessary (bporc, etc.). - * - * OMPI_ERR_NOT_SUPPORTED will be returned if the mca module does not - * support spawning of new applications from - */ -typedef int (*mca_pcm_base_job_launch_procs_fn_t)(ompi_job_handle_t job_handle, - mca_pcm_rte_node_t *nodes, - size_t nodes_len, const char* file, - int argc, const char* argv[], - const char *env[]); - - - /** - * Do rendezvous duties after launching parallel job - * - * @param job_handle Parallel job handle to run through startup - * - * @retval OMPI_SUCCESS Success - * @retval OMPI_FAILURE Unknown failure - * - * Do the civic duties required to complete the rendezvous part of - * the startup protocol. After this, the MPI application should - * know who all its neighbors are. It is, of course, completely - * possible that the MCA module has been in the background doing - * this all along and didn't bother to tell you. When this function - * returns, it is safe to assume that all rendezvous is complete - * (ie, you can exit and not mess anything up - * - * This function only needs to be called by the launching procs. - */ -typedef int (*mca_pcm_base_job_rendezvous_fn_t)(ompi_job_handle_t job_handle); - - - /** - * Wait for job completion - * - * @param job_handle Parallel job handle to wait on - * - * @retval OMPI_SUCCESS Success - * @retval OMPI_ERR_INTERUPTED Interupted (due to signal, etc.) - * - * The OMPI parallel version of "wait". It is not required to wait - * on a job at termination, as job results will be expunged over - * time as resource limits dictate. - */ -typedef int (*mca_pcm_base_job_wait_fn_t)(ompi_job_handle_t job_handle); - - - /** - * Request job status - * - * @param job_handle Parallel job handle to query - * @param running Job is running, if true - * - * @retval OMPI_SUCCESS Success - * @retval OMPI_ERR_BAD_PARAM Invalid job handle - * - * Ask if job is running. If job has recently finished, this does - * not imply wait the pcm interface will call wait for you. - */ -typedef int (*mca_pcm_base_job_running_fn_t)(ompi_job_handle_t job_handle, - int* running); - - - /** - * Request list of job handles running in current environment - * - * @param handles Pointer to job handles array - * @param handles_len length of handles array - * - * @retval OMPI_ERR_NOT_IMPLEMENTED Not implemented - * - * Query the environment about currently running jobs. Intended for - * applications outside MPI and mpirun, to be user friendly and all - * those things. mca modules are not required to support this function. - * - * \warning This function is not yet implemented. - */ -typedef int (*mca_pcm_base_job_list_running_fn_t)(ompi_job_handle_t **handles, - size_t handles_len); - - - /** - * Do process startup code - * - * @retval OMPI_SUCCESS Success - * @retval OMPI_ERR_FATAL Fatal error occurred - * @retval OMPI_ERROR Unkonwn failure - * - * Do all communication work required to get peer list and establish - * the out of band communictaion mechanism. If a pcm interface uses - * fork()/exec() to start other processes on the current node, it - * should do so and complete all rendezvous before returning from - * this function. - * - * The mca module is free to start the oob interface as soon as it - * as provided the oob interface enough information to do so (tight - * integration with the oob mca module is probably required to meet - * this constraint). - */ -typedef int (*mca_pcm_base_proc_startup_fn_t)(void); - - - /** - * Get peers list - * - * @param procs Ordered array of ompi_proc_t entries describing the job peers - * - * @retval OMPI_SUCCESS success - * @retval OMPI_ERROR Unknown error - * - * Get list of peers in the parallel job. Should not require any - * communication with other nodes (communication with processes on - * this node are allowed). - * - * \warning This function is not implemented and its argument list - * may change in the very near future. - */ -typedef int (*mca_pcm_base_proc_get_peers_fn_t)(mca_pcm_proc_t **procs, size_t *nprocs); - - - /** - * Get my entry in the peers list - * - * @retval OMPI_ERR_NOT_IMPLEMENTED Function not implemented - * - * Get my entry in the peers list - * - */ -typedef mca_pcm_proc_t* (*mca_pcm_base_proc_get_me_fn_t)(void); - - /** - * Get my entry in the peers list - * - * @retval OMPI_ERR_NOT_IMPLEMENTED Function not implemented - * - * Get my entry in the peers list - * - * \warning This function is not implemented and its argument list - * will obviously change in the very near future. - */ -typedef int (*mca_pcm_base_proc_get_parent_fn_t)(void); - typedef int (*mca_pcm_base_finalize_fn_t)(void); - - -/* - * Ver 1.0.0 - */ + +/* Ver 1.0.0 */ struct mca_pcm_base_module_1_0_0_t { mca_base_module_t pcmm_version; mca_base_module_data_1_0_0_t pcmm_data; - mca_pcm_base_init_fn_t pcmm_init; mca_pcm_base_finalize_fn_t pcmm_finalize; }; typedef struct mca_pcm_base_module_1_0_0_t mca_pcm_base_module_1_0_0_t; struct mca_pcm_1_0_0_t { - mca_pcm_base_query_get_nodes_fn_t pcm_query_get_nodes; - - mca_pcm_base_handle_new_fn_t pcm_handle_new; - mca_pcm_base_handle_get_fn_t pcm_handle_get; - mca_pcm_base_handle_free_fn_t pcm_handle_free; - - mca_pcm_base_job_can_spawn_fn_t pcm_job_can_spawn; - mca_pcm_base_job_set_arguments_fn_t pcm_job_set_arguments; - mca_pcm_base_job_launch_procs_fn_t pcm_job_launch_procs; - mca_pcm_base_job_rendezvous_fn_t pcm_job_rendezvous; - mca_pcm_base_job_wait_fn_t pcm_job_wait; - mca_pcm_base_job_running_fn_t pcm_job_running; - mca_pcm_base_job_list_running_fn_t pcm_job_list_running; - - mca_pcm_base_proc_startup_fn_t pcm_proc_startup; - mca_pcm_base_proc_get_peers_fn_t pcm_proc_get_peers; - mca_pcm_base_proc_get_me_fn_t pcm_proc_get_me; - mca_pcm_base_proc_get_parent_fn_t pcm_proc_get_parent; + mca_pcm_base_get_peers_fn_t pcm_peers; + mca_pcm_base_get_self_fn_t pcm_self; }; typedef struct mca_pcm_1_0_0_t mca_pcm_1_0_0_t; @@ -418,8 +96,8 @@ typedef struct mca_pcm_1_0_0_t mca_pcm_1_0_0_t; /* pcm v1.0 */ \ "pcm", 1, 0, 0 -typedef mca_pcm_base_module_1_0_0_t mca_pcm_base_module_t; -typedef mca_pcm_1_0_0_t mca_pcm_t; +typedef struct mca_pcm_base_module_1_0_0_t mca_pcm_base_module_t; +typedef struct mca_pcm_1_0_0_t mca_pcm_t; /* * Global functions for MCA overall collective open and close diff --git a/src/mca/pcm/rsh/Makefile.am b/src/mca/pcm/rsh/Makefile.am index 214271b914..3c7dfb537f 100644 --- a/src/mca/pcm/rsh/Makefile.am +++ b/src/mca/pcm/rsh/Makefile.am @@ -1,8 +1,9 @@ +# -*- makefile -*- # # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +11,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_pcm_rsh_DSO -component_noinst = -component_install = mca_pcm_rsh.la +if OMPI_BUILD_pcm_rsh_LOADABLE_MODULE +module_noinst = +module_install = mca_pcm_rsh.la else -component_noinst = libmca_pcm_rsh.la -component_install = +module_noinst = libmca_pcm_rsh.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_pcm_rsh_la_SOURCES = -mca_pcm_rsh_la_LIBADD = src/libmca_pcm_rsh.la $(LIBOMPI_LA) +mca_pcm_rsh_la_LIBADD = \ + src/libmca_pcm_rsh.la \ + $(LIBOMPI_LA) mca_pcm_rsh_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_pcm_rsh_la_SOURCES = libmca_pcm_rsh_la_LIBADD = src/libmca_pcm_rsh.la libmca_pcm_rsh_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/pcm/rsh/src/pcm_rsh.c b/src/mca/pcm/rsh/src/pcm_rsh.c index 0867f6e6a8..5933a0d1e1 100644 --- a/src/mca/pcm/rsh/src/pcm_rsh.c +++ b/src/mca/pcm/rsh/src/pcm_rsh.c @@ -16,159 +16,3 @@ #include #include -static const char *handle_base = "pcm_rsh_job"; -static unsigned long handle_count = 0; -static int have_connected = 0; -static int have_launched = 0; - -int -mca_pcm_rsh_query_get_nodes(mca_pcm_rte_node_t **nodes, size_t * nodes_len, - int *available_procs) -{ - return OMPI_ERR_NOT_SUPPORTED; -} - - -ompi_job_handle_t -mca_pcm_rsh_handle_new(ompi_job_handle_t parent) -{ - char *new_handle = NULL; - int mypid = 0; - char hostname[MAXHOSTNAMELEN]; - size_t handle_len = strlen(handle_base) + strlen("___") + - (8 * sizeof(int) * 2) + MAXHOSTNAMELEN; - - /* we don't support spawning just yet */ - if (parent != NULL) return NULL; - - gethostname(hostname, MAXHOSTNAMELEN); - - new_handle = (char*) malloc(handle_len); - if (new_handle == NULL) return new_handle; - - snprintf(new_handle, handle_len, "%s_%s_%d_%lu", handle_base, hostname, - mypid, handle_count++); - return new_handle; -} - - -ompi_job_handle_t -mca_pcm_rsh_handle_get(void) -{ - return mca_pcm_rsh_my_job_handle; -} - - -void -mca_pcm_rsh_handle_free(ompi_job_handle_t * job_handle) -{ - if (*job_handle != NULL) { - free(*job_handle); - *job_handle = NULL; - } -} - - -int -mca_pcm_rsh_job_can_spawn(ompi_job_handle_t job_handle) -{ - if (mca_pcm_rsh_handle_get() != NULL) { - /* no support for spawned jobs */ - return OMPI_ERR_NOT_SUPPORTED; - } else { - return OMPI_SUCCESS; - } -} - - -int -mca_pcm_rsh_job_set_arguments(ompi_job_handle_t job_handle, - mca_pcm_control_args_t * opts, - size_t opts_len) -{ - if (have_launched != 0) return OMPI_ERROR; - - - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_rsh_job_launch_procs(ompi_job_handle_t job_handle, - mca_pcm_rte_node_t *nodes, - size_t nodes_len, const char *file, - int argc, const char *argv[], - const char *env[]) -{ - if (have_connected != 0) return OMPI_ERROR; - - have_launched = 1; - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_rsh_job_rendezvous(ompi_job_handle_t job_handle) -{ - if (have_connected != 0 || have_launched == 0) return OMPI_ERROR; - - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_rsh_job_wait(ompi_job_handle_t job_handle) -{ - if (have_connected == 0 || have_launched == 0) return OMPI_ERROR; - - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_rsh_job_running(ompi_job_handle_t job_handle, - int *running) -{ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_rsh_job_list_running(ompi_job_handle_t ** handles, - size_t handles_len) -{ - return OMPI_ERR_NOT_IMPLEMENTED; -} - - -int -mca_pcm_rsh_proc_startup(void) -{ - if (have_connected != 0) return OMPI_ERROR; - have_connected = 1; - - return OMPI_SUCCESS; -} - - -int -mca_pcm_rsh_proc_get_peers(mca_pcm_proc_t **procs, size_t *nprocs) -{ - if (have_connected == 0) return OMPI_ERROR; - - return OMPI_SUCCESS; -} - - -mca_pcm_proc_t* -mca_pcm_rsh_proc_get_me(void) -{ - return NULL; -} - - -int -mca_pcm_rsh_proc_get_parent(void) -{ - return OMPI_ERR_NOT_IMPLEMENTED; -} diff --git a/src/mca/pcm/rsh/src/pcm_rsh.h b/src/mca/pcm/rsh/src/pcm_rsh.h index 800f08837b..5d0b3e3595 100644 --- a/src/mca/pcm/rsh/src/pcm_rsh.h +++ b/src/mca/pcm/rsh/src/pcm_rsh.h @@ -24,61 +24,3 @@ struct mca_pcm_1_0_0_t* mca_pcm_rsh_init(int *priority, bool *have_hidden_threads); int mca_pcm_rsh_finalize(void); - -/* - * "Action" functions - */ -int mca_pcm_rsh_query_get_nodes(mca_pcm_rte_node_t **nodes, size_t *nodes_len, - int *available_procs); - -ompi_job_handle_t mca_pcm_rsh_handle_new(ompi_job_handle_t parent); -ompi_job_handle_t mca_pcm_rsh_handle_get(void); -void mca_pcm_rsh_handle_free(ompi_job_handle_t *job_handle); - -int mca_pcm_rsh_job_can_spawn(ompi_job_handle_t job_handle); -int mca_pcm_rsh_job_set_arguments(ompi_job_handle_t job_handle, - mca_pcm_control_args_t* opts, - size_t opts_len); -int mca_pcm_rsh_job_launch_procs(ompi_job_handle_t job_handle, - mca_pcm_rte_node_t *nodes, - size_t nodes_len, const char* file, - int argc, const char* argv[], - const char *env[]); -int mca_pcm_rsh_job_rendezvous(ompi_job_handle_t job_handle); -int mca_pcm_rsh_job_wait(ompi_job_handle_t job_handle); -int mca_pcm_rsh_job_running(ompi_job_handle_t job_handle, - int* running); -int mca_pcm_rsh_job_list_running(ompi_job_handle_t **handles, - size_t handles_len); - -int mca_pcm_rsh_proc_startup(void); -int mca_pcm_rsh_proc_get_peers(mca_pcm_proc_t **procs, size_t *nprocs); -mca_pcm_proc_t* mca_pcm_rsh_proc_get_me(void); -int mca_pcm_rsh_proc_get_parent(void); - -/* - * Private types / data / etc. - */ - -struct mca_pcm_rsh_connection_item_t { - ompi_list_item_t super; - - pid_t rshpid; - int vpid; - int status; -}; -typedef struct mca_pcm_rsh_connection_t mca_pcm_rsh_connection_t; - -extern ompi_list_t mca_pcm_rsh_connections; - -extern ompi_job_handle_t mca_pcm_rsh_my_job_handle; -extern int mca_pcm_rsh_my_vpid; - -extern char *mca_pcm_rsh_rsh; - - -/* - * Private functions - */ -void mca_pcm_rsh_lock_connections(void); -void mca_pcm_rsh_unlock_connections(void); diff --git a/src/mca/pcm/rsh/src/pcm_rsh_module.c b/src/mca/pcm/rsh/src/pcm_rsh_module.c index 2ef799e179..9e4f4d8c80 100644 --- a/src/mca/pcm/rsh/src/pcm_rsh_module.c +++ b/src/mca/pcm/rsh/src/pcm_rsh_module.c @@ -42,52 +42,9 @@ mca_pcm_base_module_1_0_0_t mca_pcm_rsh_module = { }; -struct mca_pcm_1_0_0_t mca_pcm_rsh_1_0_0 = { - mca_pcm_rsh_query_get_nodes, - - mca_pcm_rsh_handle_new, - mca_pcm_rsh_handle_get, - mca_pcm_rsh_handle_free, - - mca_pcm_rsh_job_can_spawn, - mca_pcm_rsh_job_set_arguments, - mca_pcm_rsh_job_launch_procs, - mca_pcm_rsh_job_rendezvous, - mca_pcm_rsh_job_wait, - mca_pcm_rsh_job_running, - mca_pcm_rsh_job_list_running, - - mca_pcm_rsh_proc_startup, - mca_pcm_rsh_proc_get_peers, - mca_pcm_rsh_proc_get_me, - mca_pcm_rsh_proc_get_parent -}; - -ompi_list_t mca_pcm_rsh_connections; - -ompi_job_handle_t mca_pcm_rsh_my_job_handle = NULL; -int mca_pcm_rsh_my_vpid = -1; - -char *mca_pcm_rsh_rsh = NULL; -char *mca_pcm_rsh_hostfile = NULL; - int mca_pcm_rsh_open(void) { - int id; - - id = mca_base_param_register_string("pcm", "rsh", "rsh", NULL, "ssh"); - mca_base_param_lookup_string(id, &mca_pcm_rsh_rsh); - - id = mca_base_param_register_int("pcm", "rsh", "vpid", NULL, -1); - mca_base_param_lookup_int(id, &mca_pcm_rsh_my_vpid); - - id = mca_base_param_register_string("pcm", "rsh", "job_handle", NULL, NULL); - mca_base_param_lookup_string(id, &mca_pcm_rsh_my_job_handle); - - id = mca_base_param_register_string("pcm", "rsh", "hostfile", NULL, NULL); - mca_base_param_lookup_string(id, &mca_pcm_rsh_hostfile); - return OMPI_SUCCESS; } @@ -95,11 +52,6 @@ mca_pcm_rsh_open(void) int mca_pcm_rsh_close(void) { -#if 0 - /* BWB - free list? */ - - OBJ_DESTRUCT(&mca_pcm_rsh_connections); -#endif return OMPI_SUCCESS; } @@ -108,18 +60,7 @@ struct mca_pcm_1_0_0_t* mca_pcm_rsh_init(int *priority, bool *allow_multi_user_threads, bool *have_hidden_threads) { -#if 0 - *priority = 0; - /* BWB - reevaluate this setting at some point */ - *allow_multi_user_threads = true; - *have_hidden_threads = false; - - /* That's about all we wrote thus far */ - - return &mca_pcm_rsh_1_0_0; -#else return NULL; -#endif } diff --git a/src/mca/pml/teg/Makefile.am b/src/mca/pml/teg/Makefile.am index 360b77c193..bbe9167933 100644 --- a/src/mca/pml/teg/Makefile.am +++ b/src/mca/pml/teg/Makefile.am @@ -2,7 +2,7 @@ # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +10,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_pml_teg_DSO -component_noinst = -component_install = mca_pml_teg.la +if OMPI_BUILD_pml_teg_LOADABLE_MODULE +module_noinst = +module_install = mca_pml_teg.la else -component_noinst = libmca_pml_teg.la -component_install = +module_noinst = libmca_pml_teg.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_pml_teg_la_SOURCES = -mca_pml_teg_la_LIBADD = src/libmca_pml_teg.la $(LIBOMPI_LA) +mca_pml_teg_la_LIBADD = \ + src/libmca_pml_teg.la \ + $(LIBOMPI_LA) mca_pml_teg_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_pml_teg_la_SOURCES = libmca_pml_teg_la_LIBADD = src/libmca_pml_teg.la libmca_pml_teg_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/ptl/elan/Makefile.am b/src/mca/ptl/elan/Makefile.am index 6bc49505a3..5435638b01 100644 --- a/src/mca/ptl/elan/Makefile.am +++ b/src/mca/ptl/elan/Makefile.am @@ -2,10 +2,13 @@ # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options +# JMS: +# CPPFLAGS including -I$(top_ompi_builddir)/src/include, etc +# does not seem to be passed down AM_CPPFLAGS = -I$(top_ompi_builddir)/src/include \ -I$(top_ompi_srcdir)/src -I$(top_ompi_srcdir)/src/include \ -I/usr/lib/qsnet/elan4/include @@ -14,25 +17,29 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_ptl_elan_DSO -component_noinst = -component_install = mca_ptl_elan.la +if OMPI_BUILD_ptl_elan_LOADABLE_MODULE +module_noinst = +module_install = mca_ptl_elan.la else -component_noinst = libmca_ptl_elan.la -component_install = +module_noinst = libmca_ptl_elan.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_ptl_elan_la_SOURCES = -mca_ptl_elan_la_LIBADD = src/libmca_ptl_elan.la $(LIBOMPI_LA) +mca_ptl_elan_la_LIBADD = \ + src/libmca_ptl_elan.la \ + $(LIBOMPI_LA) mca_ptl_elan_la_LDFLAGS = -module -avoid-version +#-lelan -lelanctrl -lrms -lrmscall -lelan4 -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_ptl_elan_la_SOURCES = libmca_ptl_elan_la_LIBADD = src/libmca_ptl_elan.la libmca_ptl_elan_la_LDFLAGS = -module -avoid-version +#-lelan -lelanctrl -lrms -lrmscall -lelan4 diff --git a/src/mca/ptl/gm/Makefile.am b/src/mca/ptl/gm/Makefile.am index 8f55be8fb3..d976bc1a06 100644 --- a/src/mca/ptl/gm/Makefile.am +++ b/src/mca/ptl/gm/Makefile.am @@ -2,7 +2,7 @@ # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OpenMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +10,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_ptl_gm_DSO -component_noinst = -component_install = mca_ptl_gm.la +if OMPI_BUILD_ptl_gm_LOADABLE_MODULE +module_noinst = +module_install = mca_ptl_gm.la else -component_noinst = libmca_ptl_gm.la -component_install = +module_noinst = libmca_ptl_gm.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_ptl_gm_la_SOURCES = -mca_ptl_gm_la_LIBADD = src/libmca_ptl_gm.la $(LIBOMPI_LA) +mca_ptl_gm_la_LIBADD = \ + src/libmca_ptl_gm.la \ + $(LIBOMPI_LA) mca_ptl_gm_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_ptl_gm_la_SOURCES = libmca_ptl_gm_la_LIBADD = src/libmca_ptl_gm.la libmca_ptl_gm_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/ptl/gm/configure.stub b/src/mca/ptl/gm/configure.stub index fb09c72835..16823e331a 100644 --- a/src/mca/ptl/gm/configure.stub +++ b/src/mca/ptl/gm/configure.stub @@ -8,128 +8,40 @@ # configure script. # AC_DEFUN([MCA_CONFIGURE_STUB],[ +# +# gm ptl configure.stub +# - # Additional --with flags that can be specified +# +# ...do whatever you need to do here, like look for the Elan4 +# libraries and header files. Built-in tests such as AC_CHECK_LIBS +# and AC_CHECK_HEADERS may be useful here. See the Autoconf docs ... +# +echo "Hello from gm configure.stub:MCA-CONFIGURE-STUB!" - AC_ARG_WITH(ptl-gm, - AC_HELP_STRING([--with-ptl-gm=DIR], - [Specify the installation directory of GM])) - AC_ARG_WITH(ptl-gm-libdir, - AC_HELP_STRING([--with-ptl-gm-libdir=DIR], - [directory where the GM library can be found, if it is not in \$GMDIR/lib or \$GMDIR/binary/lib])) - - # Add to CPPFLAGS if necessary - - EXTRA_CPPFLAGS= - if test -n "$with_ptl_gm"; then - if test -d "$with_ptl_gm/include"; then - EXTRA_CPPFLAGS="-I$with_ptl_gm/include" - else - AC_MSG_WARN([*** Warning: cannot find $with_ptl_gm/include]) - AC_MSG_WARN([*** Will still try to configure gm ptl anyway...]) - fi - fi - - # See if we can find gm.h - - CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS" - AC_CHECK_HEADERS(gm.h,, - AC_MSG_ERROR([*** Cannot find working gm.h.])) - - # Add to LDFLAGS if necessary - - EXTRA_LDFLAGS= - if test -n "$with_ptl_gm_libdir"; then - if test -d "$with_ptl_gm_libdir/lib"; then - EXTRA_LDFLAGS="-L$with_ptl_gm_libdir/lib" - elif test -d "$with_ptl_bm_libdir/binary/lib"; then - EXTRA_LDFLAGS="-L$with_ptl_gm_libdir/binary/lib" - else - AC_MSG_WARN([*** Warning: cannot find $with_ptl_gm_libdir/lib]) - AC_MSG_WARN([*** or $with_ptl_gm_libdir/binary/lib]) - AC_MSG_WARN([*** Will still try to configure gm ptl anyway...]) - fi - elif test -n "$with_ptl_gm"; then - if test -d "$with_ptl_gm/lib"; then - EXTRA_LDFLAGS="-L$with_ptl_gm/lib" - elif test -d "$with_ptl_bm/binary/lib"; then - EXTRA_LDFLAGS="-L$with_ptl_gm/binary/lib" - else - AC_MSG_WARN([*** Warning: cannot find $with_ptl_gm/lib]) - AC_MSG_WARN([*** or $with_ptl_gm/binary/lib]) - AC_MSG_WARN([*** Will still try to configure gm ptl anyway...]) - fi - fi - - # Try to find libgm - - LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" - AC_CHECK_LIB([gm], [main], [], - AC_MSG_ERROR([*** Cannot find libgm])) - - # - # See if we have GM_API_VERSION. If we do, use it. If not, find the - # highest one available. It seems that GM_API_VERSION was introduced - # somewhere after 1.3 but before 1.6. :-\ - # - - AC_MSG_CHECKING(for GM_API_VERSION) - AC_TRY_COMPILE([#include], - [int foo = GM_API_VERSION;], - have_gm_api_ver_msg=yes gm_api_ver=GM_API_VERSION, - have_gm_api_ver_msg=no gm_api_ver="") - AC_MSG_RESULT([$have_gm_api_ver_msg]) - if test "$gm_api_ver" = ""; then - found=0 - for val in 5 4 3; do - if test "$found" = "0"; then - var="GM_API_VERSION_1_$val" - AC_MSG_CHECKING(for $var) - AC_TRY_COMPILE([#include], - [int foo = $var;], - msg=yes found=1 gm_api_ver=$var, - msg=no found=0 gm_api_ver="") - AC_MSG_RESULT($msg) - fi - done - fi - if test "$gm_api_ver" = ""; then - AC_MSG_WARN([*** Could not find a supported GM_API_VERSION]) - AC_MSG_ERROR([*** Cannot continue]) - fi - AC_DEFINE_UNQUOTED(OMPI_MCA_PTL_GM_API_VERSION, $gm_api_ver, - [Version of the GM API to use]) - unset gm_api_ver have_gm_api_ver_msg found val msg - - # - # Do we have gm_put()? - # gm_put() was introduced in gm 2.0, and is exactly identical to gm - # 1.6's gm_directed_send_with_callback(). The name was simply changed - # for consistency/symmtery with gm_get(). - # - - AC_MSG_CHECKING([for gm_put()]) - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include - ]], - [[gm_put(0, 0, 0, 0, 0, 0, 0, 0, 0);]])], - [HAVE_RDMA_PUT=1 MSG=yes], - [HAVE_RDMA_PUT=0 MSG="no, use gm_directed_send_with_callback()"]) - AC_DEFINE_UNQUOTED(OMPI_MCA_PTL_GM_HAVE_RDMA_PUT, $HAVE_RDMA_PUT, - [Whether we have gm_put() or gm_directed_send_with_callback()]) - AC_MSG_RESULT([$MSG]) - - # - # Do we have gm_get()? - # gm_get() was introduced in gm 2.0. - # - - AC_MSG_CHECKING([for gm_get()]) - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include - ]], - [[gm_get(0, 0, 0, 0, 0, 0, 0, 0, 0);]])], - [HAVE_RDMA_GET=1 MSG=yes], - [HAVE_RDMA_GET=0 MSG=no]) - AC_DEFINE_UNQUOTED(OMPI_MCA_PTL_GM_HAVE_RDMA_GET, $HAVE_RDMA_GET, - [Whether we have get_get() or not]) - AC_MSG_RESULT([$MSG]) +# +# done with gm ptl configure.stub +# +])dnl + + +# +# Since MCA_CONFIGURE_STUB is not invoked when we are configured with +# --enable dist, we provide this alternate macro is that invoked +# instead. Not all modules will need this -- probably only modules +# thaty use AM_CONDITIONALS will require doing anything here. If you +# don't need it, you can remove this whole AC_DEFUN. +# +AC_DEFUN([MCA_CONFIGURE_DIST_STUB],[ +# +# gm ptl configure-dist.stub +# + +# ...probably only need this if have AM_CONDITIONALs in the +# MCA_CONFIGURE_STUB. +echo "Hello from gm configure.stub:MCA-CONFIGURE-DIST-STUB!" + +# +# done with gm ptl configure-dist.stub +# ])dnl diff --git a/src/mca/ptl/self/Makefile.am b/src/mca/ptl/self/Makefile.am index de921601c5..f225c438db 100644 --- a/src/mca/ptl/self/Makefile.am +++ b/src/mca/ptl/self/Makefile.am @@ -2,7 +2,7 @@ # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OpenMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +10,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_ptl_self_DSO -component_noinst = -component_install = mca_ptl_self.la +if OMPI_BUILD_ptl_self_LOADABLE_MODULE +module_noinst = +module_install = mca_ptl_self.la else -component_noinst = libmca_ptl_self.la -component_install = +module_noinst = libmca_ptl_self.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_ptl_self_la_SOURCES = -mca_ptl_self_la_LIBADD = src/libmca_ptl_self.la $(LIBOMPI_LA) +mca_ptl_self_la_LIBADD = \ + src/libmca_ptl_self.la \ + $(LIBOMPI_LA) mca_ptl_self_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_ptl_self_la_SOURCES = libmca_ptl_self_la_LIBADD = src/libmca_ptl_self.la libmca_ptl_self_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/ptl/sm/Makefile.am b/src/mca/ptl/sm/Makefile.am index 4cbae98157..e5210ff9d8 100644 --- a/src/mca/ptl/sm/Makefile.am +++ b/src/mca/ptl/sm/Makefile.am @@ -2,7 +2,7 @@ # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +10,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_ptl_sm_DSO -component_noinst = -component_install = mca_ptl_sm.la +if OMPI_BUILD_ptl_sm_LOADABLE_MODULE +module_noinst = +module_install = mca_ptl_sm.la else -component_noinst = libmca_ptl_sm.la -component_install = +module_noinst = libmca_ptl_sm.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_ptl_sm_la_SOURCES = -mca_ptl_sm_la_LIBADD = src/libmca_ptl_sm.la $(LIBOMPI_LA) +mca_ptl_sm_la_LIBADD = \ + src/libmca_ptl_sm.la \ + $(LIBOMPI_LA) mca_ptl_sm_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_ptl_sm_la_SOURCES = libmca_ptl_sm_la_LIBADD = src/libmca_ptl_sm.la libmca_ptl_sm_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/ptl/tcp/Makefile.am b/src/mca/ptl/tcp/Makefile.am index 0607dac8bc..2ce7d9fe1e 100644 --- a/src/mca/ptl/tcp/Makefile.am +++ b/src/mca/ptl/tcp/Makefile.am @@ -2,7 +2,7 @@ # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +10,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la -if OMPI_BUILD_ptl_tcp_DSO -component_noinst = -component_install = mca_ptl_tcp.la +if OMPI_BUILD_ptl_tcp_LOADABLE_MODULE +module_noinst = +module_install = mca_ptl_tcp.la else -component_noinst = libmca_ptl_tcp.la -component_install = +module_noinst = libmca_ptl_tcp.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_ptl_tcp_la_SOURCES = -mca_ptl_tcp_la_LIBADD = src/libmca_ptl_tcp.la $(LIBOMPI_LA) +mca_ptl_tcp_la_LIBADD = \ + src/libmca_ptl_tcp.la \ + $(LIBOMPI_LA) mca_ptl_tcp_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_ptl_tcp_la_SOURCES = libmca_ptl_tcp_la_LIBADD = src/libmca_ptl_tcp.la libmca_ptl_tcp_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/ptl/tcp/src/ptl_tcp_module.c b/src/mca/ptl/tcp/src/ptl_tcp_module.c index ad6d571f2c..00248c6553 100644 --- a/src/mca/ptl/tcp/src/ptl_tcp_module.c +++ b/src/mca/ptl/tcp/src/ptl_tcp_module.c @@ -507,8 +507,7 @@ static void mca_ptl_tcp_module_accept(void) */ static void mca_ptl_tcp_module_recv_handler(int sd, short flags, void* user) { - void* guid; - uint32_t size; + ompi_process_name_t guid; struct sockaddr_in addr; int retval; mca_ptl_tcp_proc_t* ptl_proc; @@ -522,30 +521,11 @@ static void mca_ptl_tcp_module_recv_handler(int sd, short flags, void* user) ompi_event_del((ompi_event_t*)user); free(user); - /* recv the size of the process identifier */ - retval = recv(sd, &size, sizeof(size), 0); - if(retval == 0) { - close(sd); - return; - } - if(retval != sizeof(size)) { + /* recv the process identifier */ + retval = recv(sd, &guid, sizeof(guid), 0); + if(retval != sizeof(guid)) { ompi_output(0, "mca_ptl_tcp_module_recv_handler: recv() return value %d != %d, errno = %d", - retval, sizeof(size), errno); - close(sd); - return; - } - - /* recv the identifier */ - size = ntohl(size); - guid = malloc(size); - if(guid == 0) { - close(sd); - return; - } - retval = recv(sd, guid, size, 0); - if(retval != size) { - ompi_output(0, "mca_ptl_tcp_module_recv_handler: recv() return value %d != %d, errno = %d", - retval, sizeof(size), errno); + retval, sizeof(guid), errno); close(sd); return; } @@ -561,7 +541,7 @@ static void mca_ptl_tcp_module_recv_handler(int sd, short flags, void* user) } /* lookup the corresponding process */ - ptl_proc = mca_ptl_tcp_proc_lookup(guid, size); + ptl_proc = mca_ptl_tcp_proc_lookup(&guid); if(NULL == ptl_proc) { ompi_output(0, "mca_ptl_tcp_module_recv_handler: unable to locate process"); close(sd); diff --git a/src/mca/ptl/tcp/src/ptl_tcp_peer.c b/src/mca/ptl/tcp/src/ptl_tcp_peer.c index e4bed891bb..c0546bff36 100644 --- a/src/mca/ptl/tcp/src/ptl_tcp_peer.c +++ b/src/mca/ptl/tcp/src/ptl_tcp_peer.c @@ -217,10 +217,8 @@ static int mca_ptl_tcp_peer_send_connect_ack(mca_ptl_base_peer_t* ptl_peer) { /* send process identifier to remote peer */ mca_ptl_tcp_proc_t* ptl_proc = mca_ptl_tcp_proc_local(); - uint32_t size_n = htonl(ptl_proc->proc_guid_size); - if(mca_ptl_tcp_peer_send_blocking(ptl_peer, &size_n, sizeof(size_n)) != sizeof(size_n) || - mca_ptl_tcp_peer_send_blocking(ptl_peer, ptl_proc->proc_guid, ptl_proc->proc_guid_size) != - ptl_proc->proc_guid_size) { + if(mca_ptl_tcp_peer_send_blocking(ptl_peer, &ptl_proc->proc_guid, sizeof(ptl_proc->proc_guid)) != + sizeof(ptl_proc->proc_guid)) { return OMPI_ERR_UNREACH; } return OMPI_SUCCESS; @@ -246,7 +244,7 @@ bool mca_ptl_tcp_peer_accept(mca_ptl_base_peer_t* ptl_peer, struct sockaddr_in* mca_ptl_tcp_proc_t *peer_proc = ptl_peer->peer_proc; if((ptl_peer->peer_sd < 0) || (ptl_peer->peer_state != MCA_PTL_TCP_CONNECTED && - peer_proc->proc_ompi->proc_vpid < this_proc->proc_ompi->proc_vpid)) { + peer_proc->proc_ompi->proc_name.procid < this_proc->proc_ompi->proc_name.procid)) { mca_ptl_tcp_peer_close(ptl_peer); ptl_peer->peer_sd = sd; if(mca_ptl_tcp_peer_send_connect_ack(ptl_peer) != OMPI_SUCCESS) { @@ -352,27 +350,15 @@ static int mca_ptl_tcp_peer_recv_blocking(mca_ptl_base_peer_t* ptl_peer, void* d static int mca_ptl_tcp_peer_recv_connect_ack(mca_ptl_base_peer_t* ptl_peer) { - uint32_t size_n, size_h; - void* guid; - int rc; + ompi_process_name_t guid; mca_ptl_tcp_proc_t* ptl_proc = ptl_peer->peer_proc; - if((rc = mca_ptl_tcp_peer_recv_blocking(ptl_peer, &size_n, sizeof(size_n))) != sizeof(size_n)) - return OMPI_ERR_UNREACH; - size_h = ntohl(size_n); - guid = malloc(size_h); - if(NULL == guid) { - ompi_output(0, "mca_ptl_tcp_peer_recv_connect_ack: malloc(%d) failed\n", size_h); - return OMPI_ERR_OUT_OF_RESOURCE; - } - - if((rc = mca_ptl_tcp_peer_recv_blocking(ptl_peer, guid, size_h)) != size_h) { - free(guid); + if((mca_ptl_tcp_peer_recv_blocking(ptl_peer, &guid, sizeof(ompi_process_name_t))) != sizeof(ompi_process_name_t)) { return OMPI_ERR_UNREACH; } /* compare this to the expected values */ - if(size_h != ptl_proc->proc_guid_size || memcmp(ptl_proc->proc_guid, guid, size_h) != 0) { + if(memcmp(&ptl_proc->proc_guid, &guid, sizeof(ompi_process_name_t)) != 0) { ompi_output(0, "mca_ptl_tcp_peer_connect: received unexpected process identifier"); mca_ptl_tcp_peer_close(ptl_peer); return OMPI_ERR_UNREACH; diff --git a/src/mca/ptl/tcp/src/ptl_tcp_proc.c b/src/mca/ptl/tcp/src/ptl_tcp_proc.c index 0beae997c5..2183df9d24 100644 --- a/src/mca/ptl/tcp/src/ptl_tcp_proc.c +++ b/src/mca/ptl/tcp/src/ptl_tcp_proc.c @@ -57,8 +57,6 @@ void mca_ptl_tcp_proc_destruct(mca_ptl_tcp_proc_t* proc) /* release resources */ if(NULL != proc->proc_peers) free(proc->proc_peers); - if(NULL != proc->proc_guid) - free(proc->proc_guid); } @@ -72,9 +70,7 @@ void mca_ptl_tcp_proc_destruct(mca_ptl_tcp_proc_t* proc) mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_create(ompi_proc_t* ompi_proc) { int rc; - size_t size = strlen(ompi_proc->proc_job) + 1; - uint32_t vpid = htonl(ompi_proc->proc_vpid); - + size_t size; mca_ptl_tcp_proc_t* ptl_proc = mca_ptl_tcp_proc_lookup_ompi(ompi_proc); if(ptl_proc != NULL) return ptl_proc; @@ -83,14 +79,7 @@ mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_create(ompi_proc_t* ompi_proc) ptl_proc->proc_ompi = ompi_proc; /* build a unique identifier (of arbitrary size) to represent the proc */ - ptl_proc->proc_guid_size = size + sizeof(uint32_t); - ptl_proc->proc_guid = malloc(ptl_proc->proc_guid_size); - if(ptl_proc->proc_guid == 0) { - OBJ_RELEASE(ptl_proc); - return 0; - } - memcpy(ptl_proc->proc_guid, ompi_proc->proc_job, size); - memcpy(((unsigned char*)ptl_proc->proc_guid)+size, &vpid, sizeof(uint32_t)); + ptl_proc->proc_guid = ompi_proc->proc_name; /* lookup tcp parameters exported by this proc */ rc = mca_base_modex_recv( @@ -146,14 +135,15 @@ static mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_lookup_ompi(ompi_proc_t* ompi_proc) * Look for an existing TCP process instance based on the globally unique * process identifier. */ -mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_lookup(void *guid, size_t size) +mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_lookup(const ompi_process_name_t *name) { mca_ptl_tcp_proc_t* tcp_proc; + ompi_process_name_t guid = *name; OMPI_THREAD_LOCK(&mca_ptl_tcp_module.tcp_lock); for(tcp_proc = (mca_ptl_tcp_proc_t*)ompi_list_get_first(&mca_ptl_tcp_module.tcp_procs); tcp_proc != (mca_ptl_tcp_proc_t*)ompi_list_get_end(&mca_ptl_tcp_module.tcp_procs); tcp_proc = (mca_ptl_tcp_proc_t*)ompi_list_get_next(tcp_proc)) { - if(tcp_proc->proc_guid_size == size && memcmp(tcp_proc->proc_guid, guid, size) == 0) { + if(memcmp(&tcp_proc->proc_guid, &guid, sizeof(guid)) == 0) { OMPI_THREAD_UNLOCK(&mca_ptl_tcp_module.tcp_lock); return tcp_proc; } diff --git a/src/mca/ptl/tcp/src/ptl_tcp_proc.h b/src/mca/ptl/tcp/src/ptl_tcp_proc.h index 99f4bd6d6f..524c8bfb9d 100644 --- a/src/mca/ptl/tcp/src/ptl_tcp_proc.h +++ b/src/mca/ptl/tcp/src/ptl_tcp_proc.h @@ -10,6 +10,7 @@ #include #include #include +#include "mca/ns/ns.h" #include "class/ompi_object.h" #include "proc/proc.h" #include "ptl_tcp.h" @@ -24,21 +25,20 @@ extern ompi_class_t mca_ptl_tcp_proc_t_class; * PTL instance that attempts to open a connection to the process. */ struct mca_ptl_tcp_proc_t { - ompi_list_item_t super; /**< allow proc to be placed on a list */ - ompi_proc_t *proc_ompi; /**< pointer to corresponding ompi_proc_t */ - void* proc_guid; /**< globally unique identifier for the process */ - size_t proc_guid_size; /**< size of the guid */ + ompi_list_item_t super; /**< allow proc to be placed on a list */ + ompi_proc_t *proc_ompi; /**< pointer to corresponding ompi_proc_t */ + ompi_process_name_t proc_guid; /**< globally unique identifier for the process */ struct mca_ptl_tcp_addr_t *proc_addrs; /**< array of addresses published by peer */ size_t proc_addr_count; /**< number of addresses published by peer */ struct mca_ptl_base_peer_t **proc_peers; /**< array of peers that have been created to access this proc */ size_t proc_peer_count; /**< number of peers */ - ompi_mutex_t proc_lock; /**< lock to protect against concurrent access to proc state */ + ompi_mutex_t proc_lock; /**< lock to protect against concurrent access to proc state */ }; typedef struct mca_ptl_tcp_proc_t mca_ptl_tcp_proc_t; mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_create(ompi_proc_t* ompi_proc); -mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_lookup(void *guid, size_t size); +mca_ptl_tcp_proc_t* mca_ptl_tcp_proc_lookup(const ompi_process_name_t*); /** diff --git a/src/mca/topo/unity/Makefile.am b/src/mca/topo/unity/Makefile.am index 3c42bc6370..ea8e45bc2b 100644 --- a/src/mca/topo/unity/Makefile.am +++ b/src/mca/topo/unity/Makefile.am @@ -1,8 +1,9 @@ +# -*- makefile -*- # # $HEADER$ # -# Use the top-level Makefile.options +# Use the top-level OMPI Makefile.options include $(top_ompi_srcdir)/config/Makefile.options @@ -10,25 +11,27 @@ SUBDIRS = src EXTRA_DIST = VERSION -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_topo_ompi_unity.la -if OMPI_BUILD_topo_unity_DSO -component_noinst = -component_install = mca_topo_unity.la +if OMPI_BUILD_topo_unity_LOADABLE_MODULE +module_noinst = +module_install = mca_topo_unity.la else -component_noinst = libmca_topo_unity.la -component_install = +module_noinst = libmca_topo_unity.la +module_install = endif -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) +mcamoduledir = $(libdir)/openmpi +mcamodule_LTLIBRARIES = $(module_install) mca_topo_unity_la_SOURCES = -mca_topo_unity_la_LIBADD = src/libmca_topo_unity.la $(LIBMPI_LA) +mca_topo_unity_la_LIBADD = \ + src/libmca_topo_unity.la \ + $(LIBMPI_LA) mca_topo_unity_la_LDFLAGS = -module -avoid-version -noinst_LTLIBRARIES = $(component_noinst) +noinst_LTLIBRARIES = $(module_noinst) libmca_topo_unity_la_SOURCES = libmca_topo_unity_la_LIBADD = src/libmca_topo_unity.la libmca_topo_unity_la_LDFLAGS = -module -avoid-version diff --git a/src/proc/proc.c b/src/proc/proc.c index 39c482e7ea..2b9037cb74 100644 --- a/src/proc/proc.c +++ b/src/proc/proc.c @@ -3,6 +3,7 @@ #include "util/output.h" #include "proc/proc.h" #include "mca/pcm/pcm.h" +#include "mca/oob/oob.h" static ompi_list_t ompi_proc_list; @@ -28,8 +29,6 @@ void ompi_proc_construct(ompi_proc_t* proc) OBJ_CONSTRUCT(&ompi_proc_lock, ompi_mutex_t); } - proc->proc_job = NULL; - proc->proc_vpid = 0; proc->proc_pml = NULL; proc->proc_modex = NULL; proc->proc_arch = 0; @@ -55,37 +54,27 @@ void ompi_proc_destruct(ompi_proc_t* proc) int ompi_proc_init(void) { - mca_pcm_proc_t *procs; - mca_pcm_proc_t *local; - size_t i, nprocs; + ompi_process_name_t *peers; + ompi_process_name_t *self; + size_t i, npeers; int rc; - if(OMPI_SUCCESS != (rc = mca_pcm.pcm_proc_startup())) { - ompi_output(0, "ompi_proc_init: pcm_proc_startup failed with errno=%d", rc); + if(OMPI_SUCCESS != (rc = mca_pcm.pcm_peers(&peers, &npeers))) { + ompi_output(0, "ompi_proc_init: mca_pcm.pcm_peers failed with errno=%d", rc); + return rc; + } + if(NULL == (self = mca_pcm.pcm_self())) { + ompi_output(0, "ompi_proc_init: mca_pcm.pcm_self failed with errno=%d", rc); return rc; } - if(NULL == (local = mca_pcm.pcm_proc_get_me())) { - ompi_output(0, "ompi_proc_init: unable to determine local proc id"); - return OMPI_ERROR; - } - - if(OMPI_SUCCESS != (rc = mca_pcm.pcm_proc_get_peers(&procs, &nprocs))) { - ompi_output(0, "ompi_proc_init: pcm_proc_get_peers failed with errno=%d", rc); - return rc; - } - - for(i=0; iproc_job = strdup(job); - proc->proc_vpid = vpid; - if(proc->proc_vpid == local->vpid && strcmp(proc->proc_job, local->job_handle) == 0) { + proc->proc_name = peers[i]; + if( peers + i == self ) { ompi_proc_local_proc = proc; } } - free(procs); return OMPI_SUCCESS; } @@ -94,24 +83,18 @@ ompi_proc_t** ompi_proc_world(size_t *size) { ompi_proc_t **procs = malloc(ompi_list_get_size(&ompi_proc_list) * sizeof(ompi_proc_t*)); ompi_proc_t *proc; - ompi_job_handle_t job; size_t count = 0; if(NULL == procs) return NULL; - if(NULL == (job = mca_pcm.pcm_handle_get())) - return NULL; - /* return only the procs that match this jobid */ OMPI_THREAD_LOCK(&ompi_proc_lock); for(proc = (ompi_proc_t*)ompi_list_get_first(&ompi_proc_list); proc != (ompi_proc_t*)ompi_list_get_end(&ompi_proc_list); proc = (ompi_proc_t*)ompi_list_get_next(proc)) { - if(strcmp(proc->proc_job,job) == 0) { - OBJ_RETAIN(proc); - procs[count++] = proc; - } + /* TSW - FIX */ + procs[count++] = proc; } OMPI_THREAD_UNLOCK(&ompi_proc_lock); *size = count; @@ -152,7 +135,7 @@ ompi_proc_t** ompi_proc_self(size_t* size) return procs; } -ompi_proc_t * ompi_proc_find ( ompi_job_handle_t jobid, uint32_t vpid ) +ompi_proc_t * ompi_proc_find ( const ompi_process_name_t * name ) { ompi_proc_t *proc; @@ -161,8 +144,9 @@ ompi_proc_t * ompi_proc_find ( ompi_job_handle_t jobid, uint32_t vpid ) for(proc = (ompi_proc_t*)ompi_list_get_first(&ompi_proc_list); proc != (ompi_proc_t*)ompi_list_get_end(&ompi_proc_list); proc = (ompi_proc_t*)ompi_list_get_next(proc)) { - if( (strcmp(proc->proc_job,jobid) == 0) && - (proc->proc_vpid == vpid ) ) + if( proc->proc_name.cellid == name->cellid && + proc->proc_name.jobid == name->jobid && + proc->proc_name.procid == name->procid ) { break; } @@ -170,3 +154,4 @@ ompi_proc_t * ompi_proc_find ( ompi_job_handle_t jobid, uint32_t vpid ) OMPI_THREAD_UNLOCK(&ompi_proc_lock); return proc; } + diff --git a/src/proc/proc.h b/src/proc/proc.h index 071bdba92a..9eff78cb97 100644 --- a/src/proc/proc.h +++ b/src/proc/proc.h @@ -9,20 +9,20 @@ #include "class/ompi_list.h" #include "datatype/datatype.h" #include "threads/mutex.h" +#include "mca/ns/ns.h" extern ompi_class_t ompi_proc_t_class; struct ompi_proc_t { - ompi_list_item_t super; /* allow proc to be placed on a list */ - ompi_job_handle_t proc_job; /* identifies a unique job */ - uint32_t proc_vpid; /* process identifier w/in the job */ + ompi_list_item_t super; /* allow proc to be placed on a list */ + ompi_process_name_t proc_name; struct mca_pml_proc_t* proc_pml; /* PML specific proc data */ struct mca_base_modex_t* proc_modex; /* MCA module exchange data */ int proc_arch; - ompi_convertor_t* proc_convertor; - ompi_mutex_t proc_lock; + ompi_convertor_t* proc_convertor; + ompi_mutex_t proc_lock; /* JMS: need to have the following information: - how am i [mpi] connected (bitmap): spawn (parent/child), @@ -62,9 +62,9 @@ static inline ompi_proc_t* ompi_proc_local(void) } /** - * Returns the proc instance for a given vpid + * Returns the proc instance for a given name */ -ompi_proc_t * ompi_proc_find ( ompi_job_handle_t jobid, uint32_t vpid ); +ompi_proc_t * ompi_proc_find ( const ompi_process_name_t* name ); #endif /* OMPI_PROC */ diff --git a/src/runtime/ompi_mpi_init.c b/src/runtime/ompi_mpi_init.c index e9bde62c68..a784f88b37 100644 --- a/src/runtime/ompi_mpi_init.c +++ b/src/runtime/ompi_mpi_init.c @@ -7,6 +7,7 @@ #include "include/constants.h" #include "runtime/runtime.h" #include "util/sys_info.h" +#include "util/proc_info.h" #include "mpi.h" #include "runtime/runtime.h" #include "communicator/communicator.h" @@ -62,6 +63,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) ompi_system_info structure */ ompi_sys_info(); + ompi_proc_info(); /* Become a OMPI process */ diff --git a/src/runtime/ompi_rte_init.c b/src/runtime/ompi_rte_init.c index 3cffa3013c..a98acfec58 100644 --- a/src/runtime/ompi_rte_init.c +++ b/src/runtime/ompi_rte_init.c @@ -48,8 +48,7 @@ int ompi_rte_init(bool *allow_multi_user_threads, bool *have_hidden_threads) /* JMS show_help */ return ret; } - if (OMPI_SUCCESS != (ret = mca_oob_base_select(&user_threads, - &hidden_threads))) { + if (OMPI_SUCCESS != (ret = mca_oob_base_init(&user_threads, &hidden_threads))) { /* JMS show_help */ return ret; } diff --git a/src/tools/mpirun/mpiboot b/src/tools/mpirun/mpiboot index 1a0289512b..049bfba6d5 100755 --- a/src/tools/mpirun/mpiboot +++ b/src/tools/mpirun/mpiboot @@ -17,9 +17,10 @@ # Some basic defaults # ###################################################################### -ompi_myvpid="" +ompi_cellid="" +ompi_jobid="" +ompi_procid="" ompi_numprocs="" -ompi_jobhandle="" ompi_commdir="" ###################################################################### @@ -34,9 +35,14 @@ for i ; do -test) exit 0 ;; - -myvpid) + -cellid) shift - ompi_myvpid="${1}" + ompi_cellid="${1}" + shift + ;; + -procid) + shift + ompi_procid="${1}" shift ;; -numprocs) @@ -44,9 +50,9 @@ for i ; do ompi_numprocs="${1}" shift ;; - -jobhandle) + -jobid) shift - ompi_jobhandle="${1}" + ompi_jobid="${1}" shift ;; -commdir) @@ -88,9 +94,10 @@ fi # Set the environment # ###################################################################### -export MCA_common_ompi_cofs_my_vpid="${ompi_myvpid}" -export MCA_common_ompi_cofs_num_procs="${ompi_numprocs}" -export MCA_common_ompi_cofs_job_handle="${ompi_jobhandle}" +export OMPI_MCA_pcm_cofs_cellid="${ompi_cellid}" +export OMPI_MCA_pcm_cofs_jobid="${ompi_jobid}" +export OMPI_MCA_pcm_cofs_procid="${ompi_procid}" +export OMPI_MCA_pcm_cofs_num_procs="${ompi_numprocs}" if test ! -z "${ompi_commdir}"; then export MCA_common_ompi_cofs_comm_dir="${ompi_commdir}" fi diff --git a/src/tools/mpirun/mpirun b/src/tools/mpirun/mpirun index 204a1bf64c..2c18817ebb 100755 --- a/src/tools/mpirun/mpirun +++ b/src/tools/mpirun/mpirun @@ -26,7 +26,7 @@ ompi_debug=0 # ###################################################################### ompi_hostfile="" -ompi_jobhandle="" +ompi_jobid="" ompi_app_args="" ompi_mydir="" ompi_cwd="`pwd`" @@ -155,7 +155,7 @@ do_start_proc() { cmd="ssh ${rmt_hostname} -f -n ${rmt_boot} " cmd="${cmd} -myvpid ${rmt_vpid}" - cmd="${cmd} -numprocs ${ompi_numprocs} -jobhandle ${ompi_jobhandle}" + cmd="${cmd} -numprocs ${ompi_numprocs} -cellid 1 -jobid ${ompi_jobid}" cmd="${cmd} -pwd ${ompi_cwd} ${ompi_prepped}" if test ! -z "${MCA_common_ompi_cofs_comm_dir}" ; then cmd="${cmd} -commdir ${MCA_common_ompi_cofs_comm_dir}" @@ -231,7 +231,7 @@ launch_procs() { # Do the prep work # ###################################################################### -ompi_jobhandle="pcm-cofs-job-handle-${$}-0" +ompi_jobid="${$}" ###################################################################### diff --git a/src/tools/ompi_info/modules.cc b/src/tools/ompi_info/modules.cc index 8f0abb2845..73d315b631 100644 --- a/src/tools/ompi_info/modules.cc +++ b/src/tools/ompi_info/modules.cc @@ -80,7 +80,7 @@ void ompi_info::open_modules() // oob module opening not implemented yet mca_oob_base_open(); - module_map["oob"] = &mca_oob_base_modules_available; + module_map["oob"] = &mca_oob_base_components; // coll module opening not implemented yet mca_coll_base_open(); diff --git a/src/util/proc_info.c b/src/util/proc_info.c index aee6f31d21..dc8a5a47c5 100644 --- a/src/util/proc_info.c +++ b/src/util/proc_info.c @@ -38,12 +38,6 @@ int ompi_proc_info(void) ompi_process_info.name->cellid = 0; ompi_process_info.name->jobid = 1; ompi_process_info.name->procid = 2; - sprintf(ompi_process_info.name->name, "%0x.%0x.%0x", - ompi_process_info.name->cellid, - ompi_process_info.name->jobid, - ompi_process_info.name->procid); - - ompi_process_info.init = true; return(OMPI_SUCCESS); }