diff --git a/contrib/Makefile.am b/contrib/Makefile.am index b985636add..cd67ee608f 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -12,7 +12,8 @@ # Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2010 IBM Corporation. All rights reserved. # Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. -# Copyright (c) 2013 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2016 Los Alamos National Security, Inc. All rights +# reserved. # Copyright (c) 2013 Intel Corporation. All rights reserved. # $COPYRIGHT$ # @@ -66,15 +67,15 @@ EXTRA_DIST = \ platform/lanl/cray_xc_cle5.2/optimized-lustre \ platform/lanl/cray_xc_cle5.2/optimized-lustre.conf \ platform/lanl/toss/debug-common \ - platform/lanl/toss/debug-nopanasas \ - platform/lanl/toss/debug-nopanasas.conf \ - platform/lanl/toss/debug-panasas \ - platform/lanl/toss/debug-panasas.conf \ + platform/lanl/toss/debug \ + platform/lanl/toss/debug.conf \ + platform/lanl/toss/debug-mlx \ + platform/lanl/toss/debug-mlx.conf \ platform/lanl/toss/optimized-common \ - platform/lanl/toss/optimized-nopanasas \ - platform/lanl/toss/optimized-nopanasas.conf \ - platform/lanl/toss/optimized-panasas \ - platform/lanl/toss/optimized-panasas.conf \ + platform/lanl/toss/optimized \ + platform/lanl/toss/optimized.conf \ + platform/lanl/toss/optimized-mlx \ + platform/lanl/toss/optimized-mlx.conf \ platform/lanl/toss/toss-common \ platform/lanl/darwin/darwin-common \ platform/lanl/darwin/debug-common \ diff --git a/contrib/platform/lanl/toss/debug b/contrib/platform/lanl/toss/debug new file mode 100644 index 0000000000..f975ae9818 --- /dev/null +++ b/contrib/platform/lanl/toss/debug @@ -0,0 +1,8 @@ +# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. +# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./toss-common +source ./debug-common + +# Enable panasas support in romio +with_io_romio_flags=--with-file-system=ufs+nfs+lustre diff --git a/contrib/platform/lanl/toss/debug-mlx b/contrib/platform/lanl/toss/debug-mlx new file mode 100644 index 0000000000..2940f2135b --- /dev/null +++ b/contrib/platform/lanl/toss/debug-mlx @@ -0,0 +1,4 @@ +# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. +# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./debug diff --git a/contrib/platform/lanl/toss/debug-mlx.conf b/contrib/platform/lanl/toss/debug-mlx.conf new file mode 100644 index 0000000000..03eea2b007 --- /dev/null +++ b/contrib/platform/lanl/toss/debug-mlx.conf @@ -0,0 +1,104 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights +# reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_base_component_show_load_errors = 0 +opal_set_max_sys_limits = 1 +orte_report_launch_progress = 1 + +# Define timeout for daemons to report back during launch +orte_startup_timeout = 10000 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=ib0,eth0 +oob_tcp_peer_retries = 1000 +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = vader,openib,self + +## Setup OpenIB - just in case +btl_openib_want_fork_support = 0 +btl_openib_receive_queues = X,4096,1024:X,12288,512:X,65536,512 + +## Disable MXM +pml = ob1 +coll = ^hcoll + +## Enable cpu affinity +hwloc_base_binding_policy = core + +## Setup MPI options +mpi_show_handle_leaks = 1 +mpi_warn_on_fork = 1 +#mpi_abort_print_stack = 1 + diff --git a/contrib/platform/lanl/toss/debug-nopanasas b/contrib/platform/lanl/toss/debug-nopanasas deleted file mode 100644 index 29c3d40cb6..0000000000 --- a/contrib/platform/lanl/toss/debug-nopanasas +++ /dev/null @@ -1,5 +0,0 @@ -# (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./toss-common -source ./debug-common diff --git a/contrib/platform/lanl/toss/debug-nopanasas.conf b/contrib/platform/lanl/toss/debug-nopanasas.conf deleted file mode 120000 index 2c6787c0c4..0000000000 --- a/contrib/platform/lanl/toss/debug-nopanasas.conf +++ /dev/null @@ -1 +0,0 @@ -debug-panasas.conf \ No newline at end of file diff --git a/contrib/platform/lanl/toss/debug-panasas b/contrib/platform/lanl/toss/debug-panasas deleted file mode 100644 index fffcf61d04..0000000000 --- a/contrib/platform/lanl/toss/debug-panasas +++ /dev/null @@ -1,13 +0,0 @@ -# (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./toss-common -source ./debug-common - -# Enable panasas support in romio -with_io_romio_flags=--with-file-system=ufs+nfs+panfs -CFLAGS="-I/opt/panfs/include" -CXXFLAGS="-I/opt/panfs/include" -FCFLAGS="-I/opt/panfs/include" -FFLAGS="-I/opt/panfs/include" -CCASFLAGS="-I/opt/panfs/include" diff --git a/contrib/platform/lanl/toss/debug-panasas.conf b/contrib/platform/lanl/toss/debug.conf similarity index 100% rename from contrib/platform/lanl/toss/debug-panasas.conf rename to contrib/platform/lanl/toss/debug.conf diff --git a/contrib/platform/lanl/toss/optimized b/contrib/platform/lanl/toss/optimized new file mode 100644 index 0000000000..59ef7e1add --- /dev/null +++ b/contrib/platform/lanl/toss/optimized @@ -0,0 +1,8 @@ +# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. +# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./toss-common +source ./optimized-common + +# Enable panasas support in romio +with_io_romio_flags=--with-file-system=ufs+nfs+lustre diff --git a/contrib/platform/lanl/toss/optimized-mlx b/contrib/platform/lanl/toss/optimized-mlx new file mode 100644 index 0000000000..38adccf8f5 --- /dev/null +++ b/contrib/platform/lanl/toss/optimized-mlx @@ -0,0 +1,4 @@ +# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. +# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./optimized diff --git a/contrib/platform/lanl/toss/optimized-mlx.conf b/contrib/platform/lanl/toss/optimized-mlx.conf new file mode 100644 index 0000000000..4bcbc3cefb --- /dev/null +++ b/contrib/platform/lanl/toss/optimized-mlx.conf @@ -0,0 +1,104 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights +# reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_base_component_show_load_errors = 0 +opal_set_max_sys_limits = 1 +orte_report_launch_progress = 1 + +# Define timeout for daemons to report back during launch +orte_startup_timeout = 10000 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include = ib0,eth0 +oob_tcp_peer_retries = 1000 +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = vader,openib,self + +## Setup OpenIB - just in case +btl_openib_want_fork_support = 0 +btl_openib_receive_queues = X,4096,1024:X,12288,512:X,65536,512 + +## Disable MXM +pml = ob1 +coll = ^hcoll + +## Enable cpu affinity +hwloc_base_binding_policy = core + +## Setup MPI options +mpi_show_handle_leaks = 0 +mpi_warn_on_fork = 1 +#mpi_abort_print_stack = 0 + diff --git a/contrib/platform/lanl/toss/optimized-nopanasas b/contrib/platform/lanl/toss/optimized-nopanasas deleted file mode 100644 index 08c80593ff..0000000000 --- a/contrib/platform/lanl/toss/optimized-nopanasas +++ /dev/null @@ -1,5 +0,0 @@ -# (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./toss-common -source ./optimized-common diff --git a/contrib/platform/lanl/toss/optimized-nopanasas.conf b/contrib/platform/lanl/toss/optimized-nopanasas.conf deleted file mode 120000 index 10407fce62..0000000000 --- a/contrib/platform/lanl/toss/optimized-nopanasas.conf +++ /dev/null @@ -1 +0,0 @@ -optimized-panasas.conf \ No newline at end of file diff --git a/contrib/platform/lanl/toss/optimized-panasas b/contrib/platform/lanl/toss/optimized-panasas deleted file mode 100644 index 06b5bc0287..0000000000 --- a/contrib/platform/lanl/toss/optimized-panasas +++ /dev/null @@ -1,13 +0,0 @@ -# (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./toss-common -source ./optimized-common - -# Enable panasas support in romio -with_io_romio_flags=--with-file-system=ufs+nfs+panfs -CFLAGS="-I/opt/panfs/include" -CXXFLAGS="-I/opt/panfs/include" -FCFLAGS="-I/opt/panfs/include" -FFLAGS="-I/opt/panfs/include" -CCASFLAGS="-I/opt/panfs/include" diff --git a/contrib/platform/lanl/toss/optimized-panasas.conf b/contrib/platform/lanl/toss/optimized.conf similarity index 100% rename from contrib/platform/lanl/toss/optimized-panasas.conf rename to contrib/platform/lanl/toss/optimized.conf