diff --git a/contrib/Makefile.am b/contrib/Makefile.am index d655e19c7c..bf78f975ad 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -12,7 +12,7 @@ # Copyright (c) 2009-2017 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2010 IBM Corporation. All rights reserved. # Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. -# Copyright (c) 2013-2016 Los Alamos National Security, Inc. All rights +# Copyright (c) 2013-2018 Los Alamos National Security, Inc. All rights # reserved. # Copyright (c) 2013 Intel Corporation. All rights reserved. # Copyright (c) 2017 Amazon.com, Inc. or its affiliates. @@ -67,17 +67,21 @@ EXTRA_DIST = \ platform/lanl/cray_xc_cle5.2/optimized-common \ platform/lanl/cray_xc_cle5.2/optimized-lustre \ platform/lanl/cray_xc_cle5.2/optimized-lustre.conf \ - platform/lanl/toss/debug-common \ - platform/lanl/toss/debug \ - platform/lanl/toss/debug.conf \ - platform/lanl/toss/debug-mlx \ - platform/lanl/toss/debug-mlx.conf \ - platform/lanl/toss/optimized-common \ - platform/lanl/toss/optimized \ - platform/lanl/toss/optimized.conf \ - platform/lanl/toss/optimized-mlx \ - platform/lanl/toss/optimized-mlx.conf \ - platform/lanl/toss/toss-common \ + platform/lanl/toss/README \ + platform/lanl/toss/common \ + platform/lanl/toss/common-optimized \ + platform/lanl/toss/cray-lustre-optimized \ + platform/lanl/toss/cray-lustre-optimized.conf \ + platform/lanl/toss/toss2-mlx-optimized \ + platform/lanl/toss/toss2-mlx-optimized.conf \ + platform/lanl/toss/toss2-qib-optimized \ + platform/lanl/toss/toss2-qib-optimized.conf \ + platform/lanl/toss/toss3-hfi-optimized \ + platform/lanl/toss/toss3-hfi-optimized.conf \ + platform/lanl/toss/toss3-mlx-optimized \ + platform/lanl/toss/toss3-mlx-optimized.conf \ + platform/lanl/toss/toss3-wc-optimized \ + platform/lanl/toss/toss3-wc-optimized.conf \ platform/lanl/darwin/darwin-common \ platform/lanl/darwin/debug-common \ platform/lanl/darwin/optimized-common \ diff --git a/contrib/platform/lanl/toss/README b/contrib/platform/lanl/toss/README new file mode 100644 index 0000000000..27beae42f8 --- /dev/null +++ b/contrib/platform/lanl/toss/README @@ -0,0 +1,99 @@ +These platform files were created from platform files shipped with the release +tarball. Each file has been modified. Here are the details on how they were +created. + +- common + Copy of contrib/platform/lanl/toss/toss-common. Removed entries in bottom + half of file that were specific to TOSS so that it could be used for Cray + platforms as well. +- common-optimized + Copy of contrib/platform/lanl/toss/optimized-common. Used the file as-is. +- toss2-qib-optimized + Copy of contrib/platform/lanl/toss/optimized with the following changes: + - source common and common-optimzed instead of toss-common and + optimized-common + - added entries that were removed from common: + - enable_mca_no_build + - with_slurm + - with_tm + - with_pmi + - with_verbs + - NOTE: common had "with_devel_headers=yes" in it that was not propagated. + This option should not be used in production as per Open MPI developer + mailing list guidance. + - Changed comment "Disable components not needed on any TOSS platform" to + "Disable components not needed on TOSS platforms with high-speed networks" + - Changed "enable panasas" to "enable lustre" +- toss2-qib-optimized.conf + - copy of contrib/platform/lanl/toss/optimized.conf with the following + changes: + - changed: orte_no_session_dirs = /lustre,/net,/users,/usr/projects + - changed: btl = ^openib + - removed: hwloc_base_binding_policy = core (outdated setting) + - added: rmaps_base_ranking_policy = core (rank by core) + - added: ras_base_launch_orted_on_hn = true (run orted on parent node of + allocation) +- toss2-mlx-optimized + - copy of toss2-qib-optimized +- toss2-mlx-optimized.conf + - copy of toss2-qib-optimized.conf with the following changes: + - remove: oob_tcp_if_include = ib0,eth0 (identification of general network + device names is problematic in RHEL7. Just let Open MPI figure it out) + - change: btl = vader,openib,self + - change: btl_openib_receive_queues = X,4096,1024:X,12288,512:X,65536,512 + (change S to X; make sure numbers match those for the same entry in + contrib/platform/lanl/toss/optimized-mlx.conf) + - addition: pml = ob1 (disable MXM) + - addition: coll = ^hcoll (disable MXM) +- toss3-hfi-optimized + - copy of toss2-qib-optimized +- toss3-hfi-optimized.conf + - copy of toss2-qib-optimized.conf with the following changes: + - remove: oob_tcp_if_include = ib0,eth0 + - add: oob_tcp_if_exclude = ib0 (Omnipath is flaky; don't use it for oob) +- toss3-wc-optimized (platform file for woodchuck which is an ethernet-only + connected cluster) + - copy of toss3-hfi-optimized with the following changes: + - change: remove "btl-tcp" from the enable_mca_no_build list + - change: comment "Disable components not needed on TOSS platforms with + high-speed networks" to "Disable components not needed on TOSS Ethernet- + connected clusters" + - change: with_verbs=no + - change: comment "Always build ibverbs support" to "Do not build ibverbs + support" +- toss3-wc-optimized.conf + - copy of toss3-hfi-optimized.conf with the following changes: + - change: comment "Add the interface for out-of-band communication and set + it up" to "Set up the interface for out-of-band communication" + - remove: oob_tcp_if_exclude = ib0 + - remove: btl (let Open MPI figure out what best to use for ethernet- + connected hardware) + - remove: btl_openib_want_fork_support (no infiniband) + - remove: btl_openib_receive_queues (no infiniband) +- cray-lustre-optimized + - copy of contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre with the + following changes: + - remove: whole if/else clause of 'test "$enable_debug" = "yes"' + - addition: source ./common + - addition: source ./common-optimized + - change: with_io_romio_flags="--with-file-system=ufs+nfs+lustre" + - remove: with_lustre=/opt/cray/lustre-cray_ari_s/default + - additions from platform/lanl/cray_xc_cle5.2/optimized-common that don't + go in common-optimzed: + - enable_mca_no_build=crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,ess-cnos,grpcomm-cnos,plm-rsh,btl-tcp,oob-ud,ras-simulator,mpool-fake + - enable_mca_static=btl:ugni,btl:self,btl:vader,pml:ob1 + - enable_mca_directpml-ob1 + - with_verbs=no + - with_tm=no + - enable_orte_static_ports=no + - enable_pty_support=no + - addition: enable_dlopen=yes (change from original platform file as per + Nathan Hjelm) +- cray-lustre-optimized.conf + - copy of contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre.conf with + the following changes: + - change: orte_no_session_dirs = /lustre,/users,/usr/projects + - remove: hwloc_base_binding_policy = core (outdated setting) + - addition: rmaps_base_ranking_policy = core (rank by core) + +# vi: filetype=txt diff --git a/contrib/platform/lanl/toss/common b/contrib/platform/lanl/toss/common new file mode 100644 index 0000000000..1085f470e3 --- /dev/null +++ b/contrib/platform/lanl/toss/common @@ -0,0 +1,20 @@ +# (c) 2013-1018 Los Alamos National Security, LLC. All rights reserved. +# Open MPI common configuration for TOSS/TOSS2 v1.7.x/1.8.x + +enable_binaries=yes +enable_heterogeneous=no +enable_shared=yes +enable_static=yes +enable_ipv6=no +enable_ft_thread=no +enable_per_user_config_files=no +enable_memchecker=no +with_valgrind=no + +# Enable the fortran bindings +enable_mpi_fortran=yes + +# Disable the C++ binding. They were deprecated in MPI-2.2 and removed in MPI-3 +enable_mpi_cxx=no +enable_mpi_cxx_seek=no +enable_cxx_exceptions=no diff --git a/contrib/platform/lanl/toss/optimized-common b/contrib/platform/lanl/toss/common-optimized similarity index 70% rename from contrib/platform/lanl/toss/optimized-common rename to contrib/platform/lanl/toss/common-optimized index 38bfc18bf6..3241fb687f 100644 --- a/contrib/platform/lanl/toss/optimized-common +++ b/contrib/platform/lanl/toss/common-optimized @@ -1,4 +1,4 @@ -# (c) 2013 Los Alamos National Security, LLC. All rights reserved. +# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved. # Open MPI common optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x enable_mem_debug=no diff --git a/contrib/platform/lanl/toss/cray-lustre-optimized b/contrib/platform/lanl/toss/cray-lustre-optimized new file mode 100644 index 0000000000..2204ee50e1 --- /dev/null +++ b/contrib/platform/lanl/toss/cray-lustre-optimized @@ -0,0 +1,34 @@ +# (c) 2012-2018 Los Alamos National Security, LLC. All rights reserved. +# Open MPI configuration for Cray XC v2.x GNU compiler, +# Lustre + +if test "$CC" = "cc" ; then + echo "ERROR: Open MPI should not be compiled with Cray's wrapper compilers (cc/CC/ftn)" + exit 1 +fi + +source ./common +source ./common-optimized + +# enable Lustre in romio +with_io_romio_flags="--with-file-system=ufs+nfs+lustre" + +# Disable components not needed +enable_mca_no_build=crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,ess-cnos,grpcomm-cnos,plm-rsh,btl-tcp,oob-ud,ras-simulator,mpool-fake +enable_mca_static=btl:ugni,btl:self,btl:vader,pml:ob1 + +# enable direct calling for ob1 +enable_mca_direct=pml-ob1 + +# do not use IB verbs +with_verbs=no + +# do not use torque +with_tm=no + +enable_dlopen=yes + +enable_orte_static_ports=no + +enable_pty_support=no + diff --git a/contrib/platform/lanl/toss/debug-mlx.conf b/contrib/platform/lanl/toss/cray-lustre-optimized.conf similarity index 84% rename from contrib/platform/lanl/toss/debug-mlx.conf rename to contrib/platform/lanl/toss/cray-lustre-optimized.conf index 03eea2b007..663387de58 100644 --- a/contrib/platform/lanl/toss/debug-mlx.conf +++ b/contrib/platform/lanl/toss/cray-lustre-optimized.conf @@ -10,8 +10,8 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights -# reserved. +# Copyright (c) 2015-2018 Los Alamos National Security, LLC. +# All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -62,43 +62,46 @@ # Basic behavior to smooth startup mca_base_component_show_load_errors = 0 -opal_set_max_sys_limits = 1 -orte_report_launch_progress = 1 +#orte_report_launch_progress = 1 + +# Set line buffering for stdout/stderr +ess_base_stream_buffering = 1 # Define timeout for daemons to report back during launch -orte_startup_timeout = 10000 +orte_startup_timeout = 360 ## Protect the shared file systems -orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects -orte_tmpdir_base = /tmp +orte_no_session_dirs = /lustre,/users,/usr/projects +orte_tmpdir_base = /var/tmp ## Require an allocation to run - protects the frontend ## from inadvertent job executions orte_allocation_required = 1 +## Deal with the allocator +orte_strip_prefix = nid +orte_retain_aliases = 1 +# 1st alias entry is the stripped node name, +# 2nd is the unstripped one +orte_hostname_alias_index = 2 + ## Add the interface for out-of-band communication ## and set it up -oob_tcp_if_include=ib0,eth0 +oob_tcp_if_include=ipogif0 oob_tcp_peer_retries = 1000 oob_tcp_sndbuf = 32768 oob_tcp_rcvbuf = 32768 ## Define the MPI interconnects -btl = vader,openib,self +btl = self,vader,ugni -## Setup OpenIB - just in case -btl_openib_want_fork_support = 0 -btl_openib_receive_queues = X,4096,1024:X,12288,512:X,65536,512 +## Setup Gemini +# TODO LANL -## Disable MXM -pml = ob1 -coll = ^hcoll - -## Enable cpu affinity -hwloc_base_binding_policy = core +## Rank by core +rmaps_base_ranking_policy = core ## Setup MPI options mpi_show_handle_leaks = 1 mpi_warn_on_fork = 1 #mpi_abort_print_stack = 1 - diff --git a/contrib/platform/lanl/toss/debug b/contrib/platform/lanl/toss/debug deleted file mode 100644 index f975ae9818..0000000000 --- a/contrib/platform/lanl/toss/debug +++ /dev/null @@ -1,8 +0,0 @@ -# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. -# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./toss-common -source ./debug-common - -# Enable panasas support in romio -with_io_romio_flags=--with-file-system=ufs+nfs+lustre diff --git a/contrib/platform/lanl/toss/debug-common b/contrib/platform/lanl/toss/debug-common deleted file mode 100644 index d7a0a08074..0000000000 --- a/contrib/platform/lanl/toss/debug-common +++ /dev/null @@ -1,8 +0,0 @@ -# (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Open MPI common debug configuration for TOSS/TOSS2 v1.7.x/1.8.x - -enable_mem_debug=yes -enable_mem_profile=yes -enable_debug_symbols=yes -enable_picky=yes -enable_debug=yes diff --git a/contrib/platform/lanl/toss/debug-mlx b/contrib/platform/lanl/toss/debug-mlx deleted file mode 100644 index 2940f2135b..0000000000 --- a/contrib/platform/lanl/toss/debug-mlx +++ /dev/null @@ -1,4 +0,0 @@ -# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. -# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./debug diff --git a/contrib/platform/lanl/toss/optimized b/contrib/platform/lanl/toss/optimized deleted file mode 100644 index 59ef7e1add..0000000000 --- a/contrib/platform/lanl/toss/optimized +++ /dev/null @@ -1,8 +0,0 @@ -# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. -# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./toss-common -source ./optimized-common - -# Enable panasas support in romio -with_io_romio_flags=--with-file-system=ufs+nfs+lustre diff --git a/contrib/platform/lanl/toss/optimized-mlx b/contrib/platform/lanl/toss/optimized-mlx deleted file mode 100644 index 38adccf8f5..0000000000 --- a/contrib/platform/lanl/toss/optimized-mlx +++ /dev/null @@ -1,4 +0,0 @@ -# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved. -# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x - -source ./optimized diff --git a/contrib/platform/lanl/toss/toss-common b/contrib/platform/lanl/toss/toss-common deleted file mode 100644 index ba000be23f..0000000000 --- a/contrib/platform/lanl/toss/toss-common +++ /dev/null @@ -1,40 +0,0 @@ -# (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Open MPI common configuration for TOSS/TOSS2 v1.7.x/1.8.x - -enable_binaries=yes -enable_heterogeneous=no -enable_shared=yes -enable_static=yes -enable_ipv6=no -enable_ft_thread=no -enable_per_user_config_files=no -enable_memchecker=no -with_valgrind=no - -# Enable the fortran bindings -enable_mpi_fortran=yes - -# Disable the C++ binding. They were deprecated in MPI-2.2 and removed in MPI-3 -enable_mpi_cxx=no -enable_mpi_cxx_seek=no -enable_cxx_exceptions=no - -# Disable components not needed on any TOSS platform -enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp - -# Enable malloc hooks for mpi_leave_pinned -with_memory_manager=linux - -# TOSS2 uses slurm -with_slurm=yes -with_tm=no - -# Enable PMI support for direct launch -with_pmi=yes - -# Always build ibverbs support -with_verbs=yes - -# Install the development headers -with_devel_headers=yes - diff --git a/contrib/platform/lanl/toss/toss2-mlx-optimized b/contrib/platform/lanl/toss/toss2-mlx-optimized new file mode 100644 index 0000000000..4b0eafa6ed --- /dev/null +++ b/contrib/platform/lanl/toss/toss2-mlx-optimized @@ -0,0 +1,21 @@ +# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved. +# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./common +source ./common-optimized + +# Disable components not needed on TOSS platforms with high-speed networks +enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp + +# TOSS2 uses slurm +with_slurm=yes +with_tm=no + +# Enable PMI support for direct launch +with_pmi=yes + +# Enable lustre support in romio +with_io_romio_flags=--with-file-system=ufs+nfs+lustre + +# Always build ibverbs support +with_verbs=yes diff --git a/contrib/platform/lanl/toss/optimized-mlx.conf b/contrib/platform/lanl/toss/toss2-mlx-optimized.conf similarity index 89% rename from contrib/platform/lanl/toss/optimized-mlx.conf rename to contrib/platform/lanl/toss/toss2-mlx-optimized.conf index f649426bc2..b44452760d 100644 --- a/contrib/platform/lanl/toss/optimized-mlx.conf +++ b/contrib/platform/lanl/toss/toss2-mlx-optimized.conf @@ -10,7 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights +# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights # reserved. # $COPYRIGHT$ # @@ -69,7 +69,7 @@ orte_report_launch_progress = 1 orte_startup_timeout = 10000 ## Protect the shared file systems -orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_no_session_dirs = /lustre,/net,/users,/usr/projects orte_tmpdir_base = /tmp ## Require an allocation to run - protects the frontend @@ -88,17 +88,22 @@ btl = vader,openib,self ## Setup OpenIB - just in case btl_openib_want_fork_support = 0 +## Use Shared Receive Queues (SRQ). Mellanox ConnectX cards should be able to +## use eXtended Reliable Connection receive queues (XRC), but our systems are +## missing the needed libraries and headers to support it. btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512 -## Disable MXM -pml = ob1 -coll = ^hcoll - -## Enable cpu affinity -hwloc_base_binding_policy = core +## Rank by core +rmaps_base_ranking_policy = core ## Setup MPI options mpi_show_handle_leaks = 0 mpi_warn_on_fork = 1 #mpi_abort_print_stack = 0 +## Run orted on parent node of allocation +ras_base_launch_orted_on_hn = true + +## Disable MXM +pml = ob1 +coll = ^hcoll diff --git a/contrib/platform/lanl/toss/toss2-qib-optimized b/contrib/platform/lanl/toss/toss2-qib-optimized new file mode 100644 index 0000000000..4b0eafa6ed --- /dev/null +++ b/contrib/platform/lanl/toss/toss2-qib-optimized @@ -0,0 +1,21 @@ +# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved. +# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./common +source ./common-optimized + +# Disable components not needed on TOSS platforms with high-speed networks +enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp + +# TOSS2 uses slurm +with_slurm=yes +with_tm=no + +# Enable PMI support for direct launch +with_pmi=yes + +# Enable lustre support in romio +with_io_romio_flags=--with-file-system=ufs+nfs+lustre + +# Always build ibverbs support +with_verbs=yes diff --git a/contrib/platform/lanl/toss/toss2-qib-optimized.conf b/contrib/platform/lanl/toss/toss2-qib-optimized.conf new file mode 100644 index 0000000000..ebcd31a6b0 --- /dev/null +++ b/contrib/platform/lanl/toss/toss2-qib-optimized.conf @@ -0,0 +1,111 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights +# reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_base_component_show_load_errors = 0 +opal_set_max_sys_limits = 1 +orte_report_launch_progress = 1 + +# Define timeout for daemons to report back during launch +orte_startup_timeout = 10000 + +## Protect the shared file systems +orte_no_session_dirs = /lustre,/net,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include = ib0,eth0 +oob_tcp_peer_retries = 1000 +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = ^openib + +## Turn off osc rdma component. This is used in one-sided communication which +## isn't supported on psm2 and omnipath interconnects. If this option isn't +## used, a runtime error about btl's will result because we are turning off +## openib above. The current implementation of one-sided communication by- +## passes the loaded components and goes straight for btl's. Since one-sided +## communication doesn't really work on psm2 and omnipath, rather than turn +## openib back on, we're going to turn off some rdma capability. +osc = ^rdma + +## Setup OpenIB - just in case +btl_openib_want_fork_support = 0 +btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512 + +## Rank by core +rmaps_base_ranking_policy = core + +## Setup MPI options +mpi_show_handle_leaks = 0 +mpi_warn_on_fork = 1 +#mpi_abort_print_stack = 0 + +## Run orted on parent node of allocation +ras_base_launch_orted_on_hn = true diff --git a/contrib/platform/lanl/toss/toss3-hfi-optimized b/contrib/platform/lanl/toss/toss3-hfi-optimized new file mode 100644 index 0000000000..4b0eafa6ed --- /dev/null +++ b/contrib/platform/lanl/toss/toss3-hfi-optimized @@ -0,0 +1,21 @@ +# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved. +# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./common +source ./common-optimized + +# Disable components not needed on TOSS platforms with high-speed networks +enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp + +# TOSS2 uses slurm +with_slurm=yes +with_tm=no + +# Enable PMI support for direct launch +with_pmi=yes + +# Enable lustre support in romio +with_io_romio_flags=--with-file-system=ufs+nfs+lustre + +# Always build ibverbs support +with_verbs=yes diff --git a/contrib/platform/lanl/toss/toss3-hfi-optimized.conf b/contrib/platform/lanl/toss/toss3-hfi-optimized.conf new file mode 100644 index 0000000000..fe5862dd9d --- /dev/null +++ b/contrib/platform/lanl/toss/toss3-hfi-optimized.conf @@ -0,0 +1,112 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights +# reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_base_component_show_load_errors = 0 +opal_set_max_sys_limits = 1 +orte_report_launch_progress = 1 + +# Define timeout for daemons to report back during launch +orte_startup_timeout = 10000 + +## Protect the shared file systems +orte_no_session_dirs = /lustre,/net,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## Set up out-of-band communication. +## For OmniPath connected machines, don't use the ib0 network for +## out-of-band communication. It's a little flaky. +oob_tcp_if_exclude = ib0 +oob_tcp_peer_retries = 1000 +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = ^openib + +## Turn off osc rdma component. This is used in one-sided communication which +## isn't supported on psm2 and omnipath interconnects. If this option isn't +## used, a runtime error about btl's will result because we are turning off +## openib above. The current implementation of one-sided communication by- +## passes the loaded components and goes straight for btl's. Since one-sided +## communication doesn't really work on psm2 and omnipath, rather than turn +## openib back on, we're going to turn off some rdma capability. +osc = ^rdma + +## Setup OpenIB - just in case +btl_openib_want_fork_support = 0 +btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512 + +## Rank by core +rmaps_base_ranking_policy = core + +## Setup MPI options +mpi_show_handle_leaks = 0 +mpi_warn_on_fork = 1 +#mpi_abort_print_stack = 0 + +## Run orted on parent node of allocation +ras_base_launch_orted_on_hn = true diff --git a/contrib/platform/lanl/toss/toss3-mlx-optimized b/contrib/platform/lanl/toss/toss3-mlx-optimized new file mode 100644 index 0000000000..4b0eafa6ed --- /dev/null +++ b/contrib/platform/lanl/toss/toss3-mlx-optimized @@ -0,0 +1,21 @@ +# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved. +# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./common +source ./common-optimized + +# Disable components not needed on TOSS platforms with high-speed networks +enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp + +# TOSS2 uses slurm +with_slurm=yes +with_tm=no + +# Enable PMI support for direct launch +with_pmi=yes + +# Enable lustre support in romio +with_io_romio_flags=--with-file-system=ufs+nfs+lustre + +# Always build ibverbs support +with_verbs=yes diff --git a/contrib/platform/lanl/toss/debug.conf b/contrib/platform/lanl/toss/toss3-mlx-optimized.conf similarity index 86% rename from contrib/platform/lanl/toss/debug.conf rename to contrib/platform/lanl/toss/toss3-mlx-optimized.conf index 7937cfd8ae..da66745206 100644 --- a/contrib/platform/lanl/toss/debug.conf +++ b/contrib/platform/lanl/toss/toss3-mlx-optimized.conf @@ -10,7 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights +# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights # reserved. # $COPYRIGHT$ # @@ -69,16 +69,14 @@ orte_report_launch_progress = 1 orte_startup_timeout = 10000 ## Protect the shared file systems -orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_no_session_dirs = /lustre,/net,/users,/usr/projects orte_tmpdir_base = /tmp ## Require an allocation to run - protects the frontend ## from inadvertent job executions orte_allocation_required = 1 -## Add the interface for out-of-band communication -## and set it up -oob_tcp_if_include=ib0,eth0 +## Set up out-of-band communication. oob_tcp_peer_retries = 1000 oob_tcp_sndbuf = 32768 oob_tcp_rcvbuf = 32768 @@ -88,13 +86,18 @@ btl = vader,openib,self ## Setup OpenIB - just in case btl_openib_want_fork_support = 0 +## Use Shared Receive Queues (SRQ). Mellanox ConnectX cards should be able to +## use eXtended Reliable Connection receive queues (XRC), but our systems are +## missing the needed libraries and headers to support it. btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512 -## Enable cpu affinity -hwloc_base_binding_policy = core +## Rank by core +rmaps_base_ranking_policy = core ## Setup MPI options -mpi_show_handle_leaks = 1 +mpi_show_handle_leaks = 0 mpi_warn_on_fork = 1 -#mpi_abort_print_stack = 1 +#mpi_abort_print_stack = 0 +## Run orted on parent node of allocation +ras_base_launch_orted_on_hn = true diff --git a/contrib/platform/lanl/toss/toss3-wc-optimized b/contrib/platform/lanl/toss/toss3-wc-optimized new file mode 100644 index 0000000000..b63a753e3b --- /dev/null +++ b/contrib/platform/lanl/toss/toss3-wc-optimized @@ -0,0 +1,21 @@ +# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved. +# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x + +source ./common +source ./common-optimized + +# Disable components not needed on TOSS Ethernet-connected clusters +enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp + +# TOSS2 uses slurm +with_slurm=yes +with_tm=no + +# Enable PMI support for direct launch +with_pmi=yes + +# Enable lustre support in romio +with_io_romio_flags=--with-file-system=ufs+nfs+lustre + +# Do not build ibverbs support +with_verbs=no diff --git a/contrib/platform/lanl/toss/optimized.conf b/contrib/platform/lanl/toss/toss3-wc-optimized.conf similarity index 88% rename from contrib/platform/lanl/toss/optimized.conf rename to contrib/platform/lanl/toss/toss3-wc-optimized.conf index 3135e12455..c5ea85149d 100644 --- a/contrib/platform/lanl/toss/optimized.conf +++ b/contrib/platform/lanl/toss/toss3-wc-optimized.conf @@ -10,7 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights +# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights # reserved. # $COPYRIGHT$ # @@ -69,32 +69,25 @@ orte_report_launch_progress = 1 orte_startup_timeout = 10000 ## Protect the shared file systems -orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_no_session_dirs = /lustre,/net,/users,/usr/projects orte_tmpdir_base = /tmp ## Require an allocation to run - protects the frontend ## from inadvertent job executions orte_allocation_required = 1 -## Add the interface for out-of-band communication -## and set it up -oob_tcp_if_include = ib0,eth0 +## Set up the interface for out-of-band communication oob_tcp_peer_retries = 1000 oob_tcp_sndbuf = 32768 oob_tcp_rcvbuf = 32768 -## Define the MPI interconnects -btl = vader,openib,self - -## Setup OpenIB - just in case -btl_openib_want_fork_support = 0 -btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512 - -## Enable cpu affinity -hwloc_base_binding_policy = core +## Rank by core +rmaps_base_ranking_policy = core ## Setup MPI options mpi_show_handle_leaks = 0 mpi_warn_on_fork = 1 #mpi_abort_print_stack = 0 +## Run orted on parent node of allocation +ras_base_launch_orted_on_hn = true