From 3069c9dc40bd779ae752c61eb0d40fd51b13f3b5 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Tue, 21 Apr 2009 20:07:43 +0000 Subject: [PATCH] Update LANL platform files This commit was SVN r21051. --- .../platform/lanl/rr-class/cross-debug-slave | 35 ++++++ .../lanl/rr-class/cross-debug-slave.conf | 84 +++++++++++++ .../lanl/rr-class/cross-optimized-slave | 35 ++++++ .../lanl/rr-class/cross-optimized-slave.conf | 84 +++++++++++++ .../lanl/rr-class/debug-nopanasas-nocell | 40 +++++++ .../lanl/rr-class/debug-nopanasas-nocell.conf | 111 ++++++++++++++++++ .../lanl/rr-class/debug-nopanasas.conf | 5 +- .../lanl/rr-class/debug-panasas-nocell | 41 +++++++ .../lanl/rr-class/debug-panasas-nocell.conf | 111 ++++++++++++++++++ .../platform/lanl/rr-class/debug-panasas.conf | 5 +- .../lanl/rr-class/optimized-nopanasas-nocell | 41 +++++++ .../rr-class/optimized-nopanasas-nocell.conf | 110 +++++++++++++++++ .../lanl/rr-class/optimized-nopanasas.conf | 5 +- .../lanl/rr-class/optimized-panasas-nocell | 41 +++++++ .../rr-class/optimized-panasas-nocell.conf | 110 +++++++++++++++++ .../lanl/rr-class/optimized-panasas.conf | 5 +- 16 files changed, 851 insertions(+), 12 deletions(-) create mode 100644 contrib/platform/lanl/rr-class/cross-debug-slave create mode 100644 contrib/platform/lanl/rr-class/cross-debug-slave.conf create mode 100644 contrib/platform/lanl/rr-class/cross-optimized-slave create mode 100644 contrib/platform/lanl/rr-class/cross-optimized-slave.conf create mode 100644 contrib/platform/lanl/rr-class/debug-nopanasas-nocell create mode 100644 contrib/platform/lanl/rr-class/debug-nopanasas-nocell.conf create mode 100644 contrib/platform/lanl/rr-class/debug-panasas-nocell create mode 100644 contrib/platform/lanl/rr-class/debug-panasas-nocell.conf create mode 100644 contrib/platform/lanl/rr-class/optimized-nopanasas-nocell create mode 100644 contrib/platform/lanl/rr-class/optimized-nopanasas-nocell.conf create mode 100644 contrib/platform/lanl/rr-class/optimized-panasas-nocell create mode 100644 contrib/platform/lanl/rr-class/optimized-panasas-nocell.conf diff --git a/contrib/platform/lanl/rr-class/cross-debug-slave b/contrib/platform/lanl/rr-class/cross-debug-slave new file mode 100644 index 0000000000..544c28efee --- /dev/null +++ b/contrib/platform/lanl/rr-class/cross-debug-slave @@ -0,0 +1,35 @@ +enable_dlopen=no +enable_mem_debug=no +enable_mem_profile=no +enable_debug_symbols=yes +enable_binaries=yes +enable_heterogeneous=yes +enable_debug=yes +enable_shared=no +enable_static=yes +enable_memchecker=no +enable_ipv6=no +enable_mpi_f77=no +enable_mpi_f90=no +enable_mpi_cxx=no +enable_mpi_cxx_seek=no +enable_cxx_exceptions=no +enable_ft_thread=no +enable_per_user_config_files=no +enable_script_wrapper_compilers=yes +enable_io_romio=no +enable_mca_no_build=crs,routed-linear,routed-radix,routed-binomial,routed-direct,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,btl-openib,btl-sm,filem,plm-tm +enable_contrib_no_build=libnbc,vt +with_tm=no +with_openib=no +with_devel_headers=yes +with_slurm=no +with_memory_manager=ptmalloc2 +CC=/opt/cell/toolchain/bin/ppu-gcc +CXX=/opt/cell/toolchain/bin/ppu-g++ +LDFLAGS="-m64" +CFLAGS="-m64" +CXXFLAGS="-m64" +with_wrapper_ldflags="-m64" +with_wrapper_cflags="-m64" +with_wrapper_cxxflags="-m64" diff --git a/contrib/platform/lanl/rr-class/cross-debug-slave.conf b/contrib/platform/lanl/rr-class/cross-debug-slave.conf new file mode 100644 index 0000000000..2c7ca21e66 --- /dev/null +++ b/contrib/platform/lanl/rr-class/cross-debug-slave.conf @@ -0,0 +1,84 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_component_show_load_errors = 0 +orte_abort_timeout = 10 +opal_set_max_sys_limits = 1 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=eth0 +oob_tcp_peer_retries = 10 +oob_tcp_disable_family = IPv6 +oob_tcp_listen_mode = listen_thread +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = tcp,self + +## Setup MPI options +mpi_show_handle_leaks = 0 +mpi_warn_on_fork = 1 diff --git a/contrib/platform/lanl/rr-class/cross-optimized-slave b/contrib/platform/lanl/rr-class/cross-optimized-slave new file mode 100644 index 0000000000..b670ec7f63 --- /dev/null +++ b/contrib/platform/lanl/rr-class/cross-optimized-slave @@ -0,0 +1,35 @@ +enable_dlopen=no +enable_mem_debug=no +enable_mem_profile=no +enable_debug_symbols=no +enable_binaries=yes +enable_heterogeneous=yes +enable_debug=no +enable_shared=no +enable_static=yes +enable_memchecker=no +enable_ipv6=no +enable_mpi_f77=no +enable_mpi_f90=no +enable_mpi_cxx=no +enable_mpi_cxx_seek=no +enable_cxx_exceptions=no +enable_ft_thread=no +enable_per_user_config_files=no +enable_script_wrapper_compilers=yes +enable_io_romio=no +enable_mca_no_build=crs,routed-linear,routed-radix,routed-binomial,routed-direct,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,btl-openib,btl-sm,filem,plm-tm +enable_contrib_no_build=libnbc,vt +with_tm=no +with_openib=no +with_devel_headers=yes +with_slurm=no +with_memory_manager=ptmalloc2 +CC=/opt/cell/toolchain/bin/ppu-gcc +CXX=/opt/cell/toolchain/bin/ppu-g++ +LDFLAGS="-m64" +CFLAGS="-m64" +CXXFLAGS="-m64" +with_wrapper_ldflags="-m64" +with_wrapper_cflags="-m64" +with_wrapper_cxxflags="-m64" diff --git a/contrib/platform/lanl/rr-class/cross-optimized-slave.conf b/contrib/platform/lanl/rr-class/cross-optimized-slave.conf new file mode 100644 index 0000000000..2c7ca21e66 --- /dev/null +++ b/contrib/platform/lanl/rr-class/cross-optimized-slave.conf @@ -0,0 +1,84 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_component_show_load_errors = 0 +orte_abort_timeout = 10 +opal_set_max_sys_limits = 1 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=eth0 +oob_tcp_peer_retries = 10 +oob_tcp_disable_family = IPv6 +oob_tcp_listen_mode = listen_thread +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = tcp,self + +## Setup MPI options +mpi_show_handle_leaks = 0 +mpi_warn_on_fork = 1 diff --git a/contrib/platform/lanl/rr-class/debug-nopanasas-nocell b/contrib/platform/lanl/rr-class/debug-nopanasas-nocell new file mode 100644 index 0000000000..b0b2560901 --- /dev/null +++ b/contrib/platform/lanl/rr-class/debug-nopanasas-nocell @@ -0,0 +1,40 @@ +enable_dlopen=no +enable_mem_debug=yes +enable_mem_profile=yes +enable_debug_symbols=yes +enable_binaries=yes +enable_heterogeneous=no +enable_picky=yes +enable_debug=yes +enable_shared=yes +enable_static=yes +enable_ipv6=no +enable_mpi_f77=yes +enable_mpi_f90=yes +enable_mpi_cxx=yes +enable_mpi_cxx_seek=yes +enable_cxx_exceptions=yes +enable_ft_thread=no +enable_per_user_config_files=no +enable_mca_no_build=crs,routed-direct,routed-linear,snapc,pml-csum,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,filem +enable_contrib_no_build=libnbc,vt +with_tm=/opt/PBS +with_openib=/opt/ofed +with_openib_libdir=/opt/ofed/lib64 +with_devel_headers=yes +with_slurm=no +with_io_romio_flags=--with-file-system=ufs+nfs +with_valgrind=no +LDFLAGS="-L/opt/PBS/lib64 -m64" +CFLAGS="-m64" +CXXFLAGS="-m64" +FCFLAGS="-m64" +FFLAGS="-m64" +CCASFLAGS="-m64" +with_wrapper_ldflags="-m64" +with_wrapper_cflags="-m64" +with_wrapper_cxxflags="-m64" +with_wrapper_fcflags="-m64" +with_wrapper_fflags="-m64" +with_wrapper_ccasflags="-m64" +with_memory_manager=ptmalloc2 diff --git a/contrib/platform/lanl/rr-class/debug-nopanasas-nocell.conf b/contrib/platform/lanl/rr-class/debug-nopanasas-nocell.conf new file mode 100644 index 0000000000..e95f754573 --- /dev/null +++ b/contrib/platform/lanl/rr-class/debug-nopanasas-nocell.conf @@ -0,0 +1,111 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_component_show_load_errors = 0 +orte_abort_timeout = 10 +opal_set_max_sys_limits = 1 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## MPI behavior +## Do NOT specify mpi_leave_pinned so system +## can figure out for itself whether or not +## it is supported and usable + +## Protect looped collectives +coll_sync_priority = 100 +coll_sync_barrier_before = 1000 + +## Activate hierarchical collectives +coll_hierarch_priority = 90 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=ib0 +oob_tcp_peer_retries = 10 +oob_tcp_disable_family = IPv6 +oob_tcp_listen_mode = listen_thread +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = sm,openib,self + +## Setup OpenIB +btl_openib_want_fork_support = 0 +btl_openib_cpc_include = oob +btl_openib_flags = 2 +btl_openib_max_btls=1 + +## Enable cpu affinity +mpi_paffinity_alone = 1 + +## Setup MPI options +mpi_show_handle_leaks = 1 +mpi_warn_on_fork = 1 +mpi_abort_print_stack = 1 + diff --git a/contrib/platform/lanl/rr-class/debug-nopanasas.conf b/contrib/platform/lanl/rr-class/debug-nopanasas.conf index 7469d2a90f..71e23812f5 100644 --- a/contrib/platform/lanl/rr-class/debug-nopanasas.conf +++ b/contrib/platform/lanl/rr-class/debug-nopanasas.conf @@ -86,7 +86,7 @@ coll_hierarch_priority = 90 ## Add the interface for out-of-band communication ## and set it up -oob_tcp_if_include=ib0,eth0 +oob_tcp_if_include=ib0 oob_tcp_peer_retries = 10 oob_tcp_disable_family = IPv6 oob_tcp_listen_mode = listen_thread @@ -94,8 +94,7 @@ oob_tcp_sndbuf = 32768 oob_tcp_rcvbuf = 32768 ## Define the MPI interconnects -btl = sm,openib,tcp,self -btl_tcp_if_include=eth0 +btl = sm,openib,self ## Setup OpenIB btl_openib_want_fork_support = 0 diff --git a/contrib/platform/lanl/rr-class/debug-panasas-nocell b/contrib/platform/lanl/rr-class/debug-panasas-nocell new file mode 100644 index 0000000000..32453e83d2 --- /dev/null +++ b/contrib/platform/lanl/rr-class/debug-panasas-nocell @@ -0,0 +1,41 @@ +enable_dlopen=no +enable_mem_debug=yes +enable_mem_profile=yes +enable_debug_symbols=yes +enable_binaries=yes +enable_heterogeneous=no +enable_picky=yes +enable_debug=yes +enable_shared=yes +enable_static=yes +enable_ipv6=no +enable_mpi_f77=yes +enable_mpi_f90=yes +enable_mpi_cxx=yes +enable_mpi_cxx_seek=yes +enable_cxx_exceptions=yes +enable_ft_thread=no +enable_per_user_config_files=no +enable_mca_no_build=crs,routed-direct,routed-linear,snapc,pml-csum,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,filem +enable_contrib_no_build=libnbc,vt +with_tm=/opt/PBS +with_openib=/opt/ofed +with_openib_libdir=/opt/ofed/lib64 +with_devel_headers=yes +with_slurm=no +with_io_romio_flags=--with-file-system=ufs+nfs+panfs +with_valgrind=no +LDFLAGS="-L/opt/PBS/lib64 -m64" +CFLAGS="-I/opt/panfs/include -m64" +CXXFLAGS="-I/opt/panfs/include -m64" +FCFLAGS="-I/opt/panfs/include -m64" +FFLAGS="-I/opt/panfs/include -m64" +CCASFLAGS="-I/opt/panfs/include -m64" +with_wrapper_ldflags="-m64" +with_wrapper_cflags="-m64" +with_wrapper_cxxflags="-m64" +with_wrapper_fcflags="-m64" +with_wrapper_fflags="-m64" +with_wrapper_ccasflags="-m64" +with_memory_manager=ptmalloc2 + diff --git a/contrib/platform/lanl/rr-class/debug-panasas-nocell.conf b/contrib/platform/lanl/rr-class/debug-panasas-nocell.conf new file mode 100644 index 0000000000..e95f754573 --- /dev/null +++ b/contrib/platform/lanl/rr-class/debug-panasas-nocell.conf @@ -0,0 +1,111 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_component_show_load_errors = 0 +orte_abort_timeout = 10 +opal_set_max_sys_limits = 1 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## MPI behavior +## Do NOT specify mpi_leave_pinned so system +## can figure out for itself whether or not +## it is supported and usable + +## Protect looped collectives +coll_sync_priority = 100 +coll_sync_barrier_before = 1000 + +## Activate hierarchical collectives +coll_hierarch_priority = 90 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=ib0 +oob_tcp_peer_retries = 10 +oob_tcp_disable_family = IPv6 +oob_tcp_listen_mode = listen_thread +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = sm,openib,self + +## Setup OpenIB +btl_openib_want_fork_support = 0 +btl_openib_cpc_include = oob +btl_openib_flags = 2 +btl_openib_max_btls=1 + +## Enable cpu affinity +mpi_paffinity_alone = 1 + +## Setup MPI options +mpi_show_handle_leaks = 1 +mpi_warn_on_fork = 1 +mpi_abort_print_stack = 1 + diff --git a/contrib/platform/lanl/rr-class/debug-panasas.conf b/contrib/platform/lanl/rr-class/debug-panasas.conf index 7469d2a90f..71e23812f5 100644 --- a/contrib/platform/lanl/rr-class/debug-panasas.conf +++ b/contrib/platform/lanl/rr-class/debug-panasas.conf @@ -86,7 +86,7 @@ coll_hierarch_priority = 90 ## Add the interface for out-of-band communication ## and set it up -oob_tcp_if_include=ib0,eth0 +oob_tcp_if_include=ib0 oob_tcp_peer_retries = 10 oob_tcp_disable_family = IPv6 oob_tcp_listen_mode = listen_thread @@ -94,8 +94,7 @@ oob_tcp_sndbuf = 32768 oob_tcp_rcvbuf = 32768 ## Define the MPI interconnects -btl = sm,openib,tcp,self -btl_tcp_if_include=eth0 +btl = sm,openib,self ## Setup OpenIB btl_openib_want_fork_support = 0 diff --git a/contrib/platform/lanl/rr-class/optimized-nopanasas-nocell b/contrib/platform/lanl/rr-class/optimized-nopanasas-nocell new file mode 100644 index 0000000000..fdbed9c831 --- /dev/null +++ b/contrib/platform/lanl/rr-class/optimized-nopanasas-nocell @@ -0,0 +1,41 @@ +enable_dlopen=no +enable_mem_debug=no +enable_mem_profile=no +enable_debug_symbols=no +enable_binaries=yes +enable_heterogeneous=no +enable_debug=no +enable_shared=yes +enable_static=yes +enable_memchecker=no +enable_ipv6=no +enable_mpi_f77=yes +enable_mpi_f90=yes +enable_mpi_cxx=yes +enable_mpi_cxx_seek=yes +enable_cxx_exceptions=no +enable_ft_thread=no +enable_per_user_config_files=no +enable_mca_no_build=crs,routed-direct,routed-linear,snapc,pml-csum,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,filem +enable_contrib_no_build=libnbc,vt +with_tm=/opt/PBS +with_openib=/opt/ofed +with_openib_libdir=/opt/ofed/lib64 +with_devel_headers=yes +with_slurm=no +with_io_romio_flags=--with-file-system=ufs+nfs +with_valgrind=no +LDFLAGS="-L/opt/PBS/lib64 -m64" +CFLAGS="-m64" +CXXFLAGS="-m64" +FCFLAGS="-m64" +FFLAGS="-m64" +CCASFLAGS="-m64" +with_wrapper_ldflags="-m64" +with_wrapper_cflags="-m64" +with_wrapper_cxxflags="-m64" +with_wrapper_fcflags="-m64" +with_wrapper_fflags="-m64" +with_wrapper_ccasflags="-m64" +with_memory_manager=ptmalloc2 + diff --git a/contrib/platform/lanl/rr-class/optimized-nopanasas-nocell.conf b/contrib/platform/lanl/rr-class/optimized-nopanasas-nocell.conf new file mode 100644 index 0000000000..838b84c762 --- /dev/null +++ b/contrib/platform/lanl/rr-class/optimized-nopanasas-nocell.conf @@ -0,0 +1,110 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_component_show_load_errors = 0 +orte_abort_timeout = 10 +opal_set_max_sys_limits = 1 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## MPI behavior +## Do NOT specify mpi_leave_pinned so system +## can figure out for itself whether or not +## it is supported and usable + +## Protect looped collectives +coll_sync_priority = 100 +coll_sync_barrier_before = 1000 + +## Activate hierarchical collectives +coll_hierarch_priority = 90 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=ib0 +oob_tcp_peer_retries = 10 +oob_tcp_disable_family = IPv6 +oob_tcp_listen_mode = listen_thread +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = sm,openib,self + +## Setup OpenIB +btl_openib_want_fork_support = 0 +btl_openib_cpc_include = oob +btl_openib_flags = 2 +btl_openib_max_btls=1 + +## Enable cpu affinity +mpi_paffinity_alone = 1 + +## Setup MPI options +mpi_show_handle_leaks = 0 +mpi_warn_on_fork = 1 + diff --git a/contrib/platform/lanl/rr-class/optimized-nopanasas.conf b/contrib/platform/lanl/rr-class/optimized-nopanasas.conf index 34800cf027..053513091f 100644 --- a/contrib/platform/lanl/rr-class/optimized-nopanasas.conf +++ b/contrib/platform/lanl/rr-class/optimized-nopanasas.conf @@ -86,7 +86,7 @@ coll_hierarch_priority = 90 ## Add the interface for out-of-band communication ## and set it up -oob_tcp_if_include=ib0,eth0 +oob_tcp_if_include=ib0 oob_tcp_peer_retries = 10 oob_tcp_disable_family = IPv6 oob_tcp_listen_mode = listen_thread @@ -94,8 +94,7 @@ oob_tcp_sndbuf = 32768 oob_tcp_rcvbuf = 32768 ## Define the MPI interconnects -btl = sm,openib,tcp,self -btl_tcp_if_include=eth0 +btl = sm,openib,self ## Setup OpenIB btl_openib_want_fork_support = 0 diff --git a/contrib/platform/lanl/rr-class/optimized-panasas-nocell b/contrib/platform/lanl/rr-class/optimized-panasas-nocell new file mode 100644 index 0000000000..579a7a91ce --- /dev/null +++ b/contrib/platform/lanl/rr-class/optimized-panasas-nocell @@ -0,0 +1,41 @@ +enable_dlopen=no +enable_mem_debug=no +enable_mem_profile=no +enable_debug_symbols=no +enable_binaries=yes +enable_heterogeneous=no +enable_debug=no +enable_shared=yes +enable_static=yes +enable_memchecker=no +enable_ipv6=no +enable_mpi_f77=yes +enable_mpi_f90=yes +enable_mpi_cxx=yes +enable_mpi_cxx_seek=yes +enable_cxx_exceptions=no +enable_ft_thread=no +enable_per_user_config_files=no +enable_mca_no_build=crs,routed-direct,routed-linear,snapc,pml-csum,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,filem +enable_contrib_no_build=libnbc,vt +with_tm=/opt/PBS +with_openib=/opt/ofed +with_openib_libdir=/opt/ofed/lib64 +with_devel_headers=yes +with_slurm=no +with_io_romio_flags=--with-file-system=ufs+nfs+panfs +with_valgrind=no +LDFLAGS="-L/opt/PBS/lib64 -m64" +CFLAGS="-I/opt/panfs/include -m64" +CXXFLAGS="-I/opt/panfs/include -m64" +FCFLAGS="-I/opt/panfs/include -m64" +FFLAGS="-I/opt/panfs/include -m64" +CCASFLAGS="-I/opt/panfs/include -m64" +with_wrapper_ldflags="-m64" +with_wrapper_cflags="-m64" +with_wrapper_cxxflags="-m64" +with_wrapper_fcflags="-m64" +with_wrapper_fflags="-m64" +with_wrapper_ccasflags="-m64" +with_memory_manager=ptmalloc2 + diff --git a/contrib/platform/lanl/rr-class/optimized-panasas-nocell.conf b/contrib/platform/lanl/rr-class/optimized-panasas-nocell.conf new file mode 100644 index 0000000000..838b84c762 --- /dev/null +++ b/contrib/platform/lanl/rr-class/optimized-panasas-nocell.conf @@ -0,0 +1,110 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_component_show_load_errors = 0 +orte_abort_timeout = 10 +opal_set_max_sys_limits = 1 + +## Protect the shared file systems +orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects +orte_tmpdir_base = /tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## MPI behavior +## Do NOT specify mpi_leave_pinned so system +## can figure out for itself whether or not +## it is supported and usable + +## Protect looped collectives +coll_sync_priority = 100 +coll_sync_barrier_before = 1000 + +## Activate hierarchical collectives +coll_hierarch_priority = 90 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=ib0 +oob_tcp_peer_retries = 10 +oob_tcp_disable_family = IPv6 +oob_tcp_listen_mode = listen_thread +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = sm,openib,self + +## Setup OpenIB +btl_openib_want_fork_support = 0 +btl_openib_cpc_include = oob +btl_openib_flags = 2 +btl_openib_max_btls=1 + +## Enable cpu affinity +mpi_paffinity_alone = 1 + +## Setup MPI options +mpi_show_handle_leaks = 0 +mpi_warn_on_fork = 1 + diff --git a/contrib/platform/lanl/rr-class/optimized-panasas.conf b/contrib/platform/lanl/rr-class/optimized-panasas.conf index 34800cf027..053513091f 100644 --- a/contrib/platform/lanl/rr-class/optimized-panasas.conf +++ b/contrib/platform/lanl/rr-class/optimized-panasas.conf @@ -86,7 +86,7 @@ coll_hierarch_priority = 90 ## Add the interface for out-of-band communication ## and set it up -oob_tcp_if_include=ib0,eth0 +oob_tcp_if_include=ib0 oob_tcp_peer_retries = 10 oob_tcp_disable_family = IPv6 oob_tcp_listen_mode = listen_thread @@ -94,8 +94,7 @@ oob_tcp_sndbuf = 32768 oob_tcp_rcvbuf = 32768 ## Define the MPI interconnects -btl = sm,openib,tcp,self -btl_tcp_if_include=eth0 +btl = sm,openib,self ## Setup OpenIB btl_openib_want_fork_support = 0