From 5071602c59bdb72fca0e78398eae3184b7ea1433 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Tue, 14 Jun 2016 07:45:06 -0700 Subject: [PATCH] PSM/PSM2: Disable signal handler hijacking by default Per discussion on https://github.com/open-mpi/ompi/pull/1767 (and some subsequent phone calls and off-issue email discussions), the PSM library is hijacking signal handlers by default. Specifically: unless the environment variables `IPATH_NO_BACKTRACE=1` (for PSM / Intel TrueScale) is set, the library constructor for this library will hijack various signal handlers for the purpose of invoking its own error reporting mechanisms. This may be a bit *surprising*, but is not a *problem*, per se. The real problem is that older versions of at least the PSM library do not unregister these signal handlers upon being unloaded from memory. Hence, a segv can actually result in a double segv (i.e., the original segv and then another segv when the now-non-existent signal handler is invoked). This PSM signal hijacking subverts Open MPI's own signal reporting mechanism, which may be a bit surprising for some users (particularly those who do not have Intel TrueScale). As such, we disable it by default so that Open MPI's own error-reporting mechanisms are used. Additionally, there is a typo in the library destructor for the PSM2 library that may cause problems in the unloading of its signal handlers. This problem can be avoided by setting `HFI_NO_BACKTRACE=1` (for PSM2 / Intel OmniPath). This is further compounded by the fact that the PSM / PSM2 libraries can be loaded by the OFI MTL and the usNIC BTL (because they are loaded by libfabric), even when there is no Intel networking hardware present. Having the PSM/PSM2 libraries behave this way when no Intel hardware is present is clearly undesirable (and is likely to be fixed in future releases of the PSM/PSM2 libraries). This commit sets the following two environment variables to disable this behavior from the PSM/PSM2 libraries (if they are not already set): * IPATH_NO_BACKTRACE=1 * HFI_NO_BACKTRACE=1 If the user has set these variables before invoking Open MPI, we will not override their values (i.e., their preferences will be honored). Signed-off-by: Jeff Squyres --- ompi/mpi/java/c/mpi_MPI.c | 6 +++++- opal/runtime/opal.h | 9 ++++++++- opal/runtime/opal_init.c | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/ompi/mpi/java/c/mpi_MPI.c b/ompi/mpi/java/c/mpi_MPI.c index 40bb28b556..ee6984760f 100644 --- a/ompi/mpi/java/c/mpi_MPI.c +++ b/ompi/mpi/java/c/mpi_MPI.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -131,6 +131,10 @@ OBJ_CLASS_INSTANCE(ompi_java_buffer_t, */ jint JNI_OnLoad(JavaVM *vm, void *reserved) { + // Ensure that PSM signal hijacking is disabled *before* loading + // the library (see comment in the function for more detail). + opal_init_psm(); + libmpi = dlopen("libmpi." OPAL_DYN_LIB_SUFFIX, RTLD_NOW | RTLD_GLOBAL); #if defined(HAVE_DL_INFO) && defined(HAVE_LIBGEN_H) diff --git a/opal/runtime/opal.h b/opal/runtime/opal.h index d92f396249..872203c5d6 100644 --- a/opal/runtime/opal.h +++ b/opal/runtime/opal.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2010-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved. * $COPYRIGHT$ * @@ -76,6 +76,13 @@ OPAL_DECLSPEC int opal_finalize(void); */ OPAL_DECLSPEC int opal_init_util(int* pargc, char*** pargv); +/** + * Disable PSM/PSM2 signal hijacking. + * + * See comment in the function for more detail. + */ +OPAL_DECLSPEC int opal_init_psm(void); + /** * Finalize the OPAL layer, excluding the MCA system. * diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c index e8d2220d84..72511e8ac6 100644 --- a/opal/runtime/opal_init.c +++ b/opal/runtime/opal_init.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2010-2015 Los Alamos National Security, LLC. @@ -265,6 +265,34 @@ opal_err2str(int errnum, const char **errmsg) } +int opal_init_psm(void) +{ + /* Very early in the init sequence -- before *ANY* MCA components + are opened -- we need to disable some behavior from the PSM and + PSM2 libraries (by default): at least some old versions of + these libraries hijack signal handlers during their library + constructors and then do not un-hijack them when the libraries + are unloaded. + + It is a bit of an abstraction break that we have to put + vendor/transport-specific code in the OPAL core, but we're + out of options, unfortunately. + + NOTE: We only disable this behavior if the corresponding + environment variables are not already set (i.e., if the + user/environment has indicated a preference for this behavior, + we won't override it). */ + if (NULL == getenv("IPATH_NO_BACKTRACE")) { + opal_setenv("IPATH_NO_BACKTRACE", "1", true, &environ); + } + if (NULL == getenv("HFI_NO_BACKTRACE")) { + opal_setenv("HFI_NO_BACKTRACE", "1", true, &environ); + } + + return OPAL_SUCCESS; +} + + int opal_init_util(int* pargc, char*** pargv) { @@ -328,6 +356,10 @@ opal_init_util(int* pargc, char*** pargv) goto return_error; } + // Disable PSM signal hijacking (see comment in function for more + // details) + opal_init_psm(); + /* Setup the parameter system */ if (OPAL_SUCCESS != (ret = mca_base_var_init())) { error = "mca_base_var_init";