Add two MCA parameters to the MPI level to control behavior during

MPI_ABORT.  From the ompi_info output:

       MCA mpi: parameter "mpi_abort_delay" (current value: "0")
                If nonzero, print out an identifying message when
                MPI_ABORT is invoked (hostname, PID of the process
                that called MPI_ABORT) and delay for that many seconds
                before exiting (a negative delay value means to never
                abort).  This allows attaching of a debugger before
                quitting the job.
       MCA mpi: parameter "mpi_abort_print_stack" (current value: "0")
                If nonzero, print out a stack trace when MPI_ABORT is
                invoked

This commit was SVN r9487.

ompi/runtime/ompi_mpi_abort.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2006 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -21,16 +22,29 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#ifdef HAVE_EXECINFO_H
+#include <execinfo.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
 
-#include "ompi/communicator/communicator.h"
+#include "opal/event/event.h"
 #include "opal/util/show_help.h"
 #include "orte/util/proc_info.h"
-#include "ompi/runtime/mpiruntime.h"
 #include "orte/runtime/runtime.h"
 #include "orte/mca/ns/ns.h"
 #include "orte/mca/rmgr/rmgr.h"
+#include "ompi/communicator/communicator.h"
 #include "ompi/proc/proc.h"
-#include "opal/event/event.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/runtime/params.h"
 
 #if HAVE_SIGNAL_H
 #include <signal.h>
@@ -67,7 +81,9 @@ ompi_mpi_abort(struct ompi_communicator_t* comm,
                bool kill_remote_of_intercomm)
 {
     orte_jobid_t my_jobid;
-    int ret=OMPI_SUCCESS;
+    int ret = OMPI_SUCCESS;
+    char hostname[MAXHOSTNAMELEN];
+    pid_t pid;
     
     /* Corner case: if we're being called as a result of the
        OMPI_ERR_INIT_FINALIZE macro (meaning that this is before
@@ -78,6 +94,54 @@ ompi_mpi_abort(struct ompi_communicator_t* comm,
         exit(errcode);
     }
 
+    /* If we're going to print anything, get the hostname and PID of
+       this process */
+
+    if (ompi_mpi_abort_print_stack ||
+        0 != ompi_mpi_abort_delay) {
+        gethostname(hostname, sizeof(hostname));
+        pid = getpid();
+    }
+
+    /* Should we print a stack trace? */
+
+    if (ompi_mpi_abort_print_stack) {
+#if OMPI_WANT_PRETTY_PRINT_STACKTRACE && ! defined(__WINDOWS__) && defined(HAVE_BACKTRACE)
+        int i;
+        int trace_size;
+        void *trace[32];
+        char **messages = (char **)NULL;
+
+        trace_size = backtrace(trace, 32);
+        messages = backtrace_symbols(trace, trace_size);
+
+        for (i = 0; i < trace_size; ++i) {
+            fprintf(stderr, "[%s:%d] [%d] func:%s\n", hostname, (int) pid, 
+                    i, messages[i]);
+            fflush(stderr);
+        }
+#endif
+    }
+
+    /* Should we wait for a while before aborting? */
+
+    if (0 != ompi_mpi_abort_delay) {
+        if (ompi_mpi_abort_delay < 0) {
+            fprintf(stderr ,"[%s:%d] Looping forever in MPI abort\n",
+                    hostname, (int) pid);
+            fflush(stderr);
+            while (1) { 
+                sleep(5); 
+            }
+        } else {
+            fprintf(stderr, "[%s:%d] Delaying for %d seconds in MPI_abort\n",
+                    hostname, (int) pid, ompi_mpi_abort_delay);
+            do {
+                sleep(1);
+            } while (--ompi_mpi_abort_delay > 0);
+        }
+    }
+
     /* BWB - XXX - Should probably publish the error code somewhere */
 
     /* Kill everyone in the job.  We may make this better someday to

ompi/runtime/ompi_mpi_params.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2006 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -43,7 +44,8 @@ bool ompi_debug_no_free_handles = false;
 bool ompi_mpi_show_mca_params = false;
 char *ompi_mpi_show_mca_params_file = NULL;
 bool ompi_mpi_paffinity_alone = false;
-
+bool ompi_mpi_abort_generate_stack_trace = false;
+int ompi_mpi_abort_delay = 0;
 
 int ompi_mpi_register_params(void)
 {
@@ -136,6 +138,36 @@ int ompi_mpi_register_params(void)
                                 true, false, 
                                 -1, NULL);
 
+    /* MPI_ABORT controls */
+
+    mca_base_param_reg_int_name("mpi", "abort_delay",
+                                "If nonzero, print out an identifying message when MPI_ABORT is invoked (hostname, PID of the process that called MPI_ABORT) and delay for that many seconds before exiting (a negative delay value means to never abort).  This allows attaching of a debugger before quitting the job.",
+                                false, false, 
+                                ompi_mpi_abort_delay,
+                                &ompi_mpi_abort_delay);
+    
+    mca_base_param_reg_int_name("mpi", "abort_print_stack",
+                                "If nonzero, print out a stack trace when MPI_ABORT is invoked",
+                                false, 
+                                /* If we do not have stack trace
+                                   capability, make this a read-only
+                                   MCA param */
+#if OMPI_WANT_PRETTY_PRINT_STACKTRACE && ! defined(__WINDOWS__) && defined(HAVE_BACKTRACE)
+                                false, 
+#else
+                                true,
+#endif
+                                (int) ompi_mpi_abort_print_stack,
+                                &value);
+#if OMPI_WANT_PRETTY_PRINT_STACKTRACE && ! defined(__WINDOWS__) && defined(HAVE_BACKTRACE)
+    /* Only take the value if we have stack trace capability */
+    ompi_mpi_abort_print_stack = (bool) value;
+#else
+    /* If we do not have stack trace capability, ensure that this is
+       hard-coded to false */
+    ompi_mpi_abort_print_stack = false;
+#endif
+
     /* The ddt engine has a few parameters */
 
     return ompi_ddt_register_params();

ompi/runtime/params.h

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2006 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -88,6 +89,22 @@ OMPI_DECLSPEC extern char * ompi_mpi_show_mca_params_file;
  */
 OMPI_DECLSPEC extern bool ompi_mpi_paffinity_alone;
 
+    /**
+     * Whether an MPI_ABORT should print out a stack trace or not.
+     */
+    OMPI_DECLSPEC bool ompi_mpi_abort_print_stack;
+
+    /**
+     * Whether  MPI_ABORT  should  print  out an  identifying  message
+     * (e.g., hostname  and PID)  and loop waiting  for a  debugger to
+     * attach.  The value of the integer is how many seconds to wait:
+     *
+     * 0 = do not print the message and do not loop
+     * negative value = print the message and loop forever
+     * positive value = print the message and delay for that many seconds
+     */
+    OMPI_DECLSPEC int ompi_mpi_abort_delay;
+
    /**
      * Register MCA parameters used by the MPI layer.
      *
@@ -98,6 +115,7 @@ OMPI_DECLSPEC extern bool ompi_mpi_paffinity_alone;
      */
 OMPI_DECLSPEC int ompi_mpi_register_params(void);
 
+
     /**
      * Display all MCA parameters used 
      *