diff --git a/ompi/mca/btl/openib/btl_openib.c b/ompi/mca/btl/openib/btl_openib.c index 81b3f324fe..d7606b8843 100644 --- a/ompi/mca/btl/openib/btl_openib.c +++ b/ompi/mca/btl/openib/btl_openib.c @@ -31,6 +31,7 @@ #endif #include "orte/util/show_help.h" #include "orte/runtime/orte_globals.h" +#include "orte/mca/errmgr/base/base.h" #include "opal/class/opal_bitmap.h" #include "opal/util/output.h" #include "opal/util/arch.h" @@ -650,18 +651,32 @@ static uint64_t calculate_max_reg (void) max_reg = (num_mtt - reserved_mtt) * getpagesize () * mtts_per_seg; } else { - /* need to update to determine the registration limit for this configuration */ + /* Need to update to determine the registration limit for this + configuration */ max_reg = mem_total; } - /* NTH: print a warning if we can't register more than 75% of physical memory */ + /* Print a warning if we can't register more than 75% of physical + memory. Abort if the abort_not_enough_reg_mem MCA param was + set. */ if (max_reg < mem_total * 3 / 4) { + char *action; + + if (mca_btl_openib_component.abort_not_enough_reg_mem) { + action = "Your MPI job will now abort."; + } else { + action = "Your MPI job will continue, but may be behave poorly and/or hang."; + } orte_show_help("help-mpi-btl-openib.txt", "reg mem limit low", true, orte_process_info.nodename, (unsigned long)(max_reg >> 20), - (unsigned long)(mem_total >> 20)); + (unsigned long)(mem_total >> 20), action); + if (mca_btl_openib_component.abort_not_enough_reg_mem) { + orte_errmgr.abort(1, NULL); + } } - /* limit us to 87.5% of the registered memory (some fluff for QPs, file systems, etc) */ + /* Limit us to 87.5% of the registered memory (some fluff for QPs, + file systems, etc) */ return (max_reg * 7) >> 3; } diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index d2ca387d10..baa5ecf87c 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -267,6 +267,10 @@ struct mca_btl_openib_component_t { /** Whether we want a warning if the user specifies a non-existent device and/or port via btl_openib_if_[in|ex]clude MCA params */ bool warn_nonexistent_if; + /** Whether we want to abort if there's not enough registered + memory available */ + bool abort_not_enough_reg_mem; + /** Dummy argv-style list; a copy of names from the if_[in|ex]clude list that we use for error checking (to ensure that they all exist) */ diff --git a/ompi/mca/btl/openib/btl_openib_mca.c b/ompi/mca/btl/openib/btl_openib_mca.c index f84b41c105..8c10aba39e 100644 --- a/ompi/mca/btl/openib/btl_openib_mca.c +++ b/ompi/mca/btl/openib/btl_openib_mca.c @@ -166,6 +166,14 @@ int btl_openib_register_mca_params(void) 1, &ival, 0)); mca_btl_openib_component.warn_nonexistent_if = (0 != ival); + /* If we print a warning about not having enough registered memory + available, do we want to abort? */ + CHECK(reg_int("abort_not_enough_reg_mem", NULL, + "If there is not enough registered memory available on the system for Open MPI to function properly, Open MPI will issue a warning. If this MCA parameter is set to true, then Open MPI will also abort all MPI jobs " + "(0 = warn, but do not abort; any other value = warn and abort)", + 0, &ival, 0)); + mca_btl_openib_component.abort_not_enough_reg_mem = (0 != ival); + if (OMPI_HAVE_IBV_FORK_INIT) { ival2 = -1; } else { diff --git a/ompi/mca/btl/openib/help-mpi-btl-openib.txt b/ompi/mca/btl/openib/help-mpi-btl-openib.txt index db137ab084..5245b2b859 100644 --- a/ompi/mca/btl/openib/help-mpi-btl-openib.txt +++ b/ompi/mca/btl/openib/help-mpi-btl-openib.txt @@ -708,3 +708,5 @@ parameters: Local host: %s Registerable memory: %lu MiB Total memory: %lu MiB + +%s