0d52271cd6
and https://svn.open-mpi.org/trac/ompi/ticket/1853, mallopt() hints do not always work -- it is possible for memory to be returned to the OS and therefore OMPI's registration cache becomes invalid. This commit removes all use of mallopt() and uses a different way to integrate ptmalloc2 than we have done in the past. In particular, we use almost exactly the same technique as MX: * Remove all uses of mallopt, to include the opal/memory mallopt component. * Name-shift all of OMPI's internal ptmalloc2 public symbols (e.g., malloc -> opal_memory_ptmalloc2_malloc). * At run-time, use the existing glibc allocator malloc hook function pointers to fully hijack the glibc allocator with our own name-shifted ptmalloc2. * Make the decision whether to hijack the glibc allocator ''at run time'' (vs. at link time, as previous ptmalloc2 integration attempts have done). Look at the OMPI_MCA_mpi_leave_pinned and OMPI_MCA_mpi_leave_pinned_pipeline environment variables and the existence of /sys/class/infiniband to determine if we should install the hooks or not. * As an added bonus, we can now tell if libopen-pal is linked statically or dynamically, and if we're linked statically, we assume that munmap intercept support doesn't work. See the opal/mca/memory/ptmalloc2/README-open-mpi.txt file for all the gory details about the implementation. Fixes trac:1853. This commit was SVN r20921. The following Trac tickets were found above: Ticket 1853 --> https://svn.open-mpi.org/trac/ompi/ticket/1853
123 строки
3.9 KiB
C
123 строки
3.9 KiB
C
/*
|
|
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2006 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2008 Myricom. All rights reserved.
|
|
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
|
|
* Copyright (c) 2009 Cisco systems, Inc. All rights reserved.
|
|
*
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "ompi_config.h"
|
|
#include "ompi/constants.h"
|
|
#include "common_mx.h"
|
|
|
|
#include <errno.h>
|
|
#ifdef HAVE_MALLOC_H
|
|
#include <malloc.h>
|
|
#endif
|
|
#include "opal/memoryhooks/memory.h"
|
|
#include "opal/util/output.h"
|
|
#include "opal/mca/base/mca_base_param.h"
|
|
#include "ompi/runtime/params.h"
|
|
#include "ompi/mca/mpool/mpool.h"
|
|
#include "ompi/mca/mpool/base/base.h"
|
|
#include "ompi/mca/mpool/fake/mpool_fake.h"
|
|
|
|
|
|
int mx__regcache_clean(void *ptr, size_t size);
|
|
|
|
static int ompi_common_mx_initialize_ref_cnt = 0;
|
|
static mca_mpool_base_module_t *ompi_common_mx_fake_mpool = 0;
|
|
|
|
int
|
|
ompi_common_mx_initialize(void)
|
|
{
|
|
mx_return_t mx_return;
|
|
struct mca_mpool_base_resources_t mpool_resources;
|
|
int index, value;
|
|
|
|
ompi_common_mx_initialize_ref_cnt++;
|
|
|
|
if(ompi_common_mx_initialize_ref_cnt == 1) {
|
|
/* set the MX error handle to always return. This function is the
|
|
* only MX function allowed to be called before mx_init in order
|
|
* to make sure that if the MX is not up and running the MX
|
|
* library does not exit the application.
|
|
*/
|
|
mx_set_error_handler(MX_ERRORS_RETURN);
|
|
|
|
/* If we have a memory manager available, and
|
|
mpi_leave_pinned == -1, then set mpi_leave_pinned to 1.
|
|
|
|
We have a memory manager if:
|
|
- we have both FREE and MUNMAP support
|
|
- we have MUNMAP support and the linux mallopt */
|
|
value = opal_mem_hooks_support_level();
|
|
if ((value & (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
|
|
== (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)) {
|
|
index = mca_base_param_find("mpi", NULL, "leave_pinned");
|
|
if (index >= 0)
|
|
if ((mca_base_param_lookup_int(index, &value) == OPAL_SUCCESS)
|
|
&& (value == -1)) {
|
|
|
|
ompi_mpi_leave_pinned = 1;
|
|
setenv("MX_RCACHE", "2", 1);
|
|
mpool_resources.regcache_clean = mx__regcache_clean;
|
|
ompi_common_mx_fake_mpool =
|
|
mca_mpool_base_module_create("fake", NULL, &mpool_resources);
|
|
if (!ompi_common_mx_fake_mpool) {
|
|
ompi_mpi_leave_pinned = 0;
|
|
setenv("MX_RCACHE", "0", 1);
|
|
opal_output(0, "Error creating fake mpool (error %s)\n",
|
|
strerror(errno));
|
|
}
|
|
}
|
|
}
|
|
|
|
/* initialize the mx library */
|
|
mx_return = mx_init();
|
|
|
|
if(MX_SUCCESS != mx_return) {
|
|
opal_output(0,
|
|
"Error in mx_init (error %s)\n",
|
|
mx_strerror(mx_return));
|
|
return OMPI_ERR_NOT_AVAILABLE;
|
|
}
|
|
|
|
}
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
int
|
|
ompi_common_mx_finalize(void)
|
|
{
|
|
mx_return_t mx_return;
|
|
ompi_common_mx_initialize_ref_cnt--;
|
|
if( 0 == ompi_common_mx_initialize_ref_cnt ) {
|
|
|
|
if (ompi_common_mx_fake_mpool)
|
|
mca_mpool_base_module_destroy(ompi_common_mx_fake_mpool);
|
|
|
|
mx_return = mx_finalize();
|
|
if(mx_return != MX_SUCCESS){
|
|
opal_output(0, "Error in mx_finalize (error %s)\n", mx_strerror(mx_return));
|
|
return OMPI_ERROR;
|
|
}
|
|
}
|
|
return OMPI_SUCCESS;
|
|
}
|