and https://svn.open-mpi.org/trac/ompi/ticket/1853, mallopt() hints do not always work -- it is possible for memory to be returned to the OS and therefore OMPI's registration cache becomes invalid. This commit removes all use of mallopt() and uses a different way to integrate ptmalloc2 than we have done in the past. In particular, we use almost exactly the same technique as MX: * Remove all uses of mallopt, to include the opal/memory mallopt component. * Name-shift all of OMPI's internal ptmalloc2 public symbols (e.g., malloc -> opal_memory_ptmalloc2_malloc). * At run-time, use the existing glibc allocator malloc hook function pointers to fully hijack the glibc allocator with our own name-shifted ptmalloc2. * Make the decision whether to hijack the glibc allocator ''at run time'' (vs. at link time, as previous ptmalloc2 integration attempts have done). Look at the OMPI_MCA_mpi_leave_pinned and OMPI_MCA_mpi_leave_pinned_pipeline environment variables and the existence of /sys/class/infiniband to determine if we should install the hooks or not. * As an added bonus, we can now tell if libopen-pal is linked statically or dynamically, and if we're linked statically, we assume that munmap intercept support doesn't work. See the opal/mca/memory/ptmalloc2/README-open-mpi.txt file for all the gory details about the implementation. Fixes trac:1853. This commit was SVN r20921. The following Trac tickets were found above: Ticket 1853 --> https://svn.open-mpi.org/trac/ompi/ticket/1853
Этот коммит содержится в:
Jeff Squyres
родитель
b7a052a81d
Коммит
0d52271cd6
@ -10,7 +10,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
|
||||
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
@ -1723,8 +1723,8 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev)
|
||||
mca_mpool_base_module_create(mca_btl_openib_component.ib_mpool_name,
|
||||
device, &mpool_resources);
|
||||
if(NULL == device->mpool){
|
||||
BTL_ERROR(("error creating IB memory pool for %s errno says %s",
|
||||
ibv_get_device_name(device->ib_dev), strerror(errno)));
|
||||
/* Don't print an error message here -- we'll get one from
|
||||
mpool_create anyway (OPAL_SOS would be good here...) */
|
||||
goto error;
|
||||
}
|
||||
|
||||
@ -2099,13 +2099,10 @@ btl_openib_component_init(int *num_btl_modules,
|
||||
mpi_leave_pinned==-1, then unless the user explicitly set
|
||||
mpi_leave_pinned_pipeline==0, then set mpi_leave_pinned to 1.
|
||||
|
||||
We have a memory manager if:
|
||||
- we have both FREE and MUNMAP support
|
||||
- we have MUNMAP support and the linux mallopt */
|
||||
if (((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
|
||||
((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) ||
|
||||
(0 != (OPAL_MEMORY_MUNMAP_SUPPORT & value) &&
|
||||
OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT)) {
|
||||
We have a memory manager if we have both FREE and MUNMAP
|
||||
support */
|
||||
if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
|
||||
((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) {
|
||||
ret = 0;
|
||||
index = mca_base_param_find("mpi", NULL, "leave_pinned");
|
||||
if (index >= 0) {
|
||||
|
||||
@ -224,9 +224,13 @@ You may need to consult with your system administrator to get this
|
||||
problem fixed.
|
||||
#
|
||||
[no active ports found]
|
||||
WARNING: There is at least one OpenFabrics device found on host '%s',
|
||||
but there are no active ports detected. This is most certainly not
|
||||
what you wanted. Check your cables, SM configuration, etc.
|
||||
WARNING: There is at least one OpenFabrics device found but there are
|
||||
no active ports detected (or Open MPI was unable to use them). This
|
||||
is most certainly not what you wanted. Check your cables, subnet
|
||||
manager configuration, etc. The openib BTL will be ignored for this
|
||||
job.
|
||||
|
||||
Local host: %s
|
||||
#
|
||||
[error in device init]
|
||||
WARNING: There was an error initializing an OpenFabrics device.
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Myricom. All rights reserved.
|
||||
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
|
||||
* Copyright (c) 2009 Cisco systems, Inc. All rights reserved.
|
||||
*
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -65,10 +66,8 @@ ompi_common_mx_initialize(void)
|
||||
- we have both FREE and MUNMAP support
|
||||
- we have MUNMAP support and the linux mallopt */
|
||||
value = opal_mem_hooks_support_level();
|
||||
if (((value & (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
|
||||
== (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
|
||||
|| ((value & OPAL_MEMORY_MUNMAP_SUPPORT) &&
|
||||
OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT)) {
|
||||
if ((value & (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
|
||||
== (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)) {
|
||||
index = mca_base_param_find("mpi", NULL, "leave_pinned");
|
||||
if (index >= 0)
|
||||
if ((mca_base_param_lookup_int(index, &value) == OPAL_SUCCESS)
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -30,15 +30,7 @@
|
||||
#include "ompi/mca/mpool/mpool.h"
|
||||
#include "opal/threads/mutex.h"
|
||||
|
||||
#if defined(HAVE_MALLOPT) && defined(M_TRIM_THRESHOLD) && defined(M_MMAP_MAX)
|
||||
#define OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT 1
|
||||
#else
|
||||
#define OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT 0
|
||||
#endif
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
BEGIN_C_DECLS
|
||||
|
||||
static inline unsigned int my_log2(unsigned long val) {
|
||||
unsigned int count = 0;
|
||||
@ -97,11 +89,7 @@ OMPI_DECLSPEC extern uint32_t mca_mpool_base_page_size_log;
|
||||
|
||||
/* only used within base -- no need to DECLSPEC */
|
||||
extern int mca_mpool_base_used_mem_hooks;
|
||||
extern int mca_mpool_base_use_mem_hooks_index;
|
||||
extern int mca_mpool_base_disable_mallopt_index;
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_MEM_BASE_H */
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# -*- text -*-
|
||||
#
|
||||
# Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
@ -30,8 +30,20 @@ PID: %d
|
||||
%d additional leak%s recorded but %s not displayed here. Set the MCA
|
||||
parameter mpi_show_mpi_alloc_mem_leaks to a larger number to see that
|
||||
many leaks, or set it to a negative number to see all leaks.
|
||||
#
|
||||
[leave pinned failed]
|
||||
Process %s on host %s attempted to use the leave pinned
|
||||
feature, but no memory registration hooks were found. To solve this
|
||||
problem, either link in the libopenmpi-malloc library or (on Linux)
|
||||
do not set the MCA parameter base_disable_mallopt.
|
||||
A process attempted to use the "leave pinned" MPI feature, but no
|
||||
memory registration hooks were found on the system at run time. This
|
||||
may be the result of running on a system that does not support memory
|
||||
hooks or having some other software subvert Open MPI's use of the
|
||||
memory hooks. You can disable Open MPI's use of memory hooks by
|
||||
setting both the mpi_leave_pinned and mpi_leave_pinned_pipeline MCA
|
||||
parameters to 0.
|
||||
|
||||
Open MPI will disable any transports that are attempting to use the
|
||||
leave pinned functionality; your job may still run, but may fall back
|
||||
to a slower network transport (such as TCP).
|
||||
|
||||
Mpool name: %s
|
||||
Process: %s
|
||||
Local host: %s
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -71,7 +71,6 @@ mca_mpool_base_module_t* mca_mpool_base_module_create(
|
||||
mca_mpool_base_module_t* module = NULL;
|
||||
opal_list_item_t* item;
|
||||
mca_mpool_base_selected_module_t *sm;
|
||||
int use_mem_hooks, disable_mallopt;
|
||||
|
||||
for (item = opal_list_get_first(&mca_mpool_base_components);
|
||||
item != opal_list_get_end(&mca_mpool_base_components);
|
||||
@ -99,45 +98,30 @@ mca_mpool_base_module_t* mca_mpool_base_module_create(
|
||||
sm->mpool_resources = resources;
|
||||
opal_list_append(&mca_mpool_base_modules, (opal_list_item_t*) sm);
|
||||
/* on the very first creation of a module we init the memory
|
||||
callback and (if needed) disable free() returning memory to the
|
||||
OS. Note that even when we disable free() with mallopt, we
|
||||
still need to register a callback to handle the case of the
|
||||
user calling mmap/munmap on his own. */
|
||||
callback */
|
||||
if (opal_list_get_size(&mca_mpool_base_modules) == 1) {
|
||||
/* Default to not using memory hooks */
|
||||
int use_mem_hooks = 0;
|
||||
|
||||
/* Lookup the current value of the MCA params and see if any
|
||||
other entity in the code base requested mem hooks */
|
||||
mca_base_param_lookup_int(mca_mpool_base_use_mem_hooks_index,
|
||||
&use_mem_hooks);
|
||||
mca_base_param_lookup_int(mca_mpool_base_disable_mallopt_index,
|
||||
&disable_mallopt);
|
||||
|
||||
/* force mem hooks if leave_pinned or leave_pinned_pipeline is
|
||||
enabled (note that either of these leave_pinned variables
|
||||
may have been set by a user MCA param or elsewhere in the
|
||||
code base) */
|
||||
if (1 == ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline) {
|
||||
/* Use the memory hooks if leave_pinned or
|
||||
leave_pinned_pipeline is enabled (note that either of these
|
||||
leave_pinned variables may have been set by a user MCA
|
||||
param or elsewhere in the code base). Yes, we could have
|
||||
coded this more succinctly, but this is more clear. */
|
||||
if (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline) {
|
||||
use_mem_hooks = 1;
|
||||
}
|
||||
|
||||
if (use_mem_hooks) {
|
||||
if (0 != (OPAL_MEMORY_FREE_SUPPORT & opal_mem_hooks_support_level())) {
|
||||
if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
|
||||
((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
|
||||
opal_mem_hooks_support_level())) {
|
||||
opal_mem_hooks_register_release(mca_mpool_base_mem_cb, NULL);
|
||||
OBJ_CONSTRUCT(&mca_mpool_base_mem_cb_array, opal_pointer_array_t);
|
||||
} else if (!disable_mallopt &&
|
||||
0 != (OPAL_MEMORY_MUNMAP_SUPPORT & opal_mem_hooks_support_level())) {
|
||||
opal_mem_hooks_register_release(mca_mpool_base_mem_cb, NULL);
|
||||
OBJ_CONSTRUCT(&mca_mpool_base_mem_cb_array, opal_pointer_array_t);
|
||||
/* mallopt_disable_free will only be set to 1 if
|
||||
HAVE_LINUX_MALLOPT, so this is safe */
|
||||
#if OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT
|
||||
mallopt(M_TRIM_THRESHOLD, -1);
|
||||
mallopt(M_MMAP_MAX, 0);
|
||||
#endif
|
||||
} else {
|
||||
orte_show_help("help-mpool-base.txt", "leave pinned failed",
|
||||
true, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
orte_process_info.nodename);
|
||||
orte_process_info.nodename, name);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -50,11 +50,6 @@ int mca_mpool_base_output = -1;
|
||||
|
||||
/* whether we actually used the mem hooks or not */
|
||||
int mca_mpool_base_used_mem_hooks = 0;
|
||||
/* should we attempt to use the available memory hooks */
|
||||
int mca_mpool_base_use_mem_hooks_index;
|
||||
/* should we attempt to use mallopt to disable free() returning memory
|
||||
to OS? */
|
||||
int mca_mpool_base_disable_mallopt_index;
|
||||
|
||||
uint32_t mca_mpool_base_page_size;
|
||||
uint32_t mca_mpool_base_page_size_log;
|
||||
@ -82,37 +77,6 @@ int mca_mpool_base_open(void)
|
||||
|
||||
OBJ_CONSTRUCT(&mca_mpool_base_modules, opal_list_t);
|
||||
|
||||
/*
|
||||
* check for use_mem_hooks (for diagnostics/testing)
|
||||
* however if leave_pinned is set we force this to be enabled
|
||||
*/
|
||||
mca_mpool_base_use_mem_hooks_index =
|
||||
mca_base_param_reg_int_name("mpool",
|
||||
"base_use_mem_hooks",
|
||||
"Use memory hooks for deregistering freed memory",
|
||||
false,
|
||||
false,
|
||||
0,
|
||||
NULL);
|
||||
mca_base_param_reg_syn_name(mca_mpool_base_use_mem_hooks_index,
|
||||
"mpool", "use_mem_hooks", true);
|
||||
|
||||
mca_mpool_base_disable_mallopt_index =
|
||||
mca_base_param_reg_int_name("mpool",
|
||||
"base_disable_mallopt",
|
||||
"Do not use mallopt to disable returning memory to "
|
||||
"the OS when leave_pinned is active and no memory "
|
||||
"components are found (this value is only changable on Linux systems that support mallopt()).",
|
||||
false,
|
||||
#if OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT
|
||||
false,
|
||||
0,
|
||||
#else
|
||||
true,
|
||||
1,
|
||||
#endif
|
||||
NULL);
|
||||
|
||||
/* get the page size for this architecture*/
|
||||
mca_mpool_base_page_size = sysconf(_SC_PAGESIZE);
|
||||
mca_mpool_base_page_size_log = my_log2(mca_mpool_base_page_size);
|
||||
|
||||
@ -38,30 +38,29 @@ AM_CPPFLAGS += -I$(srcdir)/sysdeps/generic
|
||||
docdir = $(datadir)/openmpi/doc
|
||||
doc_DATA = ptmalloc2-COPYRIGHT
|
||||
|
||||
noinst_LTLIBRARIES = libmca_memory_ptmalloc2.la
|
||||
# These are the files from ptmalloc2 that we care about
|
||||
ptmalloc2_sources = \
|
||||
malloc.c \
|
||||
malloc-stats.c \
|
||||
malloc.h
|
||||
|
||||
# This component is only ever built statically (i.e., slurped into
|
||||
# libopen-pal) -- it is never built as a DSO.
|
||||
noinst_LTLIBRARIES = libmca_memory_ptmalloc2.la
|
||||
libmca_memory_ptmalloc2_la_SOURCES = \
|
||||
opal_ptmalloc2_component.c \
|
||||
opal_ptmalloc2_munmap.c
|
||||
|
||||
if OMPI_WANT_EXTERNAL_PTMALLOC2
|
||||
lib_LTLIBRARIES = libopenmpi-malloc.la
|
||||
|
||||
libopenmpi_malloc_la_SOURCES = \
|
||||
malloc.c \
|
||||
malloc-stats.c \
|
||||
malloc.h
|
||||
else
|
||||
libmca_memory_ptmalloc2_la_SOURCES += \
|
||||
malloc.c \
|
||||
malloc-stats.c \
|
||||
malloc.h
|
||||
endif
|
||||
|
||||
|
||||
libmca_memory_ptmalloc2_la_LDFLAGS = -module -avoid-version $(memory_ptmalloc2_LDFLAGS)
|
||||
opal_ptmalloc2_munmap.c \
|
||||
rename.h \
|
||||
$(ptmalloc2_sources)
|
||||
libmca_memory_ptmalloc2_la_LDFLAGS = \
|
||||
-module -avoid-version $(memory_ptmalloc2_LDFLAGS)
|
||||
libmca_memory_ptmalloc2_la_LIBADD = $(memory_ptmalloc2_LIBS)
|
||||
|
||||
# For hysterical raisins, we create a dummy libopenmpi_malloc.la. See
|
||||
# README-open-mpi.txt for details
|
||||
lib_LTLIBRARIES = libopenmpi_malloc.la
|
||||
libopenmpi_malloc_la_SOURCES = dummy.c
|
||||
|
||||
# these are included directly and shouldn't be built solo
|
||||
EXTRA_libmca_memory_ptmalloc2_la_SOURCES = \
|
||||
arena.c \
|
||||
|
||||
150
opal/mca/memory/ptmalloc2/README-open-mpi.txt
Обычный файл
150
opal/mca/memory/ptmalloc2/README-open-mpi.txt
Обычный файл
@ -0,0 +1,150 @@
|
||||
30 March 2009
|
||||
|
||||
This file documents Open MPI's usage of ptmalloc2. This is perhaps
|
||||
our 7,208,499th iteration of ptmalloc2 support, so let's document it
|
||||
here so that some future developer might spend *slightly* less time
|
||||
understanding what the heck is going on.
|
||||
|
||||
See glibc documentation about malloc hooks before continuing. This is
|
||||
pretty much required reading before reading the rest of this file /
|
||||
having a hope of understanding what's going on here:
|
||||
|
||||
http://www.gnu.org/software/libc/manual/html_mono/libc.html#Hooks-for-Malloc
|
||||
|
||||
The overall goal is that we're using the Linux glibc hooks to wholly
|
||||
replace the underlying allocator. We *used* to use horrid linker
|
||||
tricks to interpose OMPI's ptmalloc2 symbols with the glibc ones --
|
||||
meaning that user apps would call our symbols and not the glibc ones.
|
||||
But that scheme is fraught with problems, not the least of which is
|
||||
that *all* MPI applications will be forced to use our overridden
|
||||
allocator (not just the ones that need it, such as the ones running on
|
||||
OpenFabrics-based networks). Instead, what we do here is, frankly,
|
||||
quite similar to what is done in MX: we use the 4 glibc hooks to
|
||||
assert our own malloc, realloc, free, and memalign functions. This
|
||||
allows the decision as to whether to use this internal ptmalloc2
|
||||
allocate to be a run-time decision. This is quite important; using
|
||||
this internal allocator has both benefits (allowing using
|
||||
mpi_leave_pinned=1 behavior) and drawbacks (breaking some debuggers,
|
||||
being unnecessary for non-OpenFabrics-based networks, etc.).
|
||||
|
||||
Here's how it works...
|
||||
|
||||
This component *must* be linked statically as part of libopen-pal; it
|
||||
*cannot* be a DSO. Specifically, this library must be present during
|
||||
pre-main() initialization phases so that its __malloc_initialize_hook
|
||||
can be found and executed. Loading it as a DSO during MPI_INIT is far
|
||||
too late. In configure.m4, we define the M4 macro
|
||||
MCA_memory_ptmalloc2_COMPILE_MODE to always compile this component in
|
||||
static mode. Yay flexible build system.
|
||||
|
||||
This component provides an munmap() function that will intercept calls
|
||||
to munmap() and do the Right Thing. That is fairly straightforward to
|
||||
do. Intercepting the malloc/free/etc. allocator is much more
|
||||
complicated.
|
||||
|
||||
All the ptmalloc2 public symbols in this component have been name
|
||||
shifted via the rename.h file. Hence, what used to be "malloc" is now
|
||||
opal_memory_ptmalloc2_malloc. Since all the public symbols are
|
||||
name-shifted, we can safely link this component in all MPI
|
||||
applications. Specifically: just because this ptmalloc2 allocator is
|
||||
present in all OMPI executables and user-level applications, it won't
|
||||
necessarily be used -- it's a separate/run-time decision as to whether
|
||||
it will be used.
|
||||
|
||||
For hysterical raisins (v1.3.0 and v1.3.1), an empty
|
||||
libopenmpi_malloc.la is created. This library *used* to be the whole
|
||||
ptmalloc2 allocator, but since we're now name-shifting all the
|
||||
ptmalloc2 symbols, it's no longer necessary to make it a separate
|
||||
library. Having an empty/dummy library of this name ensures that
|
||||
users who added -lopenmpi_malloc to their compile/link lines won't
|
||||
fail linking. They may get a linker warning about no symbols being
|
||||
used in this library -- but that's a Good Thing; it'll encourage users
|
||||
to stop linking in this library.
|
||||
|
||||
We set the __malloc_initialize_hook variable to point to
|
||||
opal_memory_ptmalloc2_malloc_init_hook (in hooks.c). This function is
|
||||
called by the underlying glibc allocator before any allocations occur
|
||||
and before the memory allocation subsystem is setup. As such, this
|
||||
function is *extremely* restricted in what it can do. It cannot call
|
||||
any form of malloc, for example (which seems fairly obvious, but it's
|
||||
worth mentioning :-) ). This function is one of the determining
|
||||
steps as to whether we'll use the internal ptmalloc2 allocator or
|
||||
not. It performs checks including (but not limited to; see hooks.c
|
||||
for the most up-to-date set of tests):
|
||||
|
||||
- see if the OMPI_MCA_mpi_leave_pinned environment variable is set.
|
||||
Yes, I know this is a horrid abstraction violation, but this
|
||||
function may be invoked pre-main -- it's certainly before MCA
|
||||
parameters have been setup. So just getenv() and see if it has been
|
||||
set.
|
||||
- look for the hard-coded filename /sys/class/infiniband.
|
||||
- if the env variable was not set, but the file is there, enable our
|
||||
ptmalloc2.
|
||||
- if the env variable was set to 0, disable our ptmalloc2
|
||||
- if the env variable is set to -1, enable our ptmalloc2 if the file
|
||||
was found
|
||||
- if the env variable is any other value, enable our ptmalloc2
|
||||
- if the file is a static library, then override all of the above and
|
||||
disable our ptmalloc2
|
||||
- if we're enabling our ptmalloc2, initialize ptmalloc (via
|
||||
ptmalloc_init()) and then set the 4 hooks to point to our
|
||||
name-shifted ptmalloc2 functions
|
||||
|
||||
Hence, sometime during process startup, this function will definitely
|
||||
be called. It will either set the 4 hook functions to point to our
|
||||
name-shifted ptmalloc2 functions, or it won't. If the 4 hook
|
||||
functions are set, then the underlying glibc allocator will always
|
||||
call our 4 functions in all the relevant places instead of calling its
|
||||
own functions. Specifically: the process is calling the underlying
|
||||
glibc allocator, but that underlying glibc allocator will make
|
||||
function pointer callbacks to our name-shifted ptmalloc2 functions to
|
||||
actually do the work.
|
||||
|
||||
Note that because we know our ptmalloc will not be providing all 5
|
||||
hook variables (because we want to use the underlying glibc hook
|
||||
variables), they are #if 0'ed out in our malloc.c. This has the
|
||||
direct consequence that the *_hook_ini() in hooks.c are never used.
|
||||
So to avoid compiler/linker warnings, I #if 0'ed those out as well.
|
||||
|
||||
All the public functions in malloc.c that call hook functions were
|
||||
modified to #if 0 the hook function invocations. After all, that's
|
||||
something that we want the *underlying* glibc allocator to do -- but
|
||||
we are putting these functions as the hooks, so we don't want to
|
||||
invoke ourselves in an infinite loop!
|
||||
|
||||
The next thing that happens in the startup sequence is that the
|
||||
ptmalloc2 memory component's "open" function is called during
|
||||
MPI_INIT. As stated above, we always intercept munmap() -- this is
|
||||
acceptable in all environments. But we need to test to see if the
|
||||
glibc memory hooks have been overridden before MPI_INIT was invoked.
|
||||
If so, we need to signal that our allocator support may not be
|
||||
complete.
|
||||
|
||||
Patrick Geofray/MX suggests a simple test: malloc() 4MB and then free
|
||||
it. Watch to see if our name-shifted ptmalloc2 free() function was
|
||||
invoked. If it was, then all of our hooks are probably in place and
|
||||
we can proceed. If not, then set flags indicating that this memory
|
||||
allocator only supports MUNMAP (not FREE/CHUNK).
|
||||
|
||||
NOTE: we *used* to simply set the FREE/CHUNK support flags during our
|
||||
ptmalloc2's internal ptmalloc_init() function. This is not a good
|
||||
idea becaus eeven after our ptmalloc_init() function has been invoked,
|
||||
someone may come in an override our memory hooks. Doing a malloc/free
|
||||
test during the ptmalloc2 memory component's open function seems to be
|
||||
the safest way to test whether we *actually* support FREE/CHUNK (this
|
||||
is what MX does, too).
|
||||
|
||||
Much later in the init sequence during MPI_INIT, components indicate
|
||||
whether they want to use mpi_leave_pinned[_pipeline] support or not.
|
||||
For example, the openib BTL queries the opal_mem_hooks_support_level()
|
||||
function to see if FREE and MUNMAP are supported. If they are, then
|
||||
the openib BTL sets mpi_leave_pinned = 1.
|
||||
|
||||
Finally, the mpool base does a final check. If
|
||||
mpi_leave_pinned[_pipeline] is set to 1 and/or use_mem_hooks is set,
|
||||
if FREE/MUNMAP are not set in the supported flags, then a warning is
|
||||
printed. Otherwise, life continues (assumedly using
|
||||
mpi_leave_pinned[_pipeline] support).
|
||||
|
||||
Simple, right?
|
||||
|
||||
@ -465,15 +465,6 @@ ptmalloc_init __MALLOC_P((void))
|
||||
if(__malloc_initialize_hook != NULL)
|
||||
(*__malloc_initialize_hook)();
|
||||
|
||||
/********************** BEGIN OMPI CHANGES *****************************/
|
||||
/* don't use __hook for this, as someone might want to use those
|
||||
features */
|
||||
opal_mem_hooks_set_support(OPAL_MEMORY_FREE_SUPPORT |
|
||||
OPAL_MEMORY_MUNMAP_SUPPORT |
|
||||
OPAL_MEMORY_CHUNK_SUPPORT);
|
||||
|
||||
/********************* BEGIN OMPI CHANGES ******************************/
|
||||
|
||||
__malloc_initialized = 1;
|
||||
}
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
@ -25,27 +25,10 @@ AC_DEFUN([MCA_memory_ptmalloc2_COMPILE_MODE], [
|
||||
])
|
||||
|
||||
|
||||
AC_DEFUN([MCA_memory_ptmalloc2_POST_CONFIG],[
|
||||
AM_CONDITIONAL([OMPI_WANT_EXTERNAL_PTMALLOC2],
|
||||
[test "$enable_ptmalloc2_internal" != "yes"])
|
||||
])
|
||||
|
||||
|
||||
# MCA_memory_ptmalloc2_CONFIG(action-if-can-compile,
|
||||
# [action-if-cant-compile])
|
||||
# ------------------------------------------------
|
||||
AC_DEFUN([MCA_memory_ptmalloc2_CONFIG],[
|
||||
AC_ARG_ENABLE([ptmalloc2-internal],
|
||||
[AC_HELP_STRING([--enable-ptmalloc2-internal],
|
||||
[Build ptmalloc2 memory manager into libopen-pal,
|
||||
instead of as separate library. Only has meaning
|
||||
if ptmalloc2 memory component exists])])
|
||||
|
||||
AC_MSG_CHECKING([if ptmalloc2 should be part of libopen-pal])
|
||||
AS_IF([test "$enable_ptmalloc2_internal" = "yes"],
|
||||
[AC_MSG_RESULT([yes])], [AC_MSG_RESULT([no])])
|
||||
|
||||
|
||||
AS_IF([test "$with_memory_manager" = "ptmalloc2"],
|
||||
[if test "`echo $host | grep apple-darwin`" != "" ; then
|
||||
AC_MSG_WARN([*** Using ptmalloc with OS X will result in failure.])
|
||||
@ -90,6 +73,12 @@ AC_DEFUN([MCA_memory_ptmalloc2_CONFIG],[
|
||||
[memory_ptmalloc2_happy="yes"],
|
||||
[memory_ptmalloc2_happy="no"])])
|
||||
|
||||
AS_IF([test "$memory_ptmalloc2_happy" = "yes"],
|
||||
[# check for link.h (for _DYNAMIC symbol)
|
||||
AC_CHECK_HEADER([link.h],
|
||||
[memory_ptmalloc2_happy="yes"],
|
||||
[memory_ptmalloc2_happy="no"])])
|
||||
|
||||
AS_IF([test "$memory_ptmalloc2_happy" = "yes"],
|
||||
[# check for init hook symbol
|
||||
AC_CHECK_DECL([__malloc_initialize_hook],
|
||||
|
||||
12
opal/mca/memory/ptmalloc2/dummy.c
Обычный файл
12
opal/mca/memory/ptmalloc2/dummy.c
Обычный файл
@ -0,0 +1,12 @@
|
||||
/*
|
||||
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
int opal_this_library_is_not_used_anymore(void) {
|
||||
return 0;
|
||||
}
|
||||
@ -33,6 +33,9 @@
|
||||
|
||||
#if !(USE_STARTER & 2)
|
||||
|
||||
/* OMPI change: these aren't used (avoid a compiler warning by if
|
||||
0'ing them out */
|
||||
#if 0
|
||||
static Void_t*
|
||||
#if __STD_C
|
||||
malloc_hook_ini(size_t sz, const __malloc_ptr_t caller)
|
||||
@ -45,7 +48,11 @@ malloc_hook_ini(sz, caller)
|
||||
ptmalloc_init();
|
||||
return public_mALLOc(sz);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* OMPI change: these aren't used (avoid a compiler warning by if
|
||||
0'ing them out */
|
||||
#if 0
|
||||
static Void_t*
|
||||
#if __STD_C
|
||||
realloc_hook_ini(Void_t* ptr, size_t sz, const __malloc_ptr_t caller)
|
||||
@ -59,7 +66,11 @@ realloc_hook_ini(ptr, sz, caller)
|
||||
ptmalloc_init();
|
||||
return public_rEALLOc(ptr, sz);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* OMPI change: these aren't used (avoid a compiler warning by if
|
||||
0'ing them out */
|
||||
#if 0
|
||||
static Void_t*
|
||||
#if __STD_C
|
||||
memalign_hook_ini(size_t alignment, size_t sz, const __malloc_ptr_t caller)
|
||||
@ -72,6 +83,7 @@ memalign_hook_ini(alignment, sz, caller)
|
||||
ptmalloc_init();
|
||||
return public_mEMALIGn(alignment, sz);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* !(USE_STARTER & 2) */
|
||||
|
||||
@ -633,8 +645,134 @@ public_sET_STATe(Void_t* msptr)
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
Per
|
||||
http://www.gnu.org/software/libc/manual/html_mono/libc.html#Hooks-for-Malloc,
|
||||
we can define the __malloc_initialize_hook variable to be a
|
||||
function that is invoked before the first allocation is ever
|
||||
performed. We use this hook to wholly replace the underlying
|
||||
allocator to our own allocator if a few conditions are met.
|
||||
|
||||
Remember that this hook is called probably at the very very very
|
||||
beginning of the process. MCA parameters haven't been setup yet --
|
||||
darn near nothing has been setup yet. Indeed, we're effectively in
|
||||
signal context because we can't call anything that calls malloc.
|
||||
So we can basically have some hard-coded tests for things to see if
|
||||
we want to setup to use our internal ptmalloc2 or not. */
|
||||
|
||||
static void *opal_memory_ptmalloc2_malloc_hook(size_t sz,
|
||||
const __malloc_ptr_t caller)
|
||||
{
|
||||
return public_mALLOc(sz);
|
||||
}
|
||||
|
||||
static void *opal_memory_ptmalloc2_realloc_hook(Void_t* ptr, size_t sz,
|
||||
const __malloc_ptr_t caller)
|
||||
{
|
||||
return public_rEALLOc(ptr, sz);
|
||||
}
|
||||
|
||||
static void *opal_memory_ptmalloc2_memalign_hook(size_t alignment, size_t sz,
|
||||
const __malloc_ptr_t caller)
|
||||
{
|
||||
return public_mEMALIGn(alignment, sz);
|
||||
}
|
||||
|
||||
static void opal_memory_ptmalloc2_free_hook(__malloc_ptr_t __ptr,
|
||||
const __malloc_ptr_t caller)
|
||||
{
|
||||
public_fREe(__ptr);
|
||||
}
|
||||
|
||||
typedef enum {
|
||||
RESULT_NO,
|
||||
RESULT_YES,
|
||||
RESULT_RUNTIME,
|
||||
RESULT_NOT_FOUND
|
||||
} check_result_t;
|
||||
|
||||
static check_result_t check(const char *name)
|
||||
{
|
||||
char *s = getenv(name);
|
||||
if (NULL == s) {
|
||||
return RESULT_NOT_FOUND;
|
||||
}
|
||||
|
||||
if ('0' == s[0] && '\0' == s[1]) {
|
||||
/* A value of 0 means "don't use!" */
|
||||
return RESULT_NO;
|
||||
} else if ('-' == s[0] && '1' == s[1] && '\0' == s[2]) {
|
||||
/* A value of -1 means "use it if it would be advantageous */
|
||||
return RESULT_RUNTIME;
|
||||
} else {
|
||||
/* Any other value means "use the hooks, Luke!" */
|
||||
return RESULT_YES;
|
||||
}
|
||||
}
|
||||
|
||||
/* OMPI's init function */
|
||||
static void opal_memory_ptmalloc2_malloc_init_hook(void)
|
||||
{
|
||||
/* Yes, checking for an MPI MCA parameter here is an abstraction
|
||||
violation. Cope. Yes, even checking for *any* MCA parameter
|
||||
here (without going through the MCA param API) is an
|
||||
abstraction violation. Fricken' cope, will ya?
|
||||
(unfortunately, there's really no good way to do this other
|
||||
than this abstraction violation :-( ) */
|
||||
struct stat st;
|
||||
check_result_t lp = check("OMPI_MCA_mpi_leave_pinned");
|
||||
check_result_t lpp = check("OMPI_MCA_mpi_leave_pinned_pipeline");
|
||||
bool want_rcache = false, found_driver = false;
|
||||
|
||||
/* If /sys/class/infiniband exists, then the OpenFabrics
|
||||
drivers are loaded. So let's default to using our hooks so
|
||||
that we can utilize leave_pinned (yes, I know, further
|
||||
abstraction violations... :-( ). */
|
||||
if (0 == stat("/sys/class/infiniband", &st)) {
|
||||
found_driver = true;
|
||||
}
|
||||
|
||||
/* Simple combination of the results of these two environment
|
||||
variables (if both "yes" and "no" are specified, then be
|
||||
conservative and assume "yes):
|
||||
|
||||
lp / lpp yes no runtime not found
|
||||
yes yes yes yes yes
|
||||
no yes no no no
|
||||
runtime yes no runtime runtime
|
||||
not found yes no runtime runtime
|
||||
*/
|
||||
if (RESULT_YES == lp || RESULT_YES == lpp) {
|
||||
want_rcache = true;
|
||||
} else if (RESULT_NO == lp || RESULT_NO == lpp) {
|
||||
want_rcache = false;
|
||||
} else {
|
||||
want_rcache = found_driver;
|
||||
}
|
||||
|
||||
if (want_rcache) {
|
||||
const char str[] = "using ptmallo\n";
|
||||
write(1, str, sizeof(str));
|
||||
/* Initialize ptmalloc */
|
||||
ptmalloc_init();
|
||||
|
||||
/* Now set the hooks to point to our functions */
|
||||
__free_hook = opal_memory_ptmalloc2_free_hook;
|
||||
__malloc_hook = opal_memory_ptmalloc2_malloc_hook;
|
||||
__memalign_hook = opal_memory_ptmalloc2_memalign_hook;
|
||||
__realloc_hook = opal_memory_ptmalloc2_realloc_hook;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* OMPI change: This is the symbol to override to make the above
|
||||
function get fired during malloc initialization time. */
|
||||
void (*__malloc_initialize_hook) (void) =
|
||||
opal_memory_ptmalloc2_malloc_init_hook;
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* c-basic-offset: 2
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
*/
|
||||
|
||||
@ -20,6 +20,9 @@
|
||||
|
||||
/* $Id: $ */
|
||||
|
||||
/* OMPI change: Name-shift all the internal symbols */
|
||||
#include "opal/mca/memory/ptmalloc2/rename.h"
|
||||
|
||||
#include <stdio.h> /* needed for malloc_stats */
|
||||
|
||||
#include <malloc-machine.h>
|
||||
|
||||
@ -7,6 +7,8 @@
|
||||
|
||||
#include "opal/sys/atomic.h"
|
||||
#include "opal/memoryhooks/memory_internal.h"
|
||||
/* Name-shift all the internal symbols */
|
||||
#include "opal/mca/memory/ptmalloc2/rename.h"
|
||||
|
||||
/*
|
||||
* Not all systems have sbrk() declared, since it's technically not a
|
||||
@ -521,23 +523,24 @@ Void_t * __default_morecore (ptrdiff_t);
|
||||
Void_t *(*__morecore)(ptrdiff_t) = __default_morecore;
|
||||
|
||||
#else /* !_LIBC */
|
||||
#define public_cALLOc calloc
|
||||
#define public_fREe free
|
||||
#define public_cFREe cfree
|
||||
#define public_mALLOc malloc
|
||||
#define public_mEMALIGn memalign
|
||||
#define public_rEALLOc realloc
|
||||
#define public_vALLOc valloc
|
||||
#define public_pVALLOc pvalloc
|
||||
#define public_mALLINFo mallinfo
|
||||
#define public_mALLOPt mallopt
|
||||
#define public_mTRIm malloc_trim
|
||||
#define public_mSTATs malloc_stats
|
||||
#define public_mUSABLe malloc_usable_size
|
||||
#define public_iCALLOc independent_calloc
|
||||
#define public_iCOMALLOc independent_comalloc
|
||||
#define public_gET_STATe malloc_get_state
|
||||
#define public_sET_STATe malloc_set_state
|
||||
/* OMPI change: put "opal_memory_ptmalloc2_" in front of all of these */
|
||||
#define public_cALLOc opal_memory_ptmalloc2_calloc
|
||||
#define public_fREe opal_memory_ptmalloc2_free
|
||||
#define public_cFREe opal_memory_ptmalloc2_cfree
|
||||
#define public_mALLOc opal_memory_ptmalloc2_malloc
|
||||
#define public_mEMALIGn opal_memory_ptmalloc2_memalign
|
||||
#define public_rEALLOc opal_memory_ptmalloc2_realloc
|
||||
#define public_vALLOc opal_memory_ptmalloc2_valloc
|
||||
#define public_pVALLOc opal_memory_ptmalloc2_pvalloc
|
||||
#define public_mALLINFo opal_memory_ptmalloc2_mallinfo
|
||||
#define public_mALLOPt opal_memory_ptmalloc2_mallopt
|
||||
#define public_mTRIm opal_memory_ptmalloc2_malloc_trim
|
||||
#define public_mSTATs opal_memory_ptmalloc2_malloc_stats
|
||||
#define public_mUSABLe opal_memory_ptmalloc2_malloc_usable_size
|
||||
#define public_iCALLOc opal_memory_ptmalloc2_independent_calloc
|
||||
#define public_iCOMALLOc opal_memory_ptmalloc2_independent_comalloc
|
||||
#define public_gET_STATe opal_memory_ptmalloc2_malloc_get_state
|
||||
#define public_sET_STATe opal_memory_ptmalloc2_malloc_set_state
|
||||
#endif /* _LIBC */
|
||||
#endif /* USE_DL_PREFIX */
|
||||
|
||||
@ -2390,6 +2393,8 @@ static Void_t** iALLOc();
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* OMPI change: these aren't used */
|
||||
#if 0
|
||||
#if !(USE_STARTER & 2)
|
||||
# define free_hook_ini NULL
|
||||
/* Forward declarations. */
|
||||
@ -2405,7 +2410,11 @@ static Void_t* memalign_hook_ini __MALLOC_P ((size_t alignment, size_t sz,
|
||||
# define realloc_hook_ini NULL
|
||||
# define memalign_hook_ini memalign_starter
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* OMPI change: we don't want any of these -- we want to use the
|
||||
underlying allocator's symbols */
|
||||
#if 0
|
||||
void weak_variable (*__malloc_initialize_hook) __MALLOC_PMT ((void)) = NULL;
|
||||
void weak_variable (*__free_hook) __MALLOC_PMT ((__malloc_ptr_t __ptr,
|
||||
const __malloc_ptr_t))
|
||||
@ -2419,6 +2428,7 @@ __malloc_ptr_t weak_variable (*__memalign_hook)
|
||||
__MALLOC_PMT ((size_t __alignment, size_t __size, const __malloc_ptr_t))
|
||||
= memalign_hook_ini;
|
||||
void weak_variable (*__after_morecore_hook) __MALLOC_P ((void)) = NULL;
|
||||
#endif
|
||||
|
||||
|
||||
/* ------------------- Support for multiple arenas -------------------- */
|
||||
@ -2780,7 +2790,6 @@ static void do_check_malloc_state(mstate av)
|
||||
/* ----------------- Support for debugging hooks -------------------- */
|
||||
#include "hooks.c"
|
||||
|
||||
|
||||
/* ----------- Routines dealing with system allocation -------------- */
|
||||
|
||||
/*
|
||||
@ -3398,10 +3407,24 @@ public_mALLOc(size_t bytes)
|
||||
mstate ar_ptr;
|
||||
Void_t *victim;
|
||||
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
__malloc_ptr_t (*hook) __MALLOC_P ((size_t, __const __malloc_ptr_t)) =
|
||||
__malloc_hook;
|
||||
if (hook != NULL)
|
||||
return (*hook)(bytes, RETURN_ADDRESS (0));
|
||||
#endif
|
||||
|
||||
/* OMPI change: put in a flag so that we can know that this function
|
||||
was invoked. This flag is checked in the memory/ptmalloc2
|
||||
component init to ensure that this ptmalloc is actually being
|
||||
used. Used a simple "extern" here to get the flag symbol rather
|
||||
than putting it in a new .h file that would only contain a small
|
||||
number of symbols. */
|
||||
{
|
||||
extern bool opal_memory_ptmalloc2_malloc_invoked;
|
||||
opal_memory_ptmalloc2_malloc_invoked = true;
|
||||
}
|
||||
|
||||
arena_get(ar_ptr, bytes);
|
||||
if(!ar_ptr)
|
||||
@ -3442,12 +3465,26 @@ public_fREe(Void_t* mem)
|
||||
mstate ar_ptr;
|
||||
mchunkptr p; /* chunk corresponding to mem */
|
||||
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
void (*hook) __MALLOC_P ((__malloc_ptr_t, __const __malloc_ptr_t)) =
|
||||
__free_hook;
|
||||
if (hook != NULL) {
|
||||
(*hook)(mem, RETURN_ADDRESS (0));
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* OMPI change: put in a flag so that we can know that this function
|
||||
was invoked. This flag is checked in the memory/ptmalloc2
|
||||
component init to ensure that this ptmalloc is actually being
|
||||
used. Used a simple "extern" here to get the flag symbol rather
|
||||
than putting it in a new .h file that would only contain a small
|
||||
number of symbols. */
|
||||
{
|
||||
extern bool opal_memory_ptmalloc2_free_invoked;
|
||||
opal_memory_ptmalloc2_free_invoked = true;
|
||||
}
|
||||
|
||||
if (mem == 0) /* free(0) has no effect */
|
||||
return;
|
||||
@ -3491,11 +3528,25 @@ public_rEALLOc(Void_t* oldmem, size_t bytes)
|
||||
|
||||
Void_t* newp; /* chunk to return */
|
||||
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
__malloc_ptr_t (*hook) __MALLOC_P ((__malloc_ptr_t, size_t,
|
||||
__const __malloc_ptr_t)) =
|
||||
__realloc_hook;
|
||||
if (hook != NULL)
|
||||
return (*hook)(oldmem, bytes, RETURN_ADDRESS (0));
|
||||
#endif
|
||||
|
||||
/* OMPI change: put in a flag so that we can know that this function
|
||||
was invoked. This flag is checked in the memory/ptmalloc2
|
||||
component init to ensure that this ptmalloc is actually being
|
||||
used. Used a simple "extern" here to get the flag symbol rather
|
||||
than putting it in a new .h file that would only contain a small
|
||||
number of symbols. */
|
||||
{
|
||||
extern bool opal_memory_ptmalloc2_realloc_invoked;
|
||||
opal_memory_ptmalloc2_realloc_invoked = true;
|
||||
}
|
||||
|
||||
#if REALLOC_ZERO_BYTES_FREES
|
||||
if (bytes == 0 && oldmem != NULL) { public_fREe(oldmem); return 0; }
|
||||
@ -3567,11 +3618,25 @@ public_mEMALIGn(size_t alignment, size_t bytes)
|
||||
mstate ar_ptr;
|
||||
Void_t *p;
|
||||
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
__malloc_ptr_t (*hook) __MALLOC_PMT ((size_t, size_t,
|
||||
__const __malloc_ptr_t)) =
|
||||
__memalign_hook;
|
||||
if (hook != NULL)
|
||||
return (*hook)(alignment, bytes, RETURN_ADDRESS (0));
|
||||
#endif
|
||||
|
||||
/* OMPI change: put in a flag so that we can know that this function
|
||||
was invoked. This flag is checked in the memory/ptmalloc2
|
||||
component init to ensure that this ptmalloc is actually being
|
||||
used. Used a simple "extern" here to get the flag symbol rather
|
||||
than putting it in a new .h file that would only contain a small
|
||||
number of symbols. */
|
||||
{
|
||||
extern bool opal_memory_ptmalloc2_memalign_invoked;
|
||||
opal_memory_ptmalloc2_memalign_invoked = true;
|
||||
}
|
||||
|
||||
/* If need less alignment than we give anyway, just relay to malloc */
|
||||
if (alignment <= MALLOC_ALIGNMENT) return public_mALLOc(bytes);
|
||||
@ -3649,8 +3714,11 @@ public_cALLOc(size_t n, size_t elem_size)
|
||||
unsigned long clearsize;
|
||||
unsigned long nclears;
|
||||
INTERNAL_SIZE_T* d;
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
__malloc_ptr_t (*hook) __MALLOC_PMT ((size_t, __const __malloc_ptr_t)) =
|
||||
__malloc_hook;
|
||||
#endif
|
||||
|
||||
/* size_t is unsigned so the behavior on overflow is defined. */
|
||||
bytes = n * elem_size;
|
||||
@ -3663,6 +3731,8 @@ public_cALLOc(size_t n, size_t elem_size)
|
||||
}
|
||||
}
|
||||
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
if (hook != NULL) {
|
||||
sz = bytes;
|
||||
mem = (*hook)(sz, RETURN_ADDRESS (0));
|
||||
@ -3675,6 +3745,7 @@ public_cALLOc(size_t n, size_t elem_size)
|
||||
return mem;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
sz = bytes;
|
||||
|
||||
@ -5437,7 +5508,6 @@ int mALLOPt(param_number, value) int param_number; int value;
|
||||
|
||||
*/
|
||||
|
||||
|
||||
/* OMPI: Need to expose our own posix_memalign, or the wrong one will
|
||||
be used */
|
||||
# include <sys/param.h>
|
||||
@ -5447,9 +5517,13 @@ int
|
||||
posix_memalign (void **memptr, size_t alignment, size_t size)
|
||||
{
|
||||
void *mem;
|
||||
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
__malloc_ptr_t (*hook) __MALLOC_PMT ((size_t, size_t,
|
||||
__const __malloc_ptr_t)) =
|
||||
__memalign_hook;
|
||||
#endif
|
||||
|
||||
/* Test whether the SIZE argument is valid. It must be a power of
|
||||
two multiple of sizeof (void *). */
|
||||
@ -5458,12 +5532,17 @@ posix_memalign (void **memptr, size_t alignment, size_t size)
|
||||
|| alignment == 0)
|
||||
return EINVAL;
|
||||
|
||||
/* OMPI change: the hook is us -- don't call the hook */
|
||||
#if 0
|
||||
/* Call the hook here, so that caller is posix_memalign's caller
|
||||
and not posix_memalign itself. */
|
||||
if (hook != NULL)
|
||||
mem = (*hook)(alignment, size, RETURN_ADDRESS (0));
|
||||
else
|
||||
mem = public_mEMALIGn (alignment, size);
|
||||
#else
|
||||
mem = public_mEMALIGn (alignment, size);
|
||||
#endif
|
||||
|
||||
if (mem != NULL) {
|
||||
*memptr = mem;
|
||||
@ -5472,6 +5551,7 @@ posix_memalign (void **memptr, size_t alignment, size_t size)
|
||||
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
#ifdef _LIBC
|
||||
weak_alias (__posix_memalign, posix_memalign)
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -16,13 +17,21 @@
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/* The goal of this component is to wholly replace the underlying
|
||||
allocator with our internal ptmalloc2 allocator.
|
||||
|
||||
See the file README-open-mpi.txt for details of how it works. */
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
/* Include <link.h> for _DYNAMIC */
|
||||
#include <link.h>
|
||||
|
||||
#include "opal/constants.h"
|
||||
#include "opal/mca/memory/memory.h"
|
||||
#include "opal/memoryhooks/memory.h"
|
||||
|
||||
static int opal_memory_ptmalloc2_open(void);
|
||||
static int ptmalloc2_open(void);
|
||||
|
||||
const opal_memory_base_component_2_0_0_t mca_memory_ptmalloc2_component = {
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
@ -37,7 +46,7 @@ const opal_memory_base_component_2_0_0_t mca_memory_ptmalloc2_component = {
|
||||
OPAL_RELEASE_VERSION,
|
||||
|
||||
/* Component open and close functions */
|
||||
opal_memory_ptmalloc2_open,
|
||||
ptmalloc2_open,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
@ -46,16 +55,60 @@ const opal_memory_base_component_2_0_0_t mca_memory_ptmalloc2_component = {
|
||||
},
|
||||
};
|
||||
|
||||
/* Public symbols */
|
||||
bool opal_memory_ptmalloc2_free_invoked = false;
|
||||
bool opal_memory_ptmalloc2_malloc_invoked = false;
|
||||
bool opal_memory_ptmalloc2_realloc_invoked = false;
|
||||
bool opal_memory_ptmalloc2_memalign_invoked = false;
|
||||
|
||||
static int
|
||||
opal_memory_ptmalloc2_open(void)
|
||||
static int ptmalloc2_open(void)
|
||||
{
|
||||
/* we always provide munmap support as part of libopen-pal.la.
|
||||
Will also provide malloc/free support if user linked us in. In
|
||||
that case, we've already likely called set_support, so don't
|
||||
crush what has already been done */
|
||||
if (0 == opal_mem_hooks_support_level()) {
|
||||
opal_mem_hooks_set_support(OPAL_MEMORY_MUNMAP_SUPPORT);
|
||||
/* We always provide munmap support as part of libopen-pal.la. */
|
||||
int val = OPAL_MEMORY_MUNMAP_SUPPORT;
|
||||
|
||||
/* We can't catch munmap if we're a static library */
|
||||
if (NULL == &_DYNAMIC) {
|
||||
val = 0;
|
||||
}
|
||||
|
||||
/* We will also provide malloc/free support if we've been
|
||||
activated. We don't rely on the __malloc_initialize_hook()
|
||||
previously being called because it's possible that our hook was
|
||||
called, but then someone else reset the hooks to point to
|
||||
something else (i.e., before MPI_INIT). So explicitly test
|
||||
here if our hooks are still in place. If they are, then enable
|
||||
FREE|CUNK_SUPPORT. If not, then don't enable that support --
|
||||
just leave it at MUNMAP_SUPPORT.
|
||||
|
||||
(Look in hooks.c for the __malloc_initialize_hook setup) */
|
||||
|
||||
/* Do a simple set of tests to see if our hooks are still the ones
|
||||
installed. Explicitly reset the flags indicating that our
|
||||
functions were invoked */
|
||||
opal_memory_ptmalloc2_malloc_invoked = false;
|
||||
opal_memory_ptmalloc2_realloc_invoked = false;
|
||||
opal_memory_ptmalloc2_memalign_invoked = false;
|
||||
opal_memory_ptmalloc2_free_invoked = false;
|
||||
|
||||
void *p = malloc(1024 * 1024 * 4);
|
||||
if (NULL == p) {
|
||||
return OPAL_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
realloc(p, 1024 * 1024 * 4 + 32);
|
||||
free(p);
|
||||
memalign(1, 1024 * 1024);
|
||||
free(p);
|
||||
|
||||
if (opal_memory_ptmalloc2_malloc_invoked &&
|
||||
opal_memory_ptmalloc2_realloc_invoked &&
|
||||
opal_memory_ptmalloc2_memalign_invoked &&
|
||||
opal_memory_ptmalloc2_free_invoked) {
|
||||
/* Happiness; our functions were invoked */
|
||||
val |= OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_CHUNK_SUPPORT;
|
||||
}
|
||||
|
||||
/* Set the support level */
|
||||
opal_mem_hooks_set_support(val);
|
||||
|
||||
return OPAL_SUCCESS;
|
||||
}
|
||||
|
||||
52
opal/mca/memory/ptmalloc2/rename.h
Обычный файл
52
opal/mca/memory/ptmalloc2/rename.h
Обычный файл
@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/* Name-shift all the internal ptmalloc22 symbols to guarantee to not
|
||||
conflict / confuse / override the internal glibc symbols. */
|
||||
|
||||
#define __default_morecore opal_memory_ptmalloc2_default_morecore
|
||||
|
||||
#define _int_malloc opal_memory_ptmalloc2_int_malloc
|
||||
#define _int_free opal_memory_ptmalloc2_int_free
|
||||
#define _int_realloc opal_memory_ptmalloc2_int_realloc
|
||||
#define _int_memalign opal_memory_ptmalloc2_int_memalign
|
||||
#define _int_valloc opal_memory_ptmalloc2_int_valloc
|
||||
#define _int_pvalloc opal_memory_ptmalloc2_int_pvalloc
|
||||
#define _int_icalloc opal_memory_ptmalloc2_int_icalloc
|
||||
#define _int_icomalloc opal_memory_ptmalloc2_int_icomalloc
|
||||
|
||||
#define mTRIm opal_memory_ptmalloc2_mTRIm
|
||||
#define mUSABLe opal_memory_ptmalloc2_mUSABLe
|
||||
#define mALLOPt opal_memory_ptmalloc2_mALLOPt
|
||||
|
||||
#define mem2mem_check opal_memory_ptmalloc2_mem2mem_check
|
||||
#define top_check opal_memory_ptmalloc2_top_check
|
||||
#define munmap_chunk opal_memory_ptmalloc2_munmap_chunk
|
||||
#define mremap_chunk opal_memory_ptmalloc2_mremap_chunk
|
||||
|
||||
#define malloc_check opal_memory_ptmalloc2_malloc_check
|
||||
#define free_check opal_memory_ptmalloc2_free_check
|
||||
#define realloc_check opal_memory_ptmalloc2_realloc_check
|
||||
#define memalign_check opal_memory_ptmalloc2_memalign_check
|
||||
|
||||
#define malloc_starter opal_memory_ptmalloc2_malloc_starter
|
||||
#define memalign_starter opal_memory_ptmalloc2_memalign_starter
|
||||
#define free_starter opal_memory_ptmalloc2_free_starter
|
||||
|
||||
#define malloc_atfork opal_memory_ptmalloc2_malloc_atfork
|
||||
#define free_atfork opal_memory_ptmalloc2_free_atfork
|
||||
|
||||
#define _int_get_arena opal_memory_ptmalloc2_int_get_arena
|
||||
#define _int_get_arena_info opal_memory_ptmalloc2_int_get_arena_info
|
||||
#define _int_get_global_info opal_memory_ptmalloc2_int_get_global_info
|
||||
#define _int_new_arena opal_memory_ptmalloc2_int_new_arena
|
||||
#define __malloc_check_init opal_memory_ptmalloc2_malloc_check_init
|
||||
#define malloc_stats opal_memory_ptmalloc2_malloc_stats
|
||||
|
||||
#define posix_memalign opal_memory_ptmalloc2_posix_memalign
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user