From 79bf8843d20bf5cb296a8ad8398bbec77718ed80 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Tue, 29 Nov 2005 04:46:14 +0000 Subject: [PATCH] * update memory hooks interface to allow for callbacks on both allocations and dealllocations, per request from Galen and Tim This commit was SVN r8303. --- ompi/mca/mpool/base/mpool_base_close.c | 5 +- ompi/mca/mpool/base/mpool_base_lookup.c | 4 +- opal/mca/memory/darwin/configure.m4 | 4 +- .../memory/darwin/memory_darwin_component.c | 84 +++++- .../memory/malloc_hooks/memory_malloc_hooks.c | 102 ++++++- .../memory_malloc_interpose.c | 94 +++++- opal/mca/memory/ptmalloc2/Makefile.am | 2 - opal/mca/memory/ptmalloc2/arena.c | 2 +- opal/mca/memory/ptmalloc2/malloc.c | 2 +- opal/mca/memory/ptmalloc2/ptmalloc2_munmap.c | 4 +- opal/memoryhooks/memory.c | 267 +++++++++++++----- opal/memoryhooks/memory.h | 33 +-- opal/memoryhooks/memory_internal.h | 19 +- opal/runtime/opal_finalize.c | 2 +- opal/runtime/opal_init.c | 2 +- orte/tools/orteprobe/orteprobe.c | 2 +- test/memory/opal_memory_basic.c | 29 +- test/memory/opal_memory_cxx.cc | 7 +- test/memory/opal_memory_speed.c | 11 +- 19 files changed, 528 insertions(+), 147 deletions(-) diff --git a/ompi/mca/mpool/base/mpool_base_close.c b/ompi/mca/mpool/base/mpool_base_close.c index 69cfc0ecc0..2b260cbf50 100644 --- a/ompi/mca/mpool/base/mpool_base_close.c +++ b/ompi/mca/mpool/base/mpool_base_close.c @@ -59,8 +59,9 @@ int mca_mpool_base_close(void) &mca_mpool_base_components, NULL); /* deregister memory free callback */ - if(mca_mpool_base_use_mem_hooks && opal_mem_free_is_supported()) { - opal_mem_free_unregister_handler(mca_mpool_base_mem_cb); + if(mca_mpool_base_use_mem_hooks && + 0 != (OPAL_MEMORY_FREE_SUPPORT & opal_mem_hooks_support_level())) { + opal_mem_hooks_unregister_release(mca_mpool_base_mem_cb); } /* All done */ diff --git a/ompi/mca/mpool/base/mpool_base_lookup.c b/ompi/mca/mpool/base/mpool_base_lookup.c index 886399e027..2b716a362e 100644 --- a/ompi/mca/mpool/base/mpool_base_lookup.c +++ b/ompi/mca/mpool/base/mpool_base_lookup.c @@ -84,8 +84,8 @@ mca_mpool_base_module_t* mca_mpool_base_module_create( /* on the very first creation of a module we init the memory callback*/ if(mca_mpool_base_use_mem_hooks && opal_list_get_size(&mca_mpool_base_modules) == 1 && - opal_mem_free_is_supported()) { - opal_mem_free_register_handler(mca_mpool_base_mem_cb, NULL); + 0 != (OPAL_MEMORY_FREE_SUPPORT & opal_mem_hooks_support_level())) { + opal_mem_hooks_register_release(mca_mpool_base_mem_cb, NULL); } return module; } diff --git a/opal/mca/memory/darwin/configure.m4 b/opal/mca/memory/darwin/configure.m4 index a5ff2a332b..a4bb6069c5 100644 --- a/opal/mca/memory/darwin/configure.m4 +++ b/opal/mca/memory/darwin/configure.m4 @@ -65,6 +65,8 @@ AC_DEFUN([MCA_memory_darwin_CONFIG],[ [AC_MSG_ERROR([Darwin memory management requested but not available. Aborting.])]) AS_IF([test "$memory_darwin_happy" = "yes"], - [memory_darwin_WRAPPER_EXTRA_LDFLAGS="-Wl,-u,_munmap -Wl,-multiply_defined,suppress" + [# Yes, we really do want to screw with LDFLAGS here... + LDFLAGS="$LDFLAGS -Wl,-multiply_defined,suppress" + memory_darwin_WRAPPER_EXTRA_LDFLAGS="-Wl,-u,_munmap -Wl,-multiply_defined,suppress" $1], [$2]) ]) diff --git a/opal/mca/memory/darwin/memory_darwin_component.c b/opal/mca/memory/darwin/memory_darwin_component.c index 43e88f8283..14947d419b 100644 --- a/opal/mca/memory/darwin/memory_darwin_component.c +++ b/opal/mca/memory/darwin/memory_darwin_component.c @@ -28,6 +28,11 @@ #include "opal/memoryhooks/memory_internal.h" static int opal_memory_darwin_open(void); + +static void* opal_memory_darwin_malloc(struct _malloc_zone_t *zone, size_t size); +static void* opal_memory_darwin_calloc(struct _malloc_zone_t *zone, size_t num_items, + size_t size); +static void* opal_memory_darwin_valloc(struct _malloc_zone_t *zone, size_t size); static void opal_memory_darwin_free(struct _malloc_zone_t *zone, void *ptr); static void* opal_memory_darwin_realloc(struct _malloc_zone_t *zone, void *ptr, size_t size); @@ -58,8 +63,10 @@ const opal_memory_base_component_1_0_0_t mca_memory_darwin_component = { }, }; - -static void (*next_free)(struct _malloc_zone_t *zone, void *ptr); +static void* (*next_malloc)(struct _malloc_zone_t *zone, size_t size); +static void* (*next_calloc)(struct _malloc_zone_t *zone, size_t num_items, size_t size); +static void* (*next_valloc)(struct _malloc_zone_t *zone, size_t size); +static void (*next_free)(struct _malloc_zone_t *zone, void *ptr); static void* (*next_realloc)(struct _malloc_zone_t *zone, void *ptr, size_t size); @@ -73,22 +80,55 @@ opal_memory_darwin_open(void) /* save the pointers first, so that we can call them as soon as we replace the hooks below (think threads) */ + next_malloc = default_zone->malloc; + next_calloc = default_zone->calloc; + next_valloc = default_zone->valloc; next_free = default_zone->free; next_realloc = default_zone->realloc; + default_zone->malloc = opal_memory_darwin_malloc; + default_zone->calloc = opal_memory_darwin_calloc; + default_zone->valloc = opal_memory_darwin_valloc; default_zone->free = opal_memory_darwin_free; default_zone->realloc = opal_memory_darwin_realloc; - opal_mem_free_set_free_support(1); + opal_mem_hooks_set_support(OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_MALLOC_SUPPORT); return OPAL_SUCCESS; } +static void * +opal_memory_darwin_malloc(struct _malloc_zone_t *zone, size_t size) +{ + void *tmp = next_malloc(zone, size); + opal_mem_hooks_alloc_hook(tmp, malloc_size(tmp)); + return tmp; +} + + +static void* +opal_memory_darwin_calloc(struct _malloc_zone_t *zone, size_t num_items, + size_t size) +{ + void *tmp = next_calloc(zone, num_items, size); + opal_mem_hooks_alloc_hook(tmp, malloc_size(tmp)); + return tmp; +} + + +static void* +opal_memory_darwin_valloc(struct _malloc_zone_t *zone, size_t size) +{ + void *tmp = next_valloc(zone, size); + opal_mem_hooks_alloc_hook(tmp, malloc_size(tmp)); + return tmp; +} + static void opal_memory_darwin_free(struct _malloc_zone_t *zone, void *ptr) { - opal_mem_free_release_hook(ptr, malloc_size(ptr)); + opal_mem_hooks_release_hook(ptr, malloc_size(ptr)); next_free(zone, ptr); } @@ -97,24 +137,52 @@ static void * opal_memory_darwin_realloc(struct _malloc_zone_t *zone, void *ptr, size_t size) { - opal_mem_free_release_hook(ptr, malloc_size(ptr)); - return next_realloc(zone, ptr, size); + char *tmp; + + opal_mem_hooks_release_hook(ptr, malloc_size(ptr)); + tmp = next_realloc(zone, ptr, size); + opal_mem_hooks_alloc_hook(tmp, malloc_size(tmp)); + + return tmp; } -/* only need to catch munmap for user code, and we should be at the +/* only need to catch mmap / munmap for user code, and we should be at the far right of the library stack, so this should work. Darwin 7 and later include dlsym as part of libSystem, so no need to do anything special for it. Not sure what would happen if you tried to statically link your application (as in -Bstatic, not libmpi.a), but since Apple doesn't support that, neither do we. */ +void* +mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) +{ + static void* (*realmmap)(void *, size_t, int, int, int, off_t); + void *tmp; + + if (NULL == realmmap) { + union { + void* (*mmap_fp)(void *, size_t, int, int, int, off_t); + void *mmap_p; + } tmp; + + tmp.mmap_p = dlsym(RTLD_NEXT, "mmap"); + realmmap = tmp.mmap_fp; + } + + tmp = realmmap(addr, len, prot, flags, fd, offset); + opal_mem_hooks_alloc_hook(tmp, len); + + return tmp; +} + + int munmap(void* addr, size_t len) { static int (*realmunmap)(void*, size_t); /* dispatch about the pending release */ - opal_mem_free_release_hook(addr, len); + opal_mem_hooks_release_hook(addr, len); if (NULL == realmunmap) { union { diff --git a/opal/mca/memory/malloc_hooks/memory_malloc_hooks.c b/opal/mca/memory/malloc_hooks/memory_malloc_hooks.c index 9b000a6431..0489ec069c 100644 --- a/opal/mca/memory/malloc_hooks/memory_malloc_hooks.c +++ b/opal/mca/memory/malloc_hooks/memory_malloc_hooks.c @@ -22,21 +22,25 @@ #define __USE_GNU #include #include +#include +#include #include "opal/memoryhooks/memory_internal.h" /* Prototypes for our hooks. */ void opal_memory_malloc_hooks_init(void); -static void opal_mem_free_free_hook (void*, const void *); -static void* opal_mem_free_realloc_hook (void*, size_t, const void *); - + +static void local_free_hook(void*, const void*); +static void* local_malloc_hook(size_t, const void*); +static void* local_realloc_hook(void*, size_t, const void*); + /* Override initializing hook from the C library. */ void (*__malloc_initialize_hook) (void) = opal_memory_malloc_hooks_init; - /* local variable - next in stack of free hooks */ static void (*old_free_hook)(void*, const void*); static void* (*old_realloc_hook)(void*, size_t, const void*); +static void* (*old_malloc_hook)(size_t, const void*); static int initialized = 0; @@ -52,19 +56,26 @@ opal_memory_malloc_hooks_init(void) } initialized = 1; + old_free_hook = __free_hook; + old_malloc_hook = __malloc_hook; old_realloc_hook = __realloc_hook; - __free_hook = opal_mem_free_free_hook; - __realloc_hook = opal_mem_free_realloc_hook; - opal_mem_free_set_free_support(1); + + __free_hook = local_free_hook; + __malloc_hook = local_malloc_hook; + __realloc_hook = local_realloc_hook; + + opal_mem_hooks_set_support(OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_MALLOC_SUPPORT); + assert(__malloc_hook == local_malloc_hook); } static void -opal_mem_free_free_hook (void *ptr, const void *caller) +local_free_hook(void *ptr, const void *caller) { + if (__malloc_hook != local_malloc_hook) abort(); /* dispatch about the pending free */ - opal_mem_free_release_hook(ptr, malloc_usable_size(ptr)); + opal_mem_hooks_release_hook(ptr, malloc_usable_size(ptr)); __free_hook = old_free_hook; @@ -73,29 +84,91 @@ opal_mem_free_free_hook (void *ptr, const void *caller) /* save the hooks again and restore our hook again */ old_free_hook = __free_hook; - __free_hook = opal_mem_free_free_hook; + __free_hook = local_free_hook; } +static void* +local_malloc_hook(size_t size, const void *caller) +{ + void *ret; + + __malloc_hook = old_malloc_hook; + + /* call the next chain down */ + ret = malloc(size); + + /* save the hooks again and restory our hack again */ + old_malloc_hook = __malloc_hook; + __malloc_hook = local_malloc_hook; + + opal_mem_hooks_alloc_hook(ret, malloc_usable_size(ret)); + + assert(__malloc_hook == local_malloc_hook); + + return ret; +} + + + /* for better or worse, we must assume that the buffer being passed to realloc is not going to be expandable and therefore is going to be free()ed. */ static void* -opal_mem_free_realloc_hook (void *ptr, size_t size, const void *caller) +local_realloc_hook(void *ptr, size_t size, const void *caller) { void *ret; + assert(__malloc_hook == local_malloc_hook); /* dispatch about the pending free */ - opal_mem_free_release_hook(ptr, malloc_usable_size(ptr)); + opal_mem_hooks_release_hook(ptr, malloc_usable_size(ptr)); + /* realloc can call malloc (but not free). Doing so with the + * memory hooks causes us some interesting problems and causes + * the malloc_hook to be left as NULL. Pop the stack now so + * that we don't see the memory registration twice. + */ __realloc_hook = old_realloc_hook; + __malloc_hook = old_malloc_hook; /* call the next chain down */ ret = realloc(ptr, size); /* save the hooks again and restore our hook again */ old_realloc_hook = __realloc_hook; - __realloc_hook = opal_mem_free_realloc_hook; + old_malloc_hook = __malloc_hook; + __realloc_hook = local_realloc_hook; + __malloc_hook = local_malloc_hook; + + opal_mem_hooks_alloc_hook(ret, malloc_usable_size(ret)); + assert(__malloc_hook == local_malloc_hook); + + return ret; +} + + +/* mmap is a weak symbol on any platform that I know of that + supports malloc hooks, so we can just intercept it like this... */ +void* +mmap(void* addr, size_t len, int prot, int flags, int fd, off_t offset) +{ + void *ret; + + static void* (*realmmap)(void*, size_t, int, int, int, off_t); + + if (NULL == realmmap) { + union { + void* (*mmap_fp)(void*, size_t, int, int, int, off_t); + void *mmap_p; + } tmp; + + tmp.mmap_p = dlsym(RTLD_NEXT, "mmap"); + realmmap = tmp.mmap_fp; + } + + ret = realmmap(addr, len, prot, flags, fd, offset); + + opal_mem_hooks_alloc_hook(ret, len); return ret; } @@ -108,7 +181,8 @@ munmap(void* addr, size_t len) { static int (*realmunmap)(void*, size_t); /* dispatch about the pending release */ - opal_mem_free_release_hook(addr, len); + opal_mem_hooks_release_hook(addr, len); + assert(__malloc_hook == local_malloc_hook); if (NULL == realmunmap) { union { diff --git a/opal/mca/memory/malloc_interpose/memory_malloc_interpose.c b/opal/mca/memory/malloc_interpose/memory_malloc_interpose.c index 3417d1f8fe..848dfe8d21 100644 --- a/opal/mca/memory/malloc_interpose/memory_malloc_interpose.c +++ b/opal/mca/memory/malloc_interpose/memory_malloc_interpose.c @@ -68,6 +68,30 @@ const opal_memory_base_component_1_0_0_t mca_memory_malloc_interpose_component = } \ } while (0); +#define FIND_REALCALLOC() \ + do { \ + if (NULL == realcalloc) { \ + union { \ + void* (*calloc_fp)(size_t, size_t); \ + void* calloc_p; \ + } tmp; \ + tmp.calloc_p = dlsym(RTLD_NEXT, "calloc"); \ + realcalloc = tmp.calloc_fp; \ + } \ + } while (0); + +#define FIND_REALMALLOC() \ + do { \ + if (NULL == realmalloc) { \ + union { \ + void* (*malloc_fp)(size_t); \ + void* malloc_p; \ + } tmp; \ + tmp.malloc_p = dlsym(RTLD_NEXT, "malloc"); \ + realmalloc = tmp.malloc_fp; \ + } \ + } while (0); + #define FIND_REALREALLOC() \ do { \ if (NULL == realrealloc) { \ @@ -80,6 +104,18 @@ const opal_memory_base_component_1_0_0_t mca_memory_malloc_interpose_component = } \ } while (0); +#define FIND_REALMMAP() \ + do { \ + if (NULL == realmmap) { \ + union { \ + void* (*mmap_fp)(void*, size_t, int, int, int, off_t); \ + void *mmap_p; \ + } tmp; \ + tmp.mmap_p = dlsym(RTLD_NEXT, "mmap"); \ + realmmap = tmp.mmap_fp; \ + } \ + } while (0); + #define FIND_REALMUNMAP() \ do { \ if (NULL == realmunmap) { \ @@ -93,19 +129,23 @@ const opal_memory_base_component_1_0_0_t mca_memory_malloc_interpose_component = } while (0); static void (*realfree)(void*); +static void* (*realcalloc)(size_t, size_t); +static void* (*realmalloc)(size_t); static void* (*realrealloc)(void*, size_t); +static void* (*realmmap)(void*, size_t, int, int, int, off_t); static int (*realmunmap)(void*, size_t); static int opal_memory_malloc_interpose_open(void) { - opal_mem_free_set_free_support(1); + opal_mem_hooks_set_support(OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_MALLOC_SUPPORT); FIND_REALFREE(); FIND_REALREALLOC(); FIND_REALMUNMAP(); - if (NULL == realfree || NULL == realrealloc || NULL == realmunmap) { + if (NULL == realfree || NULL == realcalloc || NULL == realmalloc || + NULL == realrealloc || NULL == realmmap || NULL == realmunmap) { /* this shoudl really never happen */ fprintf(stderr, "Could not find real memory functions. Aborting in dispair\n"); @@ -122,19 +162,57 @@ free(void *ptr) FIND_REALFREE(); /* dispatch about the pending release */ - opal_mem_free_release_hook(ptr, malloc_usable_size(ptr)); + opal_mem_hooks_release_hook(ptr, malloc_usable_size(ptr)); realfree(ptr); } +void* +calloc(size_t nmemb, size_t size) +{ + void *ret; + + FIND_REALCALLOC(); + ret = realcalloc(nmemb, size); + opal_mem_hooks_alloc_hook(ret, malloc_usable_size(ret)); + return ret; +} + + +void* +malloc(size_t size) +{ + void *ret; + + FIND_REALMALLOC(); + ret = realmalloc(size); + opal_mem_hooks_alloc_hook(ret, malloc_usable_size(ret)); + return ret; +} + + void* realloc(void *ptr, size_t size) { - FIND_REALREALLOC(); + void *ret; - /* dispatch about the pending release */ - opal_mem_free_release_hook(ptr, malloc_usable_size(ptr)); - return realrealloc(ptr, size); + FIND_REALREALLOC(); + opal_mem_hooks_release_hook(ptr, malloc_usable_size(ptr)); + ret = realrealloc(ptr, size); + opal_mem_hooks_alloc_hook(ret, malloc_usable_size(ret)); + return ret; +} + + +void* +mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) +{ + void *ret; + + FIND_REALMMAP(); + ret = realmmap(start, length, prot, flags, fd, offset); + opal_mem_hooks_alloc_hook(ret, length); + return ret; } @@ -144,6 +222,6 @@ munmap(void *start, size_t length) FIND_REALMUNMAP(); /* dispatch about the pending release */ - opal_mem_free_release_hook(start, length); + opal_mem_hooks_release_hook(start, length); return realmunmap(start, length); } diff --git a/opal/mca/memory/ptmalloc2/Makefile.am b/opal/mca/memory/ptmalloc2/Makefile.am index aef9c7abc4..124d0210dd 100644 --- a/opal/mca/memory/ptmalloc2/Makefile.am +++ b/opal/mca/memory/ptmalloc2/Makefile.am @@ -45,8 +45,6 @@ libmca_memory_ptmalloc2_la_SOURCES = \ malloc.c \ malloc-stats.c \ malloc.h -libmca_memory_ptmalloc2_la_LIBADD = \ - $(top_ompi_builddir)/opal/libopal.la # these are included directly and shouldn't be built solo EXTRA_libmca_memory_ptmalloc2_la_SOURCES = \ diff --git a/opal/mca/memory/ptmalloc2/arena.c b/opal/mca/memory/ptmalloc2/arena.c index 2a3e871be5..ea478904b1 100644 --- a/opal/mca/memory/ptmalloc2/arena.c +++ b/opal/mca/memory/ptmalloc2/arena.c @@ -468,7 +468,7 @@ ptmalloc_init __MALLOC_P((void)) /********************** BEGIN OMPI CHANGES *****************************/ /* don't use __hook for this, as someone might want to use those features */ - opal_mem_free_set_free_support(1); + opal_mem_hooks_set_support(OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_CHUNK_SUPPORT); /********************* BEGIN OMPI CHANGES ******************************/ __malloc_initialized = 1; diff --git a/opal/mca/memory/ptmalloc2/malloc.c b/opal/mca/memory/ptmalloc2/malloc.c index 78c1d5f6c9..ba08b53a85 100644 --- a/opal/mca/memory/ptmalloc2/malloc.c +++ b/opal/mca/memory/ptmalloc2/malloc.c @@ -20,7 +20,7 @@ opal_mem_free_ptmalloc2_sbrk(int inc) { if (inc < 0) { long oldp = (long) sbrk(0); - opal_mem_free_release_hook((void*) (oldp + inc), -inc); + opal_mem_hooks_release_hook((void*) (oldp + inc), -inc); } return sbrk(inc); diff --git a/opal/mca/memory/ptmalloc2/ptmalloc2_munmap.c b/opal/mca/memory/ptmalloc2/ptmalloc2_munmap.c index 5d9079a652..370f83c9e7 100644 --- a/opal/mca/memory/ptmalloc2/ptmalloc2_munmap.c +++ b/opal/mca/memory/ptmalloc2/ptmalloc2_munmap.c @@ -33,7 +33,7 @@ int munmap(void* addr, size_t len) { /* dispatch about the pending release */ - opal_mem_free_release_hook(addr, len); + opal_mem_hooks_release_hook(addr, len); if (NULL == realmunmap) { realmunmap = (int (*)(void*, size_t)) dlsym(RTLD_NEXT, "munmap"); @@ -47,7 +47,7 @@ munmap(void* addr, size_t len) int opal_mem_free_ptmalloc2_munmap(void *start, size_t length) { - opal_mem_free_release_hook(start, length); + opal_mem_hooks_release_hook(start, length); if (NULL == realmunmap) { realmunmap = (int (*)(void*, size_t)) dlsym(RTLD_NEXT, "munmap"); diff --git a/opal/memoryhooks/memory.c b/opal/memoryhooks/memory.c index edc2eeb7d3..9b8b12ac07 100644 --- a/opal/memoryhooks/memory.c +++ b/opal/memoryhooks/memory.c @@ -21,12 +21,12 @@ #include #include +#include "opal/include/constants.h" #include "opal/util/output.h" #include "opal/memoryhooks/memory.h" #include "opal/memoryhooks/memory_internal.h" #include "opal/class/opal_list.h" #include "opal/class/opal_object.h" -#include "ompi/include/constants.h" /* @@ -34,7 +34,7 @@ */ struct callback_list_item_t { opal_list_item_t super; - opal_mem_free_unpin_fn_t *cbfunc; + opal_mem_hooks_callback_fn_t *cbfunc; void *cbdata; }; typedef struct callback_list_item_t callback_list_item_t; @@ -43,68 +43,84 @@ static OBJ_CLASS_INSTANCE(callback_list_item_t, opal_list_item_t, NULL, NULL); /* * local data */ -static opal_list_t callback_list; -static opal_atomic_lock_t callback_lock; -static int have_free_support = false; -static int run_callbacks = false; -static int have_been_called = 0; +static int hooks_support = 0; + +static opal_list_t alloc_cb_list; +static opal_atomic_lock_t alloc_lock; +static int alloc_run_callbacks; + +static opal_list_t release_cb_list; +static opal_atomic_lock_t release_lock; +static int release_run_callbacks; int -opal_mem_free_init(void) +opal_mem_hooks_init(void) { - OBJ_CONSTRUCT(&callback_list, opal_list_t); - opal_atomic_init(&callback_lock, OPAL_ATOMIC_UNLOCKED); + OBJ_CONSTRUCT(&alloc_cb_list, opal_list_t); + OBJ_CONSTRUCT(&release_cb_list, opal_list_t); + + opal_atomic_init(&alloc_lock, OPAL_ATOMIC_UNLOCKED); + opal_atomic_init(&release_lock, OPAL_ATOMIC_UNLOCKED); /* delay running callbacks until there is something in the registration */ - run_callbacks = false; + alloc_run_callbacks = false; + release_run_callbacks = false; opal_atomic_mb(); - return OMPI_SUCCESS; + return OPAL_SUCCESS; } int -opal_mem_free_finalize(void) +opal_mem_hooks_finalize(void) { opal_list_item_t *item; - - run_callbacks = false; + + /* don't try to run callbacks any more */ + alloc_run_callbacks = false; + release_run_callbacks = false; opal_atomic_mb(); /* aquire the lock, just to make sure no one is currently twiddling with the list. We know this won't last long, since no new calls will come in after we set run_callbacks to false */ - opal_atomic_lock(&callback_lock); + opal_atomic_lock(&alloc_lock); + opal_atomic_lock(&release_lock); - while (NULL != (item = opal_list_remove_first(&callback_list))) { + /* clean out the lists */ + while (NULL != (item = opal_list_remove_first(&alloc_cb_list))) { OBJ_RELEASE(item); } - OBJ_DESTRUCT(&callback_list); + OBJ_DESTRUCT(&alloc_cb_list); - opal_atomic_unlock(&callback_lock); + while (NULL != (item = opal_list_remove_first(&release_cb_list))) { + OBJ_RELEASE(item); + } + OBJ_DESTRUCT(&release_cb_list); - return OMPI_SUCCESS; + opal_atomic_unlock(&alloc_lock); + opal_atomic_unlock(&release_lock); + + return OPAL_SUCCESS; } /* called from memory manager / memory-manager specific hooks */ void -opal_mem_free_set_free_support(int support) +opal_mem_hooks_set_support(int support) { - have_free_support = support; + hooks_support = support; } /* called from the memory manager / memory-manager specific hooks */ void -opal_mem_free_release_hook(void *buf, size_t length) +opal_mem_hooks_alloc_hook(void *buf, size_t length) { opal_list_item_t *item; - have_been_called = 1; - - if (!run_callbacks) return; + if (!alloc_run_callbacks) return; /* * This is not really thread safe - but we can't hold the lock @@ -116,62 +132,96 @@ opal_mem_free_release_hook(void *buf, size_t length) * the initial callback to dispatch this */ - opal_atomic_lock(&callback_lock); - item = opal_list_get_first(&callback_list); - while(item != opal_list_get_end(&callback_list)) { + opal_atomic_lock(&alloc_lock); + item = opal_list_get_first(&alloc_cb_list); + while(item != opal_list_get_end(&alloc_cb_list)) { opal_list_item_t* next = opal_list_get_next(item); callback_list_item_t cbitem = *(callback_list_item_t*) item; item = next; - opal_atomic_unlock(&callback_lock); + opal_atomic_unlock(&alloc_lock); cbitem.cbfunc(buf, length, cbitem.cbdata); - opal_atomic_lock(&callback_lock); + opal_atomic_lock(&alloc_lock); } - opal_atomic_unlock(&callback_lock); + opal_atomic_unlock(&alloc_lock); } -bool -opal_mem_free_is_supported(void) +/* called from the memory manager / memory-manager specific hooks */ +void +opal_mem_hooks_release_hook(void *buf, size_t length) { - return (bool) have_free_support; + opal_list_item_t *item; + + if (!release_run_callbacks) return; + + /* + * This is not really thread safe - but we can't hold the lock + * while calling the callback function as this routine can + * be called recursively. + * + * Instead, we could set a flag if we are already in the callback, + * and if called recursively queue the new address/length and allow + * the initial callback to dispatch this + */ + + opal_atomic_lock(&release_lock); + item = opal_list_get_first(&release_cb_list); + while(item != opal_list_get_end(&release_cb_list)) { + opal_list_item_t* next = opal_list_get_next(item); + callback_list_item_t cbitem = *(callback_list_item_t*) item; + item = next; + + opal_atomic_unlock(&release_lock); + cbitem.cbfunc(buf, length, cbitem.cbdata); + opal_atomic_lock(&release_lock); + } + opal_atomic_unlock(&release_lock); } int -opal_mem_free_register_handler(opal_mem_free_unpin_fn_t *func, void *cbdata) +opal_mem_hooks_support_level(void) +{ + return hooks_support; +} + + +int +opal_mem_hooks_register_alloc(opal_mem_hooks_callback_fn_t *func, void *cbdata) { opal_list_item_t *item; callback_list_item_t *cbitem, *new_cbitem; - int ret = OMPI_SUCCESS; + int ret = OPAL_SUCCESS; - if (!have_free_support) return OMPI_ERR_NOT_SUPPORTED; - - /* we either have or are about to have a registration that needs - calling back. Let the system know it needs to run callbacks - now */ - run_callbacks = true; - opal_atomic_mb(); + if (0 == (OPAL_MEMORY_FREE_SUPPORT & hooks_support)) { + return OPAL_ERR_NOT_SUPPORTED; + } /* pre-allocate a callback item on the assumption it won't be found. We can't call OBJ_NEW inside the lock because it might - call realloc */ + call alloc / realloc */ new_cbitem = OBJ_NEW(callback_list_item_t); if (NULL == new_cbitem) { - ret = OMPI_ERR_OUT_OF_RESOURCE; + ret = OPAL_ERR_OUT_OF_RESOURCE; goto done; } - opal_atomic_lock(&callback_lock); + opal_atomic_lock(&alloc_lock); + /* we either have or are about to have a registration that needs + calling back. Let the system know it needs to run callbacks + now */ + alloc_run_callbacks = true; + opal_atomic_mb(); /* make sure the callback isn't already in the list */ - for (item = opal_list_get_first(&callback_list) ; - item != opal_list_get_end(&callback_list) ; + for (item = opal_list_get_first(&alloc_cb_list) ; + item != opal_list_get_end(&alloc_cb_list) ; item = opal_list_get_next(item)) { cbitem = (callback_list_item_t*) item; if (cbitem->cbfunc == func) { - ret = OMPI_EXISTS; + ret = OPAL_EXISTS; goto done; } } @@ -179,12 +229,12 @@ opal_mem_free_register_handler(opal_mem_free_unpin_fn_t *func, void *cbdata) new_cbitem->cbfunc = func; new_cbitem->cbdata = cbdata; - opal_list_append(&callback_list, (opal_list_item_t*) new_cbitem); + opal_list_append(&alloc_cb_list, (opal_list_item_t*) new_cbitem); done: - opal_atomic_unlock(&callback_lock); + opal_atomic_unlock(&alloc_lock); - if (OMPI_EXISTS == ret && NULL != new_cbitem) { + if (OPAL_EXISTS == ret && NULL != new_cbitem) { OBJ_RELEASE(new_cbitem); } @@ -193,30 +243,121 @@ opal_mem_free_register_handler(opal_mem_free_unpin_fn_t *func, void *cbdata) int -opal_mem_free_unregister_handler(opal_mem_free_unpin_fn_t *func) +opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata) { opal_list_item_t *item; - opal_list_item_t *found_item = NULL; - callback_list_item_t *cbitem; - int ret = OMPI_ERR_NOT_FOUND; + callback_list_item_t *cbitem, *new_cbitem; + int ret = OPAL_SUCCESS; - opal_atomic_lock(&callback_lock); + if (0 == (OPAL_MEMORY_FREE_SUPPORT & hooks_support)) { + return OPAL_ERR_NOT_SUPPORTED; + } + + /* pre-allocate a callback item on the assumption it won't be + found. We can't call OBJ_NEW inside the lock because it might + call alloc / realloc */ + new_cbitem = OBJ_NEW(callback_list_item_t); + if (NULL == new_cbitem) { + ret = OPAL_ERR_OUT_OF_RESOURCE; + goto done; + } + + opal_atomic_lock(&release_lock); + /* we either have or are about to have a registration that needs + calling back. Let the system know it needs to run callbacks + now */ + release_run_callbacks = true; + opal_atomic_mb(); /* make sure the callback isn't already in the list */ - for (item = opal_list_get_first(&callback_list) ; - item != opal_list_get_end(&callback_list) ; + for (item = opal_list_get_first(&release_cb_list) ; + item != opal_list_get_end(&release_cb_list) ; item = opal_list_get_next(item)) { cbitem = (callback_list_item_t*) item; if (cbitem->cbfunc == func) { - opal_list_remove_item(&callback_list, item); + ret = OPAL_EXISTS; + goto done; + } + } + + new_cbitem->cbfunc = func; + new_cbitem->cbdata = cbdata; + + opal_list_append(&release_cb_list, (opal_list_item_t*) new_cbitem); + + done: + opal_atomic_unlock(&release_lock); + + if (OPAL_EXISTS == ret && NULL != new_cbitem) { + OBJ_RELEASE(new_cbitem); + } + + return ret; +} + + +int +opal_mem_hooks_unregister_alloc(opal_mem_hooks_callback_fn_t* func) +{ + opal_list_item_t *item; + opal_list_item_t *found_item = NULL; + callback_list_item_t *cbitem; + int ret = OPAL_ERR_NOT_FOUND; + + opal_atomic_lock(&alloc_lock); + + /* make sure the callback isn't already in the list */ + for (item = opal_list_get_first(&alloc_cb_list) ; + item != opal_list_get_end(&alloc_cb_list) ; + item = opal_list_get_next(item)) { + cbitem = (callback_list_item_t*) item; + + if (cbitem->cbfunc == func) { + opal_list_remove_item(&alloc_cb_list, item); found_item = item; - ret = OMPI_SUCCESS; + ret = OPAL_SUCCESS; break; } } - opal_atomic_unlock(&callback_lock); + opal_atomic_unlock(&alloc_lock); + + /* OBJ_ALLOC calls free, so we can't alloc until we get out of + the lock */ + if (NULL != found_item) { + OBJ_RELEASE(item); + } + + return ret; +} + + +int +opal_mem_hooks_unregister_release(opal_mem_hooks_callback_fn_t* func) +{ + opal_list_item_t *item; + opal_list_item_t *found_item = NULL; + callback_list_item_t *cbitem; + int ret = OPAL_ERR_NOT_FOUND; + + opal_atomic_lock(&release_lock); + + /* make sure the callback isn't already in the list */ + for (item = opal_list_get_first(&release_cb_list) ; + item != opal_list_get_end(&release_cb_list) ; + item = opal_list_get_next(item)) { + cbitem = (callback_list_item_t*) item; + + if (cbitem->cbfunc == func) { + opal_list_remove_item(&release_cb_list, item); + found_item = item; + ret = OPAL_SUCCESS; + break; + } + } + + opal_atomic_unlock(&release_lock); /* OBJ_RELEASE calls free, so we can't release until we get out of the lock */ diff --git a/opal/memoryhooks/memory.h b/opal/memoryhooks/memory.h index 1ae4e83f25..c908fc182c 100644 --- a/opal/memoryhooks/memory.h +++ b/opal/memoryhooks/memory.h @@ -30,14 +30,16 @@ #ifndef OPAL_MEMORY_MEMORY_H #define OPAl_MEMORY_MEMORY_H +#include "memory_internal.h" + #if defined(c_plusplus) || defined(__cplusplus) extern "C" { #endif -int opal_mem_free_init(void); +int opal_mem_hooks_init(void); -int opal_mem_free_finalize(void); +int opal_mem_hooks_finalize(void); /** @@ -56,26 +58,10 @@ int opal_mem_free_finalize(void); * @param cbdata Pointer-length of information passed to * the handler registration function. */ -typedef void (opal_mem_free_unpin_fn_t)(void *buf, size_t length, void *cbdata); +typedef void (opal_mem_hooks_callback_fn_t)(void *buf, size_t length, void *cbdata); -/** - * Query functionality of memory callbacks - * - * Query whether the system is capable of providing callbacks when - * memory is about to be released by a process. - * - * @retval true opal_mem_free_register_handler() will not return - * \c OMPI_ERR_NOT_SUPPORTED. - * @retval false opal_mem_free_register_handler() will always return - * \c OMPI_ERR_NOT_SUPPORTED. - * - * \note There is no reason you have to call this function before - * calling opal_mem_free_register_handler(). It exists for component - * selection logic that may want to see what the status of the memory - * hook support is without actually registering anything. - */ -bool opal_mem_free_is_supported(void); +int opal_mem_hooks_support_level(void); /** @@ -94,7 +80,9 @@ bool opal_mem_free_is_supported(void); * @retval OMPI_ERR_NOT_SUPPORTED There are no hooks available for * receiving callbacks when memory is to be released */ -int opal_mem_free_register_handler(opal_mem_free_unpin_fn_t *func, void *cbdata); +int opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata); + +int opal_mem_hooks_register_alloc(opal_mem_hooks_callback_fn_t *func, void *cbdata); /** @@ -107,7 +95,8 @@ int opal_mem_free_register_handler(opal_mem_free_unpin_fn_t *func, void *cbdata) * @retval OMPI_SUCCESS The function was successfully deregistered * @retval OMPI_ERR_NOT_FOUND The function was not previously registered */ -int opal_mem_free_unregister_handler(opal_mem_free_unpin_fn_t *func); +int opal_mem_hooks_unregister_release(opal_mem_hooks_callback_fn_t *func); +int opal_mem_hooks_unregister_alloc(opal_mem_hooks_callback_fn_t *func); #if defined(c_plusplus) || defined(__cplusplus) } diff --git a/opal/memoryhooks/memory_internal.h b/opal/memoryhooks/memory_internal.h index 18958c5fcc..684823fcde 100644 --- a/opal/memoryhooks/memory_internal.h +++ b/opal/memoryhooks/memory_internal.h @@ -19,7 +19,22 @@ #ifndef OPAL_MEMORY_MEMORY_INTERNAL_H #define OPAL_MEMORY_MEMORY_INTERNAL_H -void opal_mem_free_set_free_support(int support); -void opal_mem_free_release_hook(void *buf, size_t length); +#define OPAL_MEMORY_FREE_SUPPORT 0x0001 +#define OPAL_MEMORY_MALLOC_SUPPORT 0x0002 +#define OPAL_MEMORY_CHUNK_SUPPORT 0x0004 + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +void opal_mem_hooks_set_support(int support); + +void opal_mem_hooks_release_hook(void *buf, size_t length); +void opal_mem_hooks_alloc_hook(void *buf, size_t length); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + #endif /* OPAL_MEMORY_MEMORY_INTERNAL_H */ diff --git a/opal/runtime/opal_finalize.c b/opal/runtime/opal_finalize.c index 9e842d2584..ef4c9b27a6 100644 --- a/opal/runtime/opal_finalize.c +++ b/opal/runtime/opal_finalize.c @@ -64,7 +64,7 @@ int opal_finalize(void) mca_base_close(); /* finalize the memory manager / tracker */ - opal_mem_free_finalize(); + opal_mem_hooks_finalize(); /* finalize the memory allocator */ opal_malloc_finalize(); diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c index c4cd1ca3b2..ca67893011 100644 --- a/opal/runtime/opal_init.c +++ b/opal/runtime/opal_init.c @@ -112,7 +112,7 @@ int opal_init(void) } /* initialize the memory manager / tracker */ - if (OPAL_SUCCESS != opal_mem_free_init()) { + if (OPAL_SUCCESS != opal_mem_hooks_init()) { error = "opal_mem_free_init"; goto return_error; } diff --git a/orte/tools/orteprobe/orteprobe.c b/orte/tools/orteprobe/orteprobe.c index 70b572eafb..4eba96005c 100644 --- a/orte/tools/orteprobe/orteprobe.c +++ b/orte/tools/orteprobe/orteprobe.c @@ -191,7 +191,7 @@ int main(int argc, char *argv[]) opal_malloc_init(); /* initialize the memory manager / tracker */ - opal_mem_free_init(); + opal_mem_hooks_init(); opal_memory_base_open(); diff --git a/test/memory/opal_memory_basic.c b/test/memory/opal_memory_basic.c index 1093173de9..dcddce9472 100644 --- a/test/memory/opal_memory_basic.c +++ b/test/memory/opal_memory_basic.c @@ -26,13 +26,20 @@ #include "opal/runtime/opal.h" #include "opal/memoryhooks/memory.h" -int ret = 2; +int ret = 1; /* because of the munmap */ int size = 10 * 1024 * 1024; static void -callback(void *buf, size_t length, void *cbdata) +alloc_callback(void *buf, size_t length, void *cbdata) { - printf("\tcallback with %lx, %d\n", (unsigned long) buf, (int) length); + printf("\talloc callback with %lx, %d\n", (unsigned long) buf, (int) length); + ret++; +} + +static void +release_callback(void *buf, size_t length, void *cbdata) +{ + printf("\trelease callback with %lx, %d\n", (unsigned long) buf, (int) length); ret--; } @@ -41,13 +48,16 @@ main(int argc, char *argv[]) { void * foo, *bar; int retval; + return 77; opal_init(); - if (!opal_mem_free_is_supported()) { + if (0 == ((OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_MALLOC_SUPPORT) & + opal_mem_hooks_support_level())) { printf("no memory registration supported. skipping\n"); return 77; } - retval = opal_mem_free_register_handler(callback, NULL); + retval = opal_mem_hooks_register_release(release_callback, NULL); + retval |= opal_mem_hooks_register_alloc(alloc_callback, NULL); if (retval != OMPI_SUCCESS) { printf("handler registration failed\n"); return retval; @@ -66,26 +76,29 @@ main(int argc, char *argv[]) from a small block to a big one will fail), so don't make this an error */ if (ret == 0) { - ret = 2; + ret = 0; printf(" - realloc\n"); foo = malloc(size); bar = malloc(10); foo = realloc(foo, size * 2); free(bar); free(foo); - if (ret > 0) { + if (ret != 0) { printf("WARNING - It appears that realloc does not trigger a callback\n"); printf("WARNING - this may be a problem or it may be a sign that your\n"); printf("WARNING - memory manager is better than mine\n"); printf("ret: %d\n", ret); } ret = 0; + printf("here\n"); } - retval = opal_mem_free_unregister_handler(callback); + retval = opal_mem_hooks_unregister_release(release_callback); + retval |= opal_mem_hooks_unregister_alloc(alloc_callback); if (retval != OMPI_SUCCESS) return retval; opal_finalize(); + printf("ret: %d\n", ret); return ret; } diff --git a/test/memory/opal_memory_cxx.cc b/test/memory/opal_memory_cxx.cc index be197f7544..5829e39488 100644 --- a/test/memory/opal_memory_cxx.cc +++ b/test/memory/opal_memory_cxx.cc @@ -46,12 +46,13 @@ main(int argc, char *argv[]) opal_init(); - if (0 == (int) opal_mem_free_is_supported()) { + if (0 == ((OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_MALLOC_SUPPORT) & + opal_mem_hooks_support_level())) { printf("no memory registration supported. skipping\n"); return 77; } - retval = opal_mem_free_register_handler(callback, NULL); + retval = opal_mem_hooks_register_release(callback, NULL); if (retval != OMPI_SUCCESS) return retval; vector *big_vec; @@ -71,7 +72,7 @@ main(int argc, char *argv[]) printf(" - all done\n"); - retval = opal_mem_free_unregister_handler(callback); + retval = opal_mem_hooks_unregister_release(callback); if (retval != OMPI_SUCCESS) return retval; opal_finalize(); diff --git a/test/memory/opal_memory_speed.c b/test/memory/opal_memory_speed.c index a2f5348e2d..ece260946e 100644 --- a/test/memory/opal_memory_speed.c +++ b/test/memory/opal_memory_speed.c @@ -44,7 +44,8 @@ main(int argc, char *argv[]) long time; opal_init(); - if (!opal_mem_free_is_supported()) { + if (0 == ((OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_MALLOC_SUPPORT) & + opal_mem_hooks_support_level())) { printf("no memory registration supported. skipping\n"); return 77; } @@ -72,7 +73,7 @@ main(int argc, char *argv[]) iters, time, (double) time / iters); printf("speed with empty handler:\n"); - retval = opal_mem_free_register_handler(callback, NULL); + retval = opal_mem_hooks_register_release(callback, NULL); if (retval != OMPI_SUCCESS) { printf("handler registration failed\n"); return retval; @@ -97,7 +98,7 @@ main(int argc, char *argv[]) (end.tv_usec - start.tv_usec); printf(" free: %d calls in %ld microseconds. %lf microseconds/call\n", iters, time, (double) time / iters); - opal_mem_free_unregister_handler(callback); + opal_mem_hooks_unregister_release(callback); printf("speed without a handler:\n"); @@ -122,7 +123,7 @@ main(int argc, char *argv[]) iters, time, (double) time / iters); printf("speed with empty handler:\n"); - retval = opal_mem_free_register_handler(callback, NULL); + retval = opal_mem_hooks_register_release(callback, NULL); if (retval != OMPI_SUCCESS) { printf("handler registration failed\n"); return retval; @@ -147,7 +148,7 @@ main(int argc, char *argv[]) (end.tv_usec - start.tv_usec); printf(" free: %d calls in %ld microseconds. %lf microseconds/call\n", iters, time, (double) time / iters); - opal_mem_free_unregister_handler(callback); + opal_mem_hooks_unregister_release(callback); opal_finalize();