1
1

Add ummunotify support to Open MPI. See

http://marc.info/?l=linux-mm-commits&m=127352503417787&w=2 for more
details.

 * Remove the ptmalloc memory component; replace it with a new "linux"
   memory component.
 * The linux memory component will conditionally compile in support
   for ummunotify.  At run-time, if it has ummunotify support and
   finds run-time support for ummunotify (i.e., /dev/ummunotify), it
   uses it.  If not, it tries to use ptmalloc via the glibc memory
   hooks. 
 * Add some more API functions to the memory framework to accomodate
   the ummunotify model (i.e., poll to see if memory has "changed").
 * Add appropriate calls in the rcache to the new memory APIs to see
   if memory has changed, and to react accordingly.
 * Add a few comments in the openib BTL to indicate why we don't need
   to notify the OPAL memory framework about specific instances of
   registered memory.
 * Add dummy API calls in the solaris malloc component (since it
   doesn't have polling/"did memory change" support).

This commit was SVN r23113.
Этот коммит содержится в:
Jeff Squyres 2010-05-11 21:43:19 +00:00
родитель 4bd25f587c
Коммит c7c3de87f5
50 изменённых файлов: 1362 добавлений и 512 удалений

Просмотреть файл

@ -45,7 +45,6 @@ const char *ibv_get_sysfs_path(void);
#include <malloc.h> #include <malloc.h>
#include <stddef.h> #include <stddef.h>
#include "ompi/constants.h"
#include "opal/event/event.h" #include "opal/event/event.h"
#include "opal/align.h" #include "opal/align.h"
#include "opal/util/output.h" #include "opal/util/output.h"
@ -66,6 +65,7 @@ const char *ibv_get_sysfs_path(void);
#include "orte/runtime/orte_globals.h" #include "orte/runtime/orte_globals.h"
#include "orte/mca/notifier/notifier.h" #include "orte/mca/notifier/notifier.h"
#include "ompi/constants.h"
#include "ompi/proc/proc.h" #include "ompi/proc/proc.h"
#include "ompi/mca/btl/btl.h" #include "ompi/mca/btl/btl.h"
#include "ompi/mca/mpool/base/base.h" #include "ompi/mca/mpool/base/base.h"
@ -518,8 +518,9 @@ static int openib_reg_mr(void *reg_data, void *base, size_t size,
openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size, IBV_ACCESS_LOCAL_WRITE | openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size, IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ); IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
if(NULL == openib_reg->mr) if (NULL == openib_reg->mr) {
return OMPI_ERR_OUT_OF_RESOURCE; return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }

Просмотреть файл

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007 Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2007 Mellanox Technologies, Inc. All rights reserved.
* *
* $COPYRIGHT$ * $COPYRIGHT$
@ -430,6 +430,10 @@ int ompi_btl_openib_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint)
BTL_ERROR(("Failed to reg mr!")); BTL_ERROR(("Failed to reg mr!"));
return OMPI_ERR_OUT_OF_RESOURCE; return OMPI_ERR_OUT_OF_RESOURCE;
} }
/* NOTE: We do not need to register this memory with the
opal_memory subsystem, because this is OMPI-controlled memory
-- we do not need to worry about this memory being freed out
from underneath us. */
/* Copy the lkey where it needs to go */ /* Copy the lkey where it needs to go */
endpoint->endpoint_cts_frag.super.sg_entry.lkey = endpoint->endpoint_cts_frag.super.sg_entry.lkey =
@ -453,6 +457,9 @@ int ompi_btl_openib_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint)
int ompi_btl_openib_connect_base_free_cts(mca_btl_base_endpoint_t *endpoint) int ompi_btl_openib_connect_base_free_cts(mca_btl_base_endpoint_t *endpoint)
{ {
/* NOTE: We don't need to deregister this memory with opal_memory
because it was not registered there in the first place (see
comment above, near call to ibv_reg_mr). */
if (NULL != endpoint->endpoint_cts_mr) { if (NULL != endpoint->endpoint_cts_mr) {
ibv_dereg_mr(endpoint->endpoint_cts_mr); ibv_dereg_mr(endpoint->endpoint_cts_mr);
endpoint->endpoint_cts_mr = NULL; endpoint->endpoint_cts_mr = NULL;

Просмотреть файл

@ -1,4 +1,3 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/* /*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -10,7 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006 Voltaire. All rights reserved. * Copyright (c) 2006 Voltaire. All rights reserved.
* Copyright (c) 2007 Mellanox Technologies. All rights reserved. * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved.

Просмотреть файл

@ -1,4 +1,3 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/* /*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -10,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -18,6 +18,9 @@
*/ */
#include "ompi_config.h" #include "ompi_config.h"
#include MCA_memory_IMPLEMENTATION_HEADER
#include "opal/mca/memory/memory.h"
#include "ompi/mca/rcache/rcache.h" #include "ompi/mca/rcache/rcache.h"
#include "rcache_rb.h" #include "rcache_rb.h"
#include "rcache_rb_tree.h" #include "rcache_rb_tree.h"
@ -55,6 +58,13 @@ int mca_rcache_rb_find( struct mca_rcache_base_module_t* rcache,
OPAL_THREAD_LOCK(&rcache->lock); OPAL_THREAD_LOCK(&rcache->lock);
*cnt = 0; *cnt = 0;
/* Check to ensure that the cache is valid */
if (OPAL_UNLIKELY(opal_memory_changed() &&
NULL != opal_memory->memoryc_process &&
OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
return rc;
}
if(ompi_rb_tree_size(&((mca_rcache_rb_module_t*)rcache)->rb_tree) == 0) { if(ompi_rb_tree_size(&((mca_rcache_rb_module_t*)rcache)->rb_tree) == 0) {
OPAL_THREAD_UNLOCK(&rcache->lock); OPAL_THREAD_UNLOCK(&rcache->lock);
return OMPI_SUCCESS; return OMPI_SUCCESS;
@ -100,6 +110,14 @@ int mca_rcache_rb_insert (
OPAL_THREAD_LOCK(&rcache->lock); OPAL_THREAD_LOCK(&rcache->lock);
reg->flags = flags; reg->flags = flags;
/* Check to ensure that the cache is valid */
if (OPAL_UNLIKELY(opal_memory_changed() &&
NULL != opal_memory->memoryc_process &&
OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
return rc;
}
if(flags & MCA_MPOOL_FLAGS_CACHE) { if(flags & MCA_MPOOL_FLAGS_CACHE) {
rc = mca_rcache_rb_mru_insert( (mca_rcache_rb_module_t*) rcache, reg); rc = mca_rcache_rb_mru_insert( (mca_rcache_rb_module_t*) rcache, reg);
if(OMPI_SUCCESS != rc) { if(OMPI_SUCCESS != rc) {
@ -119,6 +137,15 @@ int mca_rcache_rb_insert (
} }
rc = mca_rcache_rb_tree_insert((mca_rcache_rb_module_t*)rcache, reg); rc = mca_rcache_rb_tree_insert((mca_rcache_rb_module_t*)rcache, reg);
OPAL_THREAD_ADD32((int32_t*) &reg->ref_count, 1); OPAL_THREAD_ADD32((int32_t*) &reg->ref_count, 1);
if (OPAL_LIKELY(OMPI_SUCCESS == rc)) {
/* If we successfully registered, then tell the memory manager
to start monitoring this region */
opal_memory->memoryc_register(reg->base,
(uint64_t) reg->bound - reg->base + 1,
(uint64_t) reg);
}
OPAL_THREAD_UNLOCK(&rcache->lock); OPAL_THREAD_UNLOCK(&rcache->lock);
return rc; return rc;
} }
@ -133,6 +160,11 @@ int mca_rcache_rb_delete (
assert(reg->ref_count >= 1); assert(reg->ref_count >= 1);
OPAL_THREAD_LOCK(&rcache->lock); OPAL_THREAD_LOCK(&rcache->lock);
/* Tell the memory manager that we no longer care about this
region */
opal_memory->memoryc_deregister(reg->base,
(uint64_t) (reg->bound - reg->base),
(uint64_t) reg);
if(flags & MCA_MPOOL_FLAGS_CACHE) { if(flags & MCA_MPOOL_FLAGS_CACHE) {
assert(reg->ref_count >= 2); assert(reg->ref_count >= 2);
OPAL_THREAD_ADD32((int32_t*)&reg->ref_count, -1); OPAL_THREAD_ADD32((int32_t*)&reg->ref_count, -1);

Просмотреть файл

@ -1,4 +1,3 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/* /*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -10,8 +9,8 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
*
* Copyright (c) 2006 Voltaire. All rights reserved. * Copyright (c) 2006 Voltaire. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2009 IBM Corporation. All rights reserved. * Copyright (c) 2009 IBM Corporation. All rights reserved.
* *
* $COPYRIGHT$ * $COPYRIGHT$
@ -23,6 +22,8 @@
#include "ompi_config.h" #include "ompi_config.h"
#include MCA_memory_IMPLEMENTATION_HEADER
#include "opal/mca/memory/memory.h"
#include "ompi/mca/rcache/rcache.h" #include "ompi/mca/rcache/rcache.h"
#include "rcache_vma.h" #include "rcache_vma.h"
#include "rcache_vma_tree.h" #include "rcache_vma_tree.h"
@ -46,6 +47,7 @@ void mca_rcache_vma_module_init( mca_rcache_vma_module_t* rcache ) {
int mca_rcache_vma_find(struct mca_rcache_base_module_t* rcache, int mca_rcache_vma_find(struct mca_rcache_base_module_t* rcache,
void* addr, size_t size, mca_mpool_base_registration_t **reg) void* addr, size_t size, mca_mpool_base_registration_t **reg)
{ {
int rc;
void* base_addr; void* base_addr;
void* bound_addr; void* bound_addr;
@ -56,6 +58,13 @@ int mca_rcache_vma_find(struct mca_rcache_base_module_t* rcache,
base_addr = down_align_addr(addr, mca_mpool_base_page_size_log); base_addr = down_align_addr(addr, mca_mpool_base_page_size_log);
bound_addr = up_align_addr((void*) ((unsigned long) addr + size - 1), mca_mpool_base_page_size_log); bound_addr = up_align_addr((void*) ((unsigned long) addr + size - 1), mca_mpool_base_page_size_log);
/* Check to ensure that the cache is valid */
if (OPAL_UNLIKELY(opal_memory_changed() &&
NULL != opal_memory->memoryc_process &&
OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
return rc;
}
*reg = mca_rcache_vma_tree_find((mca_rcache_vma_module_t*)rcache, (unsigned char*)base_addr, *reg = mca_rcache_vma_tree_find((mca_rcache_vma_module_t*)rcache, (unsigned char*)base_addr,
(unsigned char*)bound_addr); (unsigned char*)bound_addr);
@ -66,6 +75,7 @@ int mca_rcache_vma_find_all(struct mca_rcache_base_module_t* rcache,
void* addr, size_t size, mca_mpool_base_registration_t **regs, void* addr, size_t size, mca_mpool_base_registration_t **regs,
int reg_cnt) int reg_cnt)
{ {
int rc;
void *base_addr, *bound_addr; void *base_addr, *bound_addr;
if(size == 0) { if(size == 0) {
@ -75,6 +85,13 @@ int mca_rcache_vma_find_all(struct mca_rcache_base_module_t* rcache,
base_addr = down_align_addr(addr, mca_mpool_base_page_size_log); base_addr = down_align_addr(addr, mca_mpool_base_page_size_log);
bound_addr = up_align_addr((void*) ((unsigned long) addr + size - 1), mca_mpool_base_page_size_log); bound_addr = up_align_addr((void*) ((unsigned long) addr + size - 1), mca_mpool_base_page_size_log);
/* Check to ensure that the cache is valid */
if (OPAL_UNLIKELY(opal_memory_changed() &&
NULL != opal_memory->memoryc_process &&
OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
return rc;
}
return mca_rcache_vma_tree_find_all((mca_rcache_vma_module_t*)rcache, return mca_rcache_vma_tree_find_all((mca_rcache_vma_module_t*)rcache,
(unsigned char*)base_addr, (unsigned char*)bound_addr, regs, (unsigned char*)base_addr, (unsigned char*)bound_addr, regs,
reg_cnt); reg_cnt);
@ -83,6 +100,7 @@ int mca_rcache_vma_find_all(struct mca_rcache_base_module_t* rcache,
int mca_rcache_vma_insert(struct mca_rcache_base_module_t* rcache, int mca_rcache_vma_insert(struct mca_rcache_base_module_t* rcache,
mca_mpool_base_registration_t* reg, size_t limit) mca_mpool_base_registration_t* reg, size_t limit)
{ {
int rc;
size_t reg_size = reg->bound - reg->base + 1; size_t reg_size = reg->bound - reg->base + 1;
mca_rcache_vma_module_t *vma_rcache = (mca_rcache_vma_module_t*)rcache; mca_rcache_vma_module_t *vma_rcache = (mca_rcache_vma_module_t*)rcache;
@ -92,13 +110,33 @@ int mca_rcache_vma_insert(struct mca_rcache_base_module_t* rcache,
return OMPI_ERR_OUT_OF_RESOURCE; return OMPI_ERR_OUT_OF_RESOURCE;
} }
return mca_rcache_vma_tree_insert(vma_rcache, reg, limit); /* Check to ensure that the cache is valid */
if (OPAL_UNLIKELY(opal_memory_changed() &&
NULL != opal_memory->memoryc_process &&
OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
return rc;
}
rc = mca_rcache_vma_tree_insert(vma_rcache, reg, limit);
if (OPAL_LIKELY(OMPI_SUCCESS == rc)) {
/* If we successfully registered, then tell the memory manager
to start monitoring this region */
opal_memory->memoryc_register(reg->base,
(uint64_t) reg_size, (uint64_t) reg);
}
return rc;
} }
int mca_rcache_vma_delete(struct mca_rcache_base_module_t* rcache, int mca_rcache_vma_delete(struct mca_rcache_base_module_t* rcache,
mca_mpool_base_registration_t* reg) mca_mpool_base_registration_t* reg)
{ {
mca_rcache_vma_module_t *vma_rcache = (mca_rcache_vma_module_t*)rcache; mca_rcache_vma_module_t *vma_rcache = (mca_rcache_vma_module_t*)rcache;
/* Tell the memory manager that we no longer care about this
region */
opal_memory->memoryc_deregister(reg->base,
(uint64_t) (reg->bound - reg->base),
(uint64_t) reg);
return mca_rcache_vma_tree_delete(vma_rcache, reg); return mca_rcache_vma_tree_delete(vma_rcache, reg);
} }

Просмотреть файл

@ -9,6 +9,7 @@
# University of Stuttgart. All rights reserved. # University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California. # Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved. # All rights reserved.
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$ # $COPYRIGHT$
# #
# Additional copyrights may follow # Additional copyrights may follow
@ -17,8 +18,10 @@
# #
headers += \ headers += \
base/base.h base/base.h \
base/empty.h
libmca_memory_la_SOURCES += \ libmca_memory_la_SOURCES += \
base/memory_base_close.c \ base/memory_base_close.c \
base/memory_base_open.c base/memory_base_open.c \
base/memory_base_empty.c

56
opal/mca/memory/base/empty.h Обычный файл
Просмотреть файл

@ -0,0 +1,56 @@
/*
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_MCA_MEMCPY_BASE_MEMORY_BASE_DEFAULT_H
#define OPAL_MCA_MEMCPY_BASE_MEMORY_BASE_DEFAULT_H
#include "opal_config.h"
/**
* Provided for memory components that don't provide an asynchronous
* method for determining if memory mappings have changed. If a
* component only has a synchronous / interactive method of checking,
* then it can override this macro with some logic to do a (hopefully)
* cheap check to see if memory mappings have changed. The intent is
* that if this cheap check returns true, the upper layer will then
* invoke the memoryc_process() function to actually process what
* changed. This function will be invoked by the upper layer with the
* syntax:
*
* if (opal_memory_changed()) { ... }
*
* Hence, if you need any kind of sophisticated logic, you might want
* to put it in an inline function and have the #define call the
* inline function.
*/
#define opal_memory_changed() 0
BEGIN_C_DECLS
/**
* Default (empty) implementation of the memoryc_register function.
*
* See opal/mca/memory/memory.h for a description of the parameters.
*/
OPAL_DECLSPEC int opal_memory_base_component_register_empty(void *start,
size_t len,
uint64_t cookie);
/**
* Default (empty) implementation of the memoryc_deregister function
*
* See opal/mca/memory/memory.h for a description of the parameters.
*/
OPAL_DECLSPEC int opal_memory_base_component_deregister_empty(void *start,
size_t len,
uint64_t cookie);
END_C_DECLS
#endif

Просмотреть файл

@ -5,11 +5,10 @@
* Copyright (c) 2004-2005 The University of Tennessee and The University * Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights * of Tennessee Research Foundation. All rights
* reserved. * reserved.
* Copyright (c) 2004-2007 High Performance Computing Center Stuttgart, * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -17,18 +16,21 @@
* $HEADER$ * $HEADER$
*/ */
#ifndef OPAL_PTMALLOC2_MUNMAP_H
#define OPAL_PTMALLOC2_MUNMAP_H
#include "opal_config.h" #include "opal_config.h"
BEGIN_C_DECLS #include "opal/constants.h"
#include "opal/mca/memory/base/empty.h"
OPAL_DECLSPEC int opal_mem_free_ptmalloc2_munmap(void *start, size_t length, int from_alloc, int call_hooks);
OPAL_DECLSPEC int munmap(void* addr, size_t len);
END_C_DECLS int opal_memory_base_component_register_empty(void *base, size_t len,
uint64_t cookie)
{
return OPAL_SUCCESS;
}
#endif /* !OPAL_PTMALLOC2_MUNMAP_H */
int opal_memory_base_component_deregister_empty(void *base, size_t len,
uint64_t cookie)
{
return OPAL_SUCCESS;
}

Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -25,6 +26,7 @@
#include "opal/mca/base/mca_base_param.h" #include "opal/mca/base/mca_base_param.h"
#include "opal/mca/memory/memory.h" #include "opal/mca/memory/memory.h"
#include "opal/mca/memory/base/base.h" #include "opal/mca/memory/base/base.h"
#include "opal/mca/memory/base/empty.h"
/* /*
@ -34,12 +36,31 @@
*/ */
#include "opal/mca/memory/base/static-components.h" #include "opal/mca/memory/base/static-components.h"
static int empty_process(void)
{
return OPAL_SUCCESS;
}
/*
* Local variables
*/
static opal_memory_base_component_2_0_0_t empty_component = {
/* Don't care about the version info */
{ 0, },
/* Don't care about the data */
{ 0, },
/* Empty / safe functions to call if no memory componet is selected */
empty_process,
opal_memory_base_component_register_empty,
opal_memory_base_component_deregister_empty,
};
/* /*
* Globals * Globals
*/ */
opal_list_t opal_memory_base_components_opened; opal_list_t opal_memory_base_components_opened;
opal_memory_base_component_2_0_0_t *opal_memory_active_component = NULL; opal_memory_base_component_2_0_0_t *opal_memory = &empty_component;
/* /*
* Function for finding and opening either all MCA components, or the one * Function for finding and opening either all MCA components, or the one
@ -61,7 +82,7 @@ int opal_memory_base_open(void)
mca_base_component_list_item_t *item; mca_base_component_list_item_t *item;
item = (mca_base_component_list_item_t*) item = (mca_base_component_list_item_t*)
opal_list_get_first(&opal_memory_base_components_opened); opal_list_get_first(&opal_memory_base_components_opened);
opal_memory_active_component = (opal_memory_base_component_2_0_0_t*) opal_memory = (opal_memory_base_component_2_0_0_t*)
item->cli_component; item->cli_component;
} }

Просмотреть файл

@ -10,6 +10,7 @@ dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
dnl University of Stuttgart. All rights reserved. dnl University of Stuttgart. All rights reserved.
dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl Copyright (c) 2004-2005 The Regents of the University of California.
dnl All rights reserved. dnl All rights reserved.
dnl Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
dnl $COPYRIGHT$ dnl $COPYRIGHT$
dnl dnl
dnl Additional copyrights may follow dnl Additional copyrights may follow
@ -33,4 +34,13 @@ AC_DEFUN([MCA_memory_CONFIG],[
AC_DEFINE_UNQUOTED([OMPI_MEMORY_HAVE_COMPONENT], [$memory_base_found], AC_DEFINE_UNQUOTED([OMPI_MEMORY_HAVE_COMPONENT], [$memory_base_found],
[Whether any opal memory mca components were found]) [Whether any opal memory mca components were found])
# See if someone set to use their header file
if test "$memory_base_include" = "" ; then
memory_base_include="base/empty.h"
fi
AC_DEFINE_UNQUOTED([MCA_memory_IMPLEMENTATION_HEADER],
["opal/mca/memory/$memory_base_include"],
[Header to include for parts of the memory implementation])
]) ])

Просмотреть файл

@ -9,6 +9,7 @@
# University of Stuttgart. All rights reserved. # University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California. # Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved. # All rights reserved.
# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$ # $COPYRIGHT$
# #
# Additional copyrights may follow # Additional copyrights may follow
@ -36,52 +37,60 @@ AM_CPPFLAGS += -I$(srcdir)/sysdeps/generic
# copyright notices. # copyright notices.
docdir = $(datadir)/openmpi/doc docdir = $(datadir)/openmpi/doc
doc_DATA = ptmalloc2-COPYRIGHT doc_DATA = COPYRIGHT-ptmalloc2.txt
# These are the files from ptmalloc2 that we care about
ptmalloc2_sources = \
malloc.c \
malloc-stats.c \
malloc.h
# Help file # Help file
dist_pkgdata_DATA = help-opal-memory-ptmalloc2.txt dist_pkgdata_DATA = help-opal-memory-linux.txt
# This component is only ever built statically (i.e., slurped into # This component is only ever built statically (i.e., slurped into
# libopen-pal) -- it is never built as a DSO. # libopen-pal) -- it is never built as a DSO.
noinst_LTLIBRARIES = libmca_memory_ptmalloc2.la noinst_LTLIBRARIES = libmca_memory_linux.la
libmca_memory_ptmalloc2_la_SOURCES = \ libmca_memory_linux_la_SOURCES = \
opal_ptmalloc2_component.c \ memory_linux.h \
opal_ptmalloc2_munmap.c \ memory_linux_component.c
rename.h \ libmca_memory_linux_la_LDFLAGS = \
$(ptmalloc2_sources) -module -avoid-version $(memory_linux_LDFLAGS)
libmca_memory_ptmalloc2_la_LDFLAGS = \ libmca_memory_linux_la_LIBADD = $(memory_linux_LIBS)
-module -avoid-version $(memory_ptmalloc2_LDFLAGS)
libmca_memory_ptmalloc2_la_LIBADD = $(memory_ptmalloc2_LIBS) # Do we have ptmalloc2 support?
if MEMORY_LINUX_PTMALLOC2
libmca_memory_linux_la_SOURCES += \
memory_linux_ptmalloc2.c \
memory_linux_munmap.c \
rename.h \
malloc.c \
malloc-stats.c \
malloc.h
endif
# Do we have ummunotify support?
if MEMORY_LINUX_UMMUNOTIFY
libmca_memory_linux_la_SOURCES += memory_linux_ummunotify.c public.h
endif
# these are included directly and shouldn't be built solo # these are included directly and shouldn't be built solo
EXTRA_libmca_memory_ptmalloc2_la_SOURCES = \ EXTRA_libmca_memory_linux_la_SOURCES = \
arena.c \ arena.c \
hooks.c hooks.c
EXTRA_DIST = \ EXTRA_DIST = \
ChangeLog \ README-open-mpi.txt \
README \ README-ptmalloc2.txt \
ChangeLog \ ChangeLog-ptmalloc2.txt \
COPYRIGHT-ptmalloc2.txt \
lran2.h \ lran2.h \
opal_ptmalloc2_munmap.h \
t-test.h \ t-test.h \
t-test1.c \ t-test1.c \
t-test2.c \ t-test2.c \
tst-mallocstate.c \ tst-mallocstate.c \
tst-mstats.c \ tst-mstats.c \
sysdeps/sproc/malloc-machine.h \ sysdeps/sproc/malloc-machine.h \
sysdeps/sproc/thread-st.h \ sysdeps/sproc/thread-st.h \
sysdeps/pthread/malloc-machine.h \ sysdeps/pthread/malloc-machine.h \
sysdeps/pthread/thread-st.h \ sysdeps/pthread/thread-st.h \
sysdeps/solaris/malloc-machine.h \ sysdeps/solaris/malloc-machine.h \
sysdeps/solaris/thread-st.h \ sysdeps/solaris/thread-st.h \
sysdeps/generic/malloc-machine.h \ sysdeps/generic/malloc-machine.h \
sysdeps/generic/thread-st.h \ sysdeps/generic/thread-st.h \
sysdeps/generic/atomic.h \ sysdeps/generic/atomic.h \
$(doc_DATA) $(doc_DATA)

Просмотреть файл

@ -18,10 +18,6 @@
write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA. */ Boston, MA 02111-1307, USA. */
/* Copyright (c) 2009-2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
*/
/* $Id: arena.c,v 1.9 2004/11/05 14:42:23 wg Exp $ */ /* $Id: arena.c,v 1.9 2004/11/05 14:42:23 wg Exp $ */
/* Compile-time constants. */ /* Compile-time constants. */
@ -294,7 +290,6 @@ ptmalloc_unlock_all2 __MALLOC_P((void))
#endif /* !defined NO_THREADS */ #endif /* !defined NO_THREADS */
/* Initialization routine. */ /* Initialization routine. */
#ifdef _LIBC #ifdef _LIBC
#include <string.h> #include <string.h>
@ -520,10 +515,6 @@ dump_heap(heap) heap_info *heap;
#endif /* MALLOC_DEBUG > 1 */ #endif /* MALLOC_DEBUG > 1 */
extern int opal_mem_free_ptmalloc2_munmap(void *start, size_t length, int from_alloc, int run_hooks);
/* Create a new heap. size is automatically rounded up to a multiple /* Create a new heap. size is automatically rounded up to a multiple
of the page size. */ of the page size. */
@ -558,9 +549,8 @@ new_heap(size, top_pad) size_t size, top_pad;
if(p1 != MAP_FAILED) { if(p1 != MAP_FAILED) {
p2 = (char *)(((unsigned long)p1 + (HEAP_MAX_SIZE-1)) & ~(HEAP_MAX_SIZE-1)); p2 = (char *)(((unsigned long)p1 + (HEAP_MAX_SIZE-1)) & ~(HEAP_MAX_SIZE-1));
ul = p2 - p1; ul = p2 - p1;
opal_mem_free_ptmalloc2_munmap(p1, ul, 1, 0); munmap(p1, ul);
opal_mem_free_ptmalloc2_munmap(p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul, munmap(p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
1, 0);
} else { } else {
/* Try to take the chance that an allocation of only HEAP_MAX_SIZE /* Try to take the chance that an allocation of only HEAP_MAX_SIZE
is already aligned. */ is already aligned. */
@ -568,12 +558,12 @@ new_heap(size, top_pad) size_t size, top_pad;
if(p2 == MAP_FAILED) if(p2 == MAP_FAILED)
return 0; return 0;
if((unsigned long)p2 & (HEAP_MAX_SIZE-1)) { if((unsigned long)p2 & (HEAP_MAX_SIZE-1)) {
opal_mem_free_ptmalloc2_munmap(p2, HEAP_MAX_SIZE, 1, 0); munmap(p2, HEAP_MAX_SIZE);
return 0; return 0;
} }
} }
if(mprotect(p2, size, PROT_READ|PROT_WRITE) != 0) { if(mprotect(p2, size, PROT_READ|PROT_WRITE) != 0) {
opal_mem_free_ptmalloc2_munmap(p2, HEAP_MAX_SIZE, 1, 0); munmap(p2, HEAP_MAX_SIZE);
return 0; return 0;
} }
h = (heap_info *)p2; h = (heap_info *)p2;

196
opal/mca/memory/linux/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,196 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2008-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AC_DEFUN([MCA_memory_linux_COMPILE_MODE], [
AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
$4="static"
AC_MSG_RESULT([$$4])
])
# MCA_memory_linux_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_memory_linux_CONFIG],[
OMPI_VAR_SCOPE_PUSH([memory_linux_ptmalloc2_happy memory_linux_ummu_happy memory_linux_requested icc_major_ver icc_minor_ver memory_linux_mmap memory_linux_munmap memory_linux_LIBS_SAVE])
# Only allow this component to build on Linux-based systems
AC_MSG_CHECKING([operating system])
case $host in
*linux*)
AC_MSG_RESULT([$host -- supported])
memory_linux_ptmalloc2_happy=yes
memory_linux_ummu_happy=yes
;;
*)
AC_MSG_RESULT([$host -- unsupported])
memory_linux_ptmalloc2_happy=no
memory_linux_ummu_happy=no
;;
esac
AS_IF([test "$with_memory_manager" = "linux"],
[memory_linux_ptmalloc2_happy=yes
memory_linux_ummu_happy=yes
memory_linux_requested=1],
[memory_linux_requested=0
AS_IF([test "$with_memory_manager" = ""],
[memory_linux_ptmalloc2_happy=yes
memory_linux_ummu_happy=yes],
[memory_linux_ptmalloc2_happy=yes
memory_linux_ummu_happy=no])])
######################################################################
# ptmalloc2
######################################################################
# Per ticket #227, Intel 9.0 v20051201 on ia64 with optimization
# of -O2 or higher will bork linux in strange in mysterious ways.
# Doh! So if the compiler vendor is intel and we're on an ia64
# box, run "icc --version" and snarf the version string. If it's
# 9.0 and the version is <= 20051201, then disable ptmalloc2.
# Executive decision: ignore optimization levels (even though -O1
# and -O0 seem to work). The upgrade to 9.1 is free, so that's a
# better path than trying to make a much more complicated test
# here.
AS_IF([test "$memory_linux_ptmalloc2_happy" = yes],
[case $host in
ia64-*)
AS_IF([test "$ompi_c_vendor" = "intel"],
[# check for v9.0 <= 20051201
icc_major_ver="`$CC --version | head -n 1 | awk '{ print [$]3 }'`"
icc_minor_ver="`$CC --version | head -n 1 | awk '{ print [$]4 }'`"
AS_IF([test "$icc_major_ver" = "9.0" -a "`expr $icc_minor_ver \<= 20051201`" = "1"],
[memory_linux_ptmalloc2_happy=no
AC_MSG_WARN([*** Detected Intel C compiler v9.0 <= 20051201 on ia64])
AC_MSG_WARN([*** This compiler/platform combination has known problems with ptmalloc2])
AC_MSG_WARN([*** Disabling ptmalloc2])])])
;;
esac])
AS_IF([test "$memory_linux_ptmalloc2_happy" = yes],
[# check for malloc.h
AC_CHECK_HEADER([malloc.h],
[memory_linux_ptmalloc2_happy=yes],
[memory_linux_ptmalloc2_happy=no])])
AS_IF([test "$memory_linux_ptmalloc2_happy" = yes],
[# check for init hook symbol
AC_CHECK_DECL([__malloc_initialize_hook],
[memory_linux_ptmalloc2_happy=yes],
[memory_linux_ptmalloc2_happy=no],
[AC_INCLUDES_DEFAULT
#include <malloc.h>])])
#
# See if we have sbrk prototyped
#
AS_IF([test "$memory_linux_ptmalloc2_happy" = yes],
[AC_CHECK_DECLS([sbrk])])
#
# Figure out how we're going to call mmap/munmap for real
#
AS_IF([test "$memory_linux_ptmalloc2_happy" = yes],
[memory_linux_mmap=0
memory_linux_munmap=1
# it's nearly impossible to call mmap from syscall(), so
# only go this route if we can't get at munmap any other
# way.
AC_CHECK_HEADER([syscall.h],
[AC_CHECK_FUNCS([syscall], [], [memory_linux_munmap=0])])
# Always look for __munmap and __mmap
AC_CHECK_FUNCS([__munmap], [memory_linux_mmap=1])
AC_CHECK_FUNCS([__mmap])
# only allow dlsym (and therefore add -ldl) if we
# really need to
AS_IF([test "$memory_linux_mmap" = "0"],
[memory_linux_LIBS_SAVE="$LIBS"
AC_CHECK_LIB([dl],
[dlsym],
[LIBS="$LIBS -ldl"
memory_linux_LIBS="-ldl"
memory_linux_mmap=1])
AC_CHECK_FUNCS([dlsym])
LIBS="$memory_linux_LIBS_SAVE"])
AS_IF([test "$memory_linux_mmap" = "0" -a "$memory_linux_munmap" = "0"],
[memory_linux_ptmalloc2_happy=no])])
# If all is good, save the extra libs for the wrapper
AS_IF([test "$memory_linux_ptmalloc2_happy" = yes],
[memory_linux_WRAPPER_EXTRA_LIBS="$memory_linux_LIBS"
value=1],
[value=0])
AC_DEFINE_UNQUOTED([MEMORY_LINUX_PTMALLOC2], [$value],
[Whether ptmalloc2 is supported on this system or not])
AM_CONDITIONAL([MEMORY_LINUX_PTMALLOC2],
[test "$memory_linux_ptmalloc2_happy" = yes])
######################################################################
# ummunotify
######################################################################
# Check for the relevant header
AS_IF([test "$memory_linux_ummu_happy" = yes],
[# check for linux/ummunotify.h
AC_CHECK_HEADER([linux/ummunotify.h],
[memory_linux_ummu_happy=yes],
[memory_linux_ummu_happy=no])])
# <stropts.h> has the Linux declaration for ioctl
AC_CHECK_HEADERS([stropts.h])
# If all is good, set the header file that we want the rest of the
# code base to use
AS_IF([test "$memory_linux_ummu_happy" = yes],
[memory_base_include="linux/public.h"
value=1],
[value=0])
AC_DEFINE_UNQUOTED([MEMORY_LINUX_UMMUNOTIFY], [$value],
[Whether ummunotify is supported on this system or not])
AM_CONDITIONAL([MEMORY_LINUX_UMMUNOTIFY],
[test "$memory_linux_ummu_happy" = yes])
######################################################################
# post processing
######################################################################
AS_IF([test "$memory_malloc_hooks_requested" = 1 -a \
"$memory_linux_ptmalloc2_happy" = no -a \
"$memory_linux_ummu_happy" = no],
[AC_MSG_ERROR([linux memory management requested but neither ptmalloc2 nor ummunotify are available. Aborting.])])
AC_SUBST([memory_linux_LIBS])
AS_IF([test "$memory_linux_ptmalloc2_happy" = yes -o \
"$memory_linux_ummu_happy" = yes],
[memory_base_found=1
$1],
[memory_base_found=0
memory_base_include=
$2])
OMPI_VAR_SCOPE_POP
])

Просмотреть файл

@ -19,5 +19,5 @@
# Specific to this module # Specific to this module
PARAM_CONFIG_PRIORITY=20 PARAM_CONFIG_PRIORITY=40
PARAM_CONFIG_FILES="Makefile" PARAM_CONFIG_FILES="Makefile"

Просмотреть файл

@ -0,0 +1,42 @@
# -*- text -*-
#
# Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English help file for Open MPI's memory/linux component.
#
[disable incorrectly set]
WARNING: The special MCA parameter "%s" was set in
an unexpected way, and is likely not working the way you want it to.
Specifically, this MCA parameter is "special" in that it can *only* be
set in the environment. Setting this value in a file -- and sometimes
even on the command line -- will not work as intended. The *only* way
to set this value is to set "OMPI_MCA_%s" in the environment before
starting your job.
Value: %d
Source: %s
#
[ummunotify eaccess]
Open MPI was unable to open the UMMU notification device. This is
likely a permissions problem on the device itself. UMMU notification
support is therefore disabled in this process; an alternate memory
hook manager *may* be used instead (if available).
Local host: %s
UMMU device: %s
#
[ummunotify open error]
Open MPI was unable to open the UMMU notification device. UMMU
notification support is therefore disabled in this process; an
alternate memory hook manager *may* be used instead (if available).
Local host: %s
UMMU device: %s
Error: %s (%d)

Просмотреть файл

@ -1,3 +1,8 @@
/*
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
*
* Additional copyrights may follow.
*/
/* Malloc implementation for multiple threads without lock contention. /* Malloc implementation for multiple threads without lock contention.
Copyright (C) 2001, 2002, 2003 Free Software Foundation, Inc. Copyright (C) 2001, 2002, 2003 Free Software Foundation, Inc.
This file is part of the GNU C Library. This file is part of the GNU C Library.
@ -28,7 +33,7 @@
#include "opal/util/show_help.h" #include "opal/util/show_help.h"
#include "opal/constants.h" #include "opal/constants.h"
extern opal_memory_base_component_2_0_0_t mca_memory_ptmalloc2_component; #include "opal/mca/memory/linux/memory_linux.h"
#ifndef DEFAULT_CHECK_ACTION #ifndef DEFAULT_CHECK_ACTION
#define DEFAULT_CHECK_ACTION 1 #define DEFAULT_CHECK_ACTION 1
@ -671,26 +676,26 @@ public_sET_STATe(Void_t* msptr)
So we can basically have some hard-coded tests for things to see if So we can basically have some hard-coded tests for things to see if
we want to setup to use our internal ptmalloc2 or not. */ we want to setup to use our internal ptmalloc2 or not. */
static void *opal_memory_ptmalloc2_malloc_hook(size_t sz, static void *opal_memory_linux_malloc_hook(size_t sz,
const __malloc_ptr_t caller) const __malloc_ptr_t caller)
{ {
return public_mALLOc(sz); return public_mALLOc(sz);
} }
static void *opal_memory_ptmalloc2_realloc_hook(Void_t* ptr, size_t sz, static void *opal_memory_linux_realloc_hook(Void_t* ptr, size_t sz,
const __malloc_ptr_t caller) const __malloc_ptr_t caller)
{ {
return public_rEALLOc(ptr, sz); return public_rEALLOc(ptr, sz);
} }
static void *opal_memory_ptmalloc2_memalign_hook(size_t alignment, size_t sz, static void *opal_memory_linux_memalign_hook(size_t alignment, size_t sz,
const __malloc_ptr_t caller) const __malloc_ptr_t caller)
{ {
return public_mEMALIGn(alignment, sz); return public_mEMALIGn(alignment, sz);
} }
static void opal_memory_ptmalloc2_free_hook(__malloc_ptr_t __ptr, static void opal_memory_linux_free_hook(__malloc_ptr_t __ptr,
const __malloc_ptr_t caller) const __malloc_ptr_t caller)
{ {
public_fREe(__ptr); public_fREe(__ptr);
} }
@ -722,7 +727,7 @@ static check_result_t check(const char *name)
} }
/* OMPI's init function */ /* OMPI's init function */
static void opal_memory_ptmalloc2_malloc_init_hook(void) static void opal_memory_linux_malloc_init_hook(void)
{ {
/* Yes, checking for an MPI MCA parameter here is an abstraction /* Yes, checking for an MPI MCA parameter here is an abstraction
violation. Cope. Yes, even checking for *any* MCA parameter violation. Cope. Yes, even checking for *any* MCA parameter
@ -742,7 +747,7 @@ static void opal_memory_ptmalloc2_malloc_init_hook(void)
causing Badness (see http://bugs.debian.org/531522). causing Badness (see http://bugs.debian.org/531522).
$FAKEROOTKEY is set by Debian's "fakeroot" build environment; $FAKEROOTKEY is set by Debian's "fakeroot" build environment;
check for that explicitly. */ check for that explicitly. */
r1 = check("OMPI_MCA_memory_ptmalloc2_disable"); r1 = check("OMPI_MCA_memory_linux_disable");
r2 = check("FAKEROOTKEY"); r2 = check("FAKEROOTKEY");
if ((RESULT_NOT_FOUND != r1 && RESULT_NO != r1) || if ((RESULT_NOT_FOUND != r1 && RESULT_NO != r1) ||
(RESULT_NOT_FOUND != r2 && RESULT_NO != r2)) { (RESULT_NOT_FOUND != r2 && RESULT_NO != r2)) {
@ -798,10 +803,10 @@ static void opal_memory_ptmalloc2_malloc_init_hook(void)
ptmalloc_init(); ptmalloc_init();
/* Now set the hooks to point to our functions */ /* Now set the hooks to point to our functions */
__free_hook = opal_memory_ptmalloc2_free_hook; __free_hook = opal_memory_linux_free_hook;
__malloc_hook = opal_memory_ptmalloc2_malloc_hook; __malloc_hook = opal_memory_linux_malloc_hook;
__memalign_hook = opal_memory_ptmalloc2_memalign_hook; __memalign_hook = opal_memory_linux_memalign_hook;
__realloc_hook = opal_memory_ptmalloc2_realloc_hook; __realloc_hook = opal_memory_linux_realloc_hook;
} }
} }
@ -814,10 +819,10 @@ static void opal_memory_ptmalloc2_malloc_init_hook(void)
libopen-pal). This declaration is not in malloc.h because this libopen-pal). This declaration is not in malloc.h because this
function only exists as a horrid workaround to force linkers to function only exists as a horrid workaround to force linkers to
pull in this .o file (see explanation below). */ pull in this .o file (see explanation below). */
void *opal_memory_ptmalloc2_hook_pull(void); void opal_memory_linux_hook_pull(bool *want_hooks);
/* OMPI change: add a dummy function here that will be called by the /* OMPI change: add a dummy function here that will be called by the
ptmalloc2 component open() function. This dummy function is linux component open() function. This dummy function is
necessary for when OMPI is built as --disable-shared necessary for when OMPI is built as --disable-shared
--enable-static --disable-dlopen, because we won't use --enable-static --disable-dlopen, because we won't use
-Wl,--export-dynamic when building OMPI. So we need to ensure that -Wl,--export-dynamic when building OMPI. So we need to ensure that
@ -825,8 +830,7 @@ void *opal_memory_ptmalloc2_hook_pull(void);
but they also end up in the final exectuable (so that but they also end up in the final exectuable (so that
__malloc_initialize_hook is there, overrides the weak symbol in __malloc_initialize_hook is there, overrides the weak symbol in
glibc, ....etc.). */ glibc, ....etc.). */
static int bogus = 37; void opal_memory_linux_hook_pull(bool *want_hooks)
void *opal_memory_ptmalloc2_hook_pull(void)
{ {
int val; int val;
@ -840,24 +844,26 @@ void *opal_memory_ptmalloc2_hook_pull(void)
_malloc_init_hook(). */ _malloc_init_hook(). */
mca_base_param_source_t source; mca_base_param_source_t source;
char *file; char *file;
int p = mca_base_param_reg_int(&mca_memory_ptmalloc2_component.memoryc_version, int p = mca_base_param_reg_int(&mca_memory_linux_component.super.memoryc_version,
"disable", "disable",
"If this MCA parameter is set to 1 **VIA ENVIRONMENT VARIABLE ONLY*** (this MCA parameter *CANNOT* be set in a file or on the mpirun command line!), the ptmalloc2 hooks will be disabled", "If this MCA parameter is set to 1 **VIA ENVIRONMENT VARIABLE ONLY*** (this MCA parameter *CANNOT* be set in a file or on the mpirun command line!), this component will be disabled and will not attempt to use either ummunotify or memory hook support",
false, false, 0, &val); false, false, 0, &val);
/* We can at least warn if someone tried to set this in a file */
if (p >= 0 &&
OPAL_SUCCESS == mca_base_param_lookup_source(p, &source, &file) &&
(MCA_BASE_PARAM_SOURCE_DEFAULT != source &&
MCA_BASE_PARAM_SOURCE_ENV != source)) {
opal_show_help("help-opal-memory-ptmalloc2.txt",
"disable incorrectly set", true,
"opal_ptmalloc2_disable",
"opal_ptmalloc2_disable", val,
MCA_BASE_PARAM_SOURCE_FILE == source ?
file : "override");
}
return &bogus; /* We can at least warn if someone tried to set this in a file */
if (p >= 0) {
if (OPAL_SUCCESS == mca_base_param_lookup_source(p, &source, &file) &&
(MCA_BASE_PARAM_SOURCE_DEFAULT != source &&
MCA_BASE_PARAM_SOURCE_ENV != source)) {
opal_show_help("help-opal-memory-linux.txt",
"disable incorrectly set", true,
"opal_linux_disable",
"opal_linux_disable", val,
MCA_BASE_PARAM_SOURCE_FILE == source ?
file : "override");
} else {
*want_hooks = OPAL_INT_TO_BOOL(!val);
}
}
} }
@ -865,7 +871,7 @@ void *opal_memory_ptmalloc2_hook_pull(void)
/* OMPI change: This is the symbol to override to make the above /* OMPI change: This is the symbol to override to make the above
function get fired during malloc initialization time. */ function get fired during malloc initialization time. */
void (*__malloc_initialize_hook) (void) = void (*__malloc_initialize_hook) (void) =
opal_memory_ptmalloc2_malloc_init_hook; opal_memory_linux_malloc_init_hook;
/* /*
* Local variables: * Local variables:

Просмотреть файл

Просмотреть файл

@ -1,3 +1,8 @@
/*
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
*
* Additional copyrights may follow.
*/
/* Malloc implementation for multiple threads; statistics printing. /* Malloc implementation for multiple threads; statistics printing.
Copyright (C) 2004 Free Software Foundation, Inc. Copyright (C) 2004 Free Software Foundation, Inc.
This file is part of the GNU C Library. This file is part of the GNU C Library.
@ -21,7 +26,7 @@
/* $Id: $ */ /* $Id: $ */
/* OMPI change: Name-shift all the internal symbols */ /* OMPI change: Name-shift all the internal symbols */
#include "opal/mca/memory/ptmalloc2/rename.h" #include "opal/mca/memory/linux/rename.h"
#include <stdio.h> /* needed for malloc_stats */ #include <stdio.h> /* needed for malloc_stats */

Просмотреть файл

@ -1,7 +1,9 @@
/********************** BEGIN OMPI CHANGES *****************************/ /********************** BEGIN OMPI CHANGES *****************************/
/* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. /*
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
*
* Additional copyrights may follow.
*/ */
#define OPAL_DISABLE_ENABLE_MEM_DEBUG 1 #define OPAL_DISABLE_ENABLE_MEM_DEBUG 1
#include "opal_config.h" #include "opal_config.h"
@ -10,8 +12,9 @@
#include "opal/sys/atomic.h" #include "opal/sys/atomic.h"
#include "opal/memoryhooks/memory_internal.h" #include "opal/memoryhooks/memory_internal.h"
#include "opal/mca/memory/linux/memory_linux.h"
/* Name-shift all the internal symbols */ /* Name-shift all the internal symbols */
#include "opal/mca/memory/ptmalloc2/rename.h" #include "opal/mca/memory/linux/rename.h"
/* /*
* Not all systems have sbrk() declared, since it's technically not a * Not all systems have sbrk() declared, since it's technically not a
@ -22,25 +25,22 @@ void *sbrk();
#endif #endif
static void* static void *opal_memory_linux_free_ptmalloc2_sbrk(int inc)
opal_mem_free_ptmalloc2_sbrk(int inc)
{ {
if (inc < 0) { if (inc < 0) {
long oldp = (long) sbrk(0); long oldp = (long) sbrk(0);
opal_mem_hooks_release_hook((void*) (oldp + inc), -inc, 1); opal_mem_hooks_release_hook((void*) (oldp + inc), -inc, 1);
} }
return sbrk(inc); return sbrk(inc);
} }
extern int opal_mem_free_ptmalloc2_munmap(void *start, size_t length, int from_alloc, int run_hooks);
/* if we are trying to catch only allocations from and releases to the /* if we are trying to catch only allocations from and releases to the
operating system, intercept sbrk, mmap, and munmap. If we want to operating system, intercept sbrk, mmap, and munmap. If we want to
intercept every call to malloc/realloc/free/etc., don't do this, as intercept every call to malloc/realloc/free/etc., don't do this, as
we need to add something into each of those calls anyway. */ we need to add something into each of those calls anyway. */
#define MORECORE opal_mem_free_ptmalloc2_sbrk #define MORECORE opal_memory_linux_free_ptmalloc2_sbrk
#define munmap(a,b) opal_mem_free_ptmalloc2_munmap(a,b,1,1) #define munmap(a,b) opal_memory_linux_free_ptmalloc2_munmap(a,b,1)
/* make some non-GCC compilers happy */ /* make some non-GCC compilers happy */
#ifndef __GNUC__ #ifndef __GNUC__
@ -3419,15 +3419,12 @@ public_mALLOc(size_t bytes)
#endif #endif
/* OMPI change: put in a flag so that we can know that this function /* OMPI change: put in a flag so that we can know that this function
was invoked. This flag is checked in the memory/ptmalloc2 was invoked. This flag is checked in the memory/linux component
component init to ensure that this ptmalloc is actually being init to ensure that this ptmalloc is actually being used. Used a
used. Used a simple "extern" here to get the flag symbol rather simple "extern" here to get the flag symbol rather than putting
than putting it in a new .h file that would only contain a small it in a new .h file that would only contain a small number of
number of symbols. */ symbols. */
{ mca_memory_linux_component.malloc_invoked = true;
extern bool opal_memory_ptmalloc2_malloc_invoked;
opal_memory_ptmalloc2_malloc_invoked = true;
}
arena_get(ar_ptr, bytes); arena_get(ar_ptr, bytes);
if(!ar_ptr) if(!ar_ptr)
@ -3479,15 +3476,12 @@ public_fREe(Void_t* mem)
#endif #endif
/* OMPI change: put in a flag so that we can know that this function /* OMPI change: put in a flag so that we can know that this function
was invoked. This flag is checked in the memory/ptmalloc2 was invoked. This flag is checked in the memory/linux component
component init to ensure that this ptmalloc is actually being init to ensure that this ptmalloc is actually being used. Used a
used. Used a simple "extern" here to get the flag symbol rather simple "extern" here to get the flag symbol rather than putting
than putting it in a new .h file that would only contain a small it in a new .h file that would only contain a small number of
number of symbols. */ symbols. */
{ mca_memory_linux_component.free_invoked = true;
extern bool opal_memory_ptmalloc2_free_invoked;
opal_memory_ptmalloc2_free_invoked = true;
}
if (mem == 0) /* free(0) has no effect */ if (mem == 0) /* free(0) has no effect */
return; return;
@ -3541,15 +3535,12 @@ public_rEALLOc(Void_t* oldmem, size_t bytes)
#endif #endif
/* OMPI change: put in a flag so that we can know that this function /* OMPI change: put in a flag so that we can know that this function
was invoked. This flag is checked in the memory/ptmalloc2 was invoked. This flag is checked in the memory/linux component
component init to ensure that this ptmalloc is actually being init to ensure that this ptmalloc is actually being used. Used a
used. Used a simple "extern" here to get the flag symbol rather simple "extern" here to get the flag symbol rather than putting
than putting it in a new .h file that would only contain a small it in a new .h file that would only contain a small number of
number of symbols. */ symbols. */
{ mca_memory_linux_component.realloc_invoked = true;
extern bool opal_memory_ptmalloc2_realloc_invoked;
opal_memory_ptmalloc2_realloc_invoked = true;
}
#if REALLOC_ZERO_BYTES_FREES #if REALLOC_ZERO_BYTES_FREES
if (bytes == 0 && oldmem != NULL) { public_fREe(oldmem); return 0; } if (bytes == 0 && oldmem != NULL) { public_fREe(oldmem); return 0; }
@ -3631,15 +3622,12 @@ public_mEMALIGn(size_t alignment, size_t bytes)
#endif #endif
/* OMPI change: put in a flag so that we can know that this function /* OMPI change: put in a flag so that we can know that this function
was invoked. This flag is checked in the memory/ptmalloc2 was invoked. This flag is checked in the memory/linux component
component init to ensure that this ptmalloc is actually being init to ensure that this ptmalloc is actually being used. Used a
used. Used a simple "extern" here to get the flag symbol rather simple "extern" here to get the flag symbol rather than putting
than putting it in a new .h file that would only contain a small it in a new .h file that would only contain a small number of
number of symbols. */ symbols. */
{ mca_memory_linux_component.memalign_invoked = true;
extern bool opal_memory_ptmalloc2_memalign_invoked;
opal_memory_ptmalloc2_memalign_invoked = true;
}
/* If need less alignment than we give anyway, just relay to malloc */ /* If need less alignment than we give anyway, just relay to malloc */
if (alignment <= MALLOC_ALIGNMENT) return public_mALLOc(bytes); if (alignment <= MALLOC_ALIGNMENT) return public_mALLOc(bytes);

Просмотреть файл

67
opal/mca/memory/linux/memory_linux.h Обычный файл
Просмотреть файл

@ -0,0 +1,67 @@
/*
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_MEMORY_LINUX_H
#define OPAL_MEMORY_LINUX_H
#include "opal_config.h"
#include "opal/mca/memory/memory.h"
BEGIN_C_DECLS
/* Component structure */
typedef struct opal_memory_linux_component_t {
opal_memory_base_component_2_0_0_t super;
/* Component data */
int verbose_level;
int enable_ummunotify;
int enable_ptmalloc2;
#if MEMORY_LINUX_UMMUNOTIFY
/* Ummunotify-specific data */
int ummunotify_fd;
#endif
#if MEMORY_LINUX_PTMALLOC2
/* Ptmalloc2-specific data */
bool free_invoked;
bool malloc_invoked;
bool realloc_invoked;
bool memalign_invoked;
bool munmap_invoked;
#endif
} opal_memory_linux_component_t;
/* memory_linux_component.c */
extern opal_memory_linux_component_t mca_memory_linux_component;
#if MEMORY_LINUX_UMMUNOTIFY
/* memory_linux_ummunotify.c */
int opal_memory_linux_ummunotify_open(void);
int opal_memory_linux_ummunotify_close(void);
#endif /* MEMORY_LINUX_UMMUNOTIFY */
#if MEMORY_LINUX_PTMALLOC2
/* memory_linux_ptmalloc2.c */
int opal_memory_linux_ptmalloc2_open(void);
int opal_memory_linux_ptmalloc2_close(void);
/* memory_linux_munmap.c */
OPAL_DECLSPEC int opal_memory_linux_free_ptmalloc2_munmap(void *start, size_t length, int from_alloc);
OPAL_DECLSPEC int munmap(void* addr, size_t len);
#endif /* !MEMORY_LINUX_PTMALLOC2 */
END_C_DECLS
#endif

Просмотреть файл

@ -0,0 +1,210 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/* This component basically fronts two different memory management
schemes: the Linux "ummunotify" kernel module and hooking in a
substitute ptmalloc2 allocator. Both of these mechanisms are
unified under a single component because the "memory" framework
both only allows one component to be selected, and that one
component must be compile-time linked into libopen-pal. Hence, if
we want to try to use either one of these mechanisms, we have to
have them both in a single component.
When using ptmalloc2, the goal of this component is to wholly
replace the underlying allocator with our internal ptmalloc2
allocator. See the file README-open-mpi.txt for details of how it
works.
When using ummunotify, we can probe to find out when the MMU map
has been changed (i.e., memory has been released back to the OS). */
#include "opal_config.h"
#include "opal/constants.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/mca/memory/memory.h"
#include "opal/mca/memory/base/empty.h"
#include "opal/memoryhooks/memory.h"
#include "opal/mca/memory/linux/memory_linux.h"
#undef opal_memory_changed
#include "opal/mca/memory/linux/public.h"
static int linux_open(void);
static int linux_close(void);
static int linux_register(void);
#if MEMORY_LINUX_UMMUNOTIFY
static bool ummunotify_opened = false;
#endif
#if MEMORY_LINUX_PTMALLOC2
static bool ptmalloc2_opened = false;
#endif
opal_memory_linux_component_t mca_memory_linux_component = {
/* First, the opal_memory_base_component_2_0_0_t */
{
/* First, the mca_component_t struct containing meta
information about the component itself */
{
OPAL_MEMORY_BASE_VERSION_2_0_0,
/* Component name and version */
"linux",
OPAL_MAJOR_VERSION,
OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
/* Component open and close functions */
linux_open,
linux_close,
NULL,
linux_register,
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
},
/* Memory framework functions. These function pointer values
are replaced by memory_linux_ummunotify.c at run time if we
end up using ummunotify support. */
NULL,
opal_memory_base_component_register_empty,
opal_memory_base_component_deregister_empty
},
/* Component-specific data, filled in later (compiler will 0/NULL
it out) */
};
/*
* Register MCA params
*/
static int linux_register(void)
{
/* Information only */
mca_base_param_reg_int(&mca_memory_linux_component.super.memoryc_version,
"ptmalloc2_available",
"Whether ptmalloc2 support is included in Open MPI or not (1 = yes, 0 = no)",
false, true, MEMORY_LINUX_PTMALLOC2, NULL);
mca_base_param_reg_int(&mca_memory_linux_component.super.memoryc_version,
"ummunotify_available",
"Whether ummunotify support is included in Open MPI or not (1 = yes, 0 = no)",
false, true, MEMORY_LINUX_UMMUNOTIFY, NULL);
/* Allow user to manually enable/disable */
mca_base_param_reg_int(&mca_memory_linux_component.super.memoryc_version,
"ptmalloc2_enable",
"Whether to enable ptmalloc2 support or not (negative = try to enable, but continue even if support is not available, 0 = do not enable support, positive = try to enable and fail if support is not available)",
false, false, -1,
&mca_memory_linux_component.enable_ptmalloc2);
mca_base_param_reg_int(&mca_memory_linux_component.super.memoryc_version,
"ummunotify_enable",
"Whether to enable ummunotify support or not (negative = try to enable, but continue even if support is not available, 0 = do not enable support, positive = try to enable and fail if support is not available)",
false, false, -1,
&mca_memory_linux_component.enable_ummunotify);
return OPAL_SUCCESS;
}
static int linux_open(void)
{
int i, v;
i = mca_base_param_find("memory", NULL, "base_verbose");
mca_base_param_lookup_int(i, &v);
mca_memory_linux_component.verbose_level = v;
/* Try initializing ummunotify first; if that fails, try
ptmalloc2. */
#if MEMORY_LINUX_UMMUNOTIFY
if (mca_memory_linux_component.enable_ummunotify) {
if (v >= 10) {
opal_output(0, "memory:linux: attempting to initialize ummunotify support");
}
if (OPAL_SUCCESS == opal_memory_linux_ummunotify_open()) {
ummunotify_opened = true;
if (v >= 10) {
opal_output(0, "memory:linux: ummunotify successfully initialized; we'll use that");
}
return OPAL_SUCCESS;
}
if (v >= 10) {
opal_output(0, "memory:linux: ummunotify failed to initialize");
}
}
#endif
#if MEMORY_LINUX_PTMALLOC2
if (mca_memory_linux_component.enable_ptmalloc2) {
if (v >= 10) {
opal_output(0, "memory:linux: attempting to initialize ptmalloc2 support");
}
if (OPAL_SUCCESS == opal_memory_linux_ptmalloc2_open()) {
ptmalloc2_opened = true;
if (v >= 10) {
opal_output(0, "memory:linux: ptmalloc2 successfully initialized; we'll use that");
}
return OPAL_SUCCESS;
}
if (v >= 10) {
opal_output(0, "memory:linux: ummunotify failed to initialize");
}
}
#endif
/* We can return OPAL_ERR_NOT_AVAILABLE if nothing is
available; that will make the MCA base silently disregard this
component. */
if (v >= 10) {
opal_output(0, "memory:linux: no memory hooks available in this process");
}
return OPAL_ERR_NOT_AVAILABLE;
}
static int linux_close(void)
{
int v = mca_memory_linux_component.verbose_level;
#if MEMORY_LINUX_UMMUNOTIFY
if (ummunotify_opened) {
if (v >= 10) {
opal_output(0, "memory:linux: shutting down ummunotify support");
}
opal_memory_linux_ummunotify_close();
ummunotify_opened = false;
}
#endif
#if MEMORY_LINUX_PTMALLOC2
if (ptmalloc2_opened) {
if (v >= 10) {
opal_output(0, "memory:linux: shutting down ptmalloc2 support");
}
opal_memory_linux_ptmalloc2_close();
ptmalloc2_opened = false;
}
#endif
return OPAL_SUCCESS;
}

Просмотреть файл

@ -37,7 +37,7 @@
#include "opal/memoryhooks/memory_internal.h" #include "opal/memoryhooks/memory_internal.h"
#include "opal_ptmalloc2_munmap.h" #include "memory_linux.h"
/* /*
* munmap is always intercepted * munmap is always intercepted
@ -48,10 +48,9 @@ int __munmap(void* addr, size_t len);
/* intercept munmap, as the user can give back memory that way as well. */ /* intercept munmap, as the user can give back memory that way as well. */
OPAL_DECLSPEC int OPAL_DECLSPEC int munmap(void* addr, size_t len)
munmap(void* addr, size_t len)
{ {
return opal_mem_free_ptmalloc2_munmap(addr, len, 0, 1); return opal_memory_linux_free_ptmalloc2_munmap(addr, len, 0);
} }
@ -59,44 +58,34 @@ munmap(void* addr, size_t len)
that we can intercept both munmap and __munmap. If that isn't that we can intercept both munmap and __munmap. If that isn't
possible, try calling __munmap from munmap and let __munmap go. If possible, try calling __munmap from munmap and let __munmap go. If
that doesn't work, try dlsym */ that doesn't work, try dlsym */
int int opal_memory_linux_free_ptmalloc2_munmap(void *start, size_t length,
opal_mem_free_ptmalloc2_munmap(void *start, size_t length, int from_alloc, int from_alloc)
int call_hooks)
{ {
{ #if !defined(HAVE___MUNMAP) && \
extern bool opal_memory_ptmalloc2_munmap_invoked; !(defined(HAVE_SYSCALL) && defined(__NR_munmap)) && defined(HAVE_DLSYM)
opal_memory_ptmalloc2_munmap_invoked = true; static int (*realmunmap)(void*, size_t);
} #endif
if (call_hooks) { mca_memory_linux_component.munmap_invoked = true;
opal_mem_hooks_release_hook(start, length, from_alloc);
} opal_mem_hooks_release_hook(start, length, from_alloc);
#if defined(HAVE___MUNMAP) #if defined(HAVE___MUNMAP)
return __munmap(start, length); return __munmap(start, length);
#elif defined(HAVE_SYSCALL) && defined(__NR_munmap) #elif defined(HAVE_SYSCALL) && defined(__NR_munmap)
return syscall(__NR_munmap, start, length); return syscall(__NR_munmap, start, length);
#elif defined(HAVE_DLSYM) #elif defined(HAVE_DLSYM)
{ if (NULL == realmunmap) {
/* Must use a typedef here because we need volatile to be an union {
attribute of the variable, not the function (which would be int (*munmap_fp)(void*, size_t);
meaningless, anyway). */ void *munmap_p;
typedef int (*munmap_fn_t)(void*, size_t); } tmp;
static volatile munmap_fn_t realmunmap = NULL;
if (NULL == realmunmap) { tmp.munmap_p = dlsym(RTLD_NEXT, "munmap");
union { realmunmap = tmp.munmap_fp;
int (*munmap_fp)(void*, size_t);
void *munmap_p;
} tmp;
tmp.munmap_p = dlsym(RTLD_NEXT, "munmap");
realmunmap = tmp.munmap_fp;
++count;
}
return realmunmap(start, length);
} }
return realmunmap(start, length);
#else #else
#error "Can not determine how to call munmap" #error "Can not determine how to call munmap"
#endif #endif

Просмотреть файл

@ -0,0 +1,136 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <sys/mman.h>
#include <stdlib.h>
#include <malloc.h>
#include "opal/constants.h"
#include "opal/util/output.h"
#include "opal/memoryhooks/memory.h"
#include "opal/memoryhooks/memory_internal.h"
#include "opal/mca/memory/linux/memory_linux.h"
/* Need to call a function in hooks.c to ensure that all those symbols
get pulled in at link time (e.g., when building libmpi.a, so that
those symbols end up in the final executable -- especially if we
use --disable-dlopen and therefore -Wl,--export-dynamic isn't used
when we build OMPI). */
extern void opal_memory_linux_hook_pull(bool *want_hooks);
/*
* Try to initialize ptmalloc2
*/
int opal_memory_linux_ptmalloc2_open(void)
{
int val = 0;
void *p;
bool want_hooks = true;
/* Call a [somewhat] dummy function in hooks.c. ***Do not remove
this call!*** See comment at the beginning of this file
explaining why it is here. It will also check to see if an
environment variable has been set to disable this component
(note that OPAL_ERR_NOT_AVAILABLE is a special return value
that will silently fail the open component call; all others
will issue an error). */
opal_memory_linux_hook_pull(&want_hooks);
if (!want_hooks) {
return OPAL_ERR_NOT_AVAILABLE;
}
/* We will also provide malloc/free support if we've been
activated. We don't exclusively rely on the
__malloc_initialize_hook() previously being called because it's
possible that our hook was called, but then someone else reset
the hooks to point to something else (i.e., before MPI_INIT).
So explicitly test here if our hooks are still in place. If
they are, then enable FREE|CHUNK_SUPPORT. If not, then don't
enable that support -- just leave it at MUNMAP_SUPPORT.
(Look in hooks.c for the __malloc_initialize_hook setup) */
/* Do a simple set of tests to see if our hooks are still the ones
installed. Explicitly reset the flags indicating that our
functions were invoked */
p = malloc(1024 * 1024 * 4);
if (NULL == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
p = realloc(p, 1024 * 1024 * 4 + 32);
if (NULL == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
free(p);
p = memalign(4, 1024 * 1024);
if (NULL == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
free(p);
#if HAVE_POSIX_MEMALIGN
/* Double check for posix_memalign, too */
if (mca_memory_linux_component.memalign_invoked) {
mca_memory_linux_component.memalign_invoked = false;
if (0 != posix_memalign(&p, sizeof(void*), 1024 * 1024)) {
return OPAL_ERR_IN_ERRNO;
}
free(p);
}
#endif
if (mca_memory_linux_component.malloc_invoked &&
mca_memory_linux_component.realloc_invoked &&
mca_memory_linux_component.memalign_invoked &&
mca_memory_linux_component.free_invoked) {
/* Happiness; our functions were invoked */
val |= OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_CHUNK_SUPPORT;
}
/* Check if our mmap layering is working */
p = mmap(NULL, 4096, PROT_READ, (MAP_ANONYMOUS | MAP_PRIVATE), -1, 0);
if (MAP_FAILED == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
munmap(p, 4096);
if (mca_memory_linux_component.munmap_invoked) {
val |= OPAL_MEMORY_MUNMAP_SUPPORT;
}
/* All done */
if (val > 0) {
opal_mem_hooks_set_support(val);
return OPAL_SUCCESS;
}
return OPAL_ERR_NOT_AVAILABLE;
}
int opal_memory_linux_ptmalloc2_close(void)
{
/* Nothing to do, really. This function exists just for
symmetry. */
return OPAL_SUCCESS;
}

Просмотреть файл

@ -0,0 +1,234 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#ifdef HAVE_STROPTS_H
#include <stropts.h>
#endif
#include <errno.h>
#include <string.h>
#include <linux/ummunotify.h>
#include "opal_stdint.h"
#include "opal/constants.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "opal/mca/memory/memory.h"
#include "opal/memoryhooks/memory.h"
#include "opal/memoryhooks/memory_internal.h"
#include "opal/mca/memory/linux/memory_linux.h"
#include "opal/mca/memory/linux/public.h"
#define DEV_UMMUNOTIFY "/dev/ummunotify"
/*
* Local functions
*/
static int ummunotify_process(void);
static int ummunotify_register(void *start, size_t len, uint64_t cookie);
static int ummunotify_deregister(void *start, size_t len, uint64_t cookie);
/*
* Local variables
*/
static bool initialized = false;
/*
* Global variables (these need to be global variables rather than in
* the component struct because they are accessed in the
* opal_memory_changed() macro defined in public.h, and we don't want
* to have to include the component structure definition in public.h).
*/
uint64_t opal_memory_linux_ummunotify_counter_last_value = 0;
volatile uint64_t *opal_memory_linux_ummunotify_counter =
&opal_memory_linux_ummunotify_counter_last_value;
int opal_memory_linux_ummunotify_open(void)
{
uint64_t *p;
/* Just to be safe... */
opal_memory_linux_ummunotify_counter_last_value = 0;
opal_memory_linux_ummunotify_counter =
&opal_memory_linux_ummunotify_counter_last_value;
/* Open the device. Try to give a meaningful error message if
we're unable to open it. */
mca_memory_linux_component.ummunotify_fd =
open(DEV_UMMUNOTIFY, O_RDONLY | O_NONBLOCK);
if (mca_memory_linux_component.ummunotify_fd < 0) {
char hostname[HOST_NAME_MAX];
gethostname(hostname, sizeof(hostname));
if (EACCES == errno) {
/* This will get a proper show_help when merged into the
linux component */
opal_show_help("help-opal-memory-linux.txt",
"ummunotify eaccess", true,
hostname, DEV_UMMUNOTIFY);
} else if (ENOENT != errno) {
/* Don't print an error if DEV_UMMUNOTIFY simply doesn't exist */
opal_show_help("help-opal-memory-linux.txt",
"ummunotify open error", true,
hostname, DEV_UMMUNOTIFY,
strerror(errno), errno);
}
return OPAL_ERR_NOT_SUPPORTED;
}
p = mmap(NULL, sizeof(*opal_memory_linux_ummunotify_counter),
PROT_READ, MAP_SHARED,
mca_memory_linux_component.ummunotify_fd, 0);
if (MAP_FAILED == opal_memory_linux_ummunotify_counter) {
close(mca_memory_linux_component.ummunotify_fd);
mca_memory_linux_component.ummunotify_fd = -1;
return OPAL_ERR_NOT_SUPPORTED;
}
opal_memory_linux_ummunotify_counter = p;
/* If everything went well, tell OMPI that we have full support
for the memory hooks and fill in the component function
pointers */
opal_mem_hooks_set_support(OPAL_MEMORY_FREE_SUPPORT |
OPAL_MEMORY_CHUNK_SUPPORT |
OPAL_MEMORY_MUNMAP_SUPPORT);
mca_memory_linux_component.super.memoryc_process = ummunotify_process;
mca_memory_linux_component.super.memoryc_register = ummunotify_register;
mca_memory_linux_component.super.memoryc_deregister = ummunotify_deregister;
initialized = true;
return OPAL_SUCCESS;
}
/*
* Called during opal_finalize (usually during MPI_FINALIZE) to tear
* down anything that can/should be torn down to disable this
* component. The application may continue for a while after
* MPI_FINALIZE, so we should do as much as possible to disable
* anything we enabled during ummunotify_open().
*/
int opal_memory_linux_ummunotify_close(void)
{
if (initialized && mca_memory_linux_component.ummunotify_fd >= 0) {
munmap((void*) opal_memory_linux_ummunotify_counter,
sizeof(*opal_memory_linux_ummunotify_counter));
close(mca_memory_linux_component.ummunotify_fd);
mca_memory_linux_component.ummunotify_fd = -1;
opal_memory_linux_ummunotify_counter =
&opal_memory_linux_ummunotify_counter_last_value;
initialized = false;
}
return OPAL_SUCCESS;
}
/*
* Called when opal_memory_changed() returns 1
*/
static int ummunotify_process(void)
{
int n;
unsigned int i;
struct ummunotify_event events[128];
/* Loop reading from the ummunot fd until there's nothing left to
read. If we get a LAST event, re-record the counter. */
while (initialized) {
n = read(mca_memory_linux_component.ummunotify_fd,
&events, sizeof(events));
if (n <= 0) {
return (EAGAIN == errno) ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
}
for (i = 0; i < n / sizeof(events[0]); ++i) {
switch (events[i].type) {
case UMMUNOTIFY_EVENT_TYPE_INVAL:
/* 0 => this callback did not come from malloc */
OPAL_OUTPUT((-1, "ummunot: invalidate start %p, end %p",
(void*) events[i].hint_start,
(void*) events[i].hint_end));
opal_mem_hooks_release_hook((void *) (uintptr_t) events[i].hint_start,
events[i].hint_end - events[i].hint_start,
0);
break;
case UMMUNOTIFY_EVENT_TYPE_LAST:
opal_memory_linux_ummunotify_counter_last_value =
events[i].user_cookie_counter;
/* Are there more events to read? */
if (opal_memory_linux_ummunotify_counter_last_value ==
*opal_memory_linux_ummunotify_counter) {
OPAL_OUTPUT((-1, "ummunot: LAST; done"));
return OPAL_SUCCESS;
}
OPAL_OUTPUT((-1, "ummunot: LAST; but looping around"));
break;
}
}
}
/* Will only get here if this component has not been
initialized */
return OPAL_SUCCESS;
}
static int ummunotify_register(void *start, size_t len, uint64_t cookie)
{
struct ummunotify_register_ioctl r;
r.reserved = 0;
r.start = (unsigned long) start;
r.end = (unsigned long) start + len;
r.user_cookie = cookie;
OPAL_OUTPUT((-1, "ummunot: register %p - %p",
start, ((char*) start) + len));
if (initialized && ioctl(mca_memory_linux_component.ummunotify_fd,
UMMUNOTIFY_REGISTER_REGION, &r)) {
OPAL_OUTPUT((-1, "Error in ioctl register!"));
return OPAL_ERR_IN_ERRNO;
}
return OPAL_SUCCESS;
}
static int ummunotify_deregister(void *start, size_t len, uint64_t cookie)
{
OPAL_OUTPUT((-1, "ummunot: deregister %p - %p",
start, ((char*) start) + len));
if (initialized && ioctl(mca_memory_linux_component.ummunotify_fd,
UMMUNOTIFY_UNREGISTER_REGION, &cookie)) {
OPAL_OUTPUT((-1, "Error in ioctl unregister!"));
return OPAL_ERR_IN_ERRNO;
}
return OPAL_SUCCESS;
}

24
opal/mca/memory/linux/public.h Обычный файл
Просмотреть файл

@ -0,0 +1,24 @@
/*
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_MEMORY_LINUX_PUBLIC_H
#define OPAL_MEMORY_LINUX_PUBLIC_H
#include "opal_config.h"
#include <sys/types.h>
OPAL_DECLSPEC extern volatile uint64_t *opal_memory_linux_ummunotify_counter;
OPAL_DECLSPEC extern uint64_t opal_memory_linux_ummunotify_counter_last_value;
#define opal_memory_changed() \
(opal_memory_linux_ummunotify_counter_last_value != \
*opal_memory_linux_ummunotify_counter)
#endif

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

@ -10,6 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -19,6 +20,7 @@
#include "opal_config.h" #include "opal_config.h"
#include "opal/mca/memory/memory.h" #include "opal/mca/memory/memory.h"
#include "opal/mca/memory/base/empty.h"
#include "opal/memoryhooks/memory_internal.h" #include "opal/memoryhooks/memory_internal.h"
#include "opal/constants.h" #include "opal/constants.h"
@ -60,6 +62,12 @@ const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
/* The component is checkpoint ready */ /* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT MCA_BASE_METADATA_PARAM_CHECKPOINT
}, },
/* This component doesn't need these functions, but need to
provide safe/empty register/deregister functions to call */
NULL,
opal_memory_base_component_register_empty,
opal_memory_base_component_deregister_empty,
}; };
/* /*

Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -16,16 +17,40 @@
* $HEADER$ * $HEADER$
*/ */
/* NOTE: This framework is really for build system only. There is no /* NOTE: This framework accomodates two kinds of memory hooks systems:
set of function pointers that must be called or set interface or
any of that. The only two functions that a component must call
(note: call, not implement) are defined in
opal/memoryhooks/memory_internal.h. Other than that, to each his
own..
Components should make some attempt to provide a component struct 1. Those that only require being setup once and then will
via the usual means, just so ompi_info has something rational to automatically call back to internal component/module functions
display. as required (e.g., the ptmalloc2 component). It is expected
that such components will have a NULL value for memoryc_process
and empty (base) implementations of memoryc_register and
memoryc_deregister.
2. Those that must be queried periodically to find out if something
has happened to the registered memory mappings (e.g., the
ummunot component). It is expected that such components will
have complete implementations for memoryc_process,
memoryc_register, and memoryc_deregister.
There will only be one of these components compiled in to Open MPI
(it will be compiled in libopen-pal; it will never be a DSO). so
the componente will rule "yes, I can run" or "no, I cannot run".
If it elects not to run, then there is no memory manager support in
this process.
Because there will only be one memory manager component in the
process, there is no need for a separate module structure --
everything is in the component for simplicity.
Note that there is an interface relevant to this framework that is
not described in this file: the opal_memory_changed() macro. This
macro should return 1 if an asynchronous agent has noticed that
memory mappings have changed. The macro should be as cheap/fast as
possible (which is why it's a macro). If it returns 1, the
memoryc_process() function pointer (below), will be invoked. If
the component does not provide this macro, then a default macro is
used that always returns 0 and therefore memoryc_process() will
never be invoked (and can legally be NULL).
*/ */
#ifndef OPAL_MCA_MEMORY_MEMORY_H #ifndef OPAL_MCA_MEMORY_MEMORY_H
@ -36,21 +61,79 @@
#include "opal/mca/mca.h" #include "opal/mca/mca.h"
#include "opal/mca/base/base.h" #include "opal/mca/base/base.h"
BEGIN_C_DECLS
/**
* Prototype for doing something once memory changes have been
* detected. This function is assumed to do whatever is necessary to
* a) figured out what has changed, and b) adjust OMPI as relevant
* (e.g., call opal_mem_hooks_release_hook). It only needs to be
* supplied for components that provide opal_memory_changed() macros
* that can return non-zero.
*/
typedef int (*opal_memory_base_component_process_fn_t)(void);
/**
* Prototype for a function that is invoked when Open MPI starts to
* "care" about a specific memory region. That is, Open MPI declares
* that it wants to be notified if the memory mapping for this region
* changes.
*
* If a component does not want/need to provide this functionality, it
* can use the value opal_memory_base_register_empty (an empty
* implementation of this function).
*/
typedef int (*opal_memory_base_component_register_fn_t)(void *base,
size_t len,
uint64_t cookie);
/**
* Prototype for a function that is the opposite of
* opal_memory_base_component_register_fn_t: this function is invoked
* when Open MPI stops to "caring" about a specific memory region.
* That is, Open MPI declares that it no longer wants to be notified
* if the memory mapping for this region changes.
*
* The parameters of this function will exactly match parameters that
* were previously passed to the call to
* opal_memory_base_component_register_fn_t.
*
* If a component does not want/need to provide this functionality, it
* can use the value opal_memory_base_deregister_empty (an empty
* implementation of this function).
*/
typedef int (*opal_memory_base_component_deregister_fn_t)(void *base,
size_t len,
uint64_t cookie);
/** /**
* Structure for memory components. * Structure for memory components.
*/ */
struct opal_memory_base_component_2_0_0_t { typedef struct opal_memory_base_component_2_0_0_t {
/** MCA base component */ /** MCA base component */
mca_base_component_t memoryc_version; mca_base_component_t memoryc_version;
/** MCA base data */ /** MCA base data */
mca_base_component_data_t memoryc_data; mca_base_component_data_t memoryc_data;
};
/**
* Convenience typedef
*/
typedef struct opal_memory_base_component_2_0_0_t opal_memory_base_component_2_0_0_t;
extern opal_memory_base_component_2_0_0_t *opal_memory_active_component; /** Function to call when something has changed, as indicated by
opal_memory_changed(). Will be ignored if the component does
not provide an opal_memory_changed() macro that returns
nonzero. */
opal_memory_base_component_process_fn_t memoryc_process;
/** Function invoked when Open MPI starts "caring" about a
specific memory region */
opal_memory_base_component_register_fn_t memoryc_register;
/** Function invoked when Open MPI stops "caring" about a
specific memory region */
opal_memory_base_component_deregister_fn_t memoryc_deregister;
} opal_memory_base_component_2_0_0_t;
OPAL_DECLSPEC extern opal_memory_base_component_2_0_0_t *opal_memory;
END_C_DECLS
/* /*
* Macro for use in components that are of type memory * Macro for use in components that are of type memory

Просмотреть файл

@ -1,133 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AC_DEFUN([MCA_memory_ptmalloc2_COMPILE_MODE], [
AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
$4="static"
AC_MSG_RESULT([$$4])
])
# MCA_memory_ptmalloc2_CONFIG(action-if-can-compile,
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_memory_ptmalloc2_CONFIG],[
AS_IF([test "$with_memory_manager" = "ptmalloc2"],
[if test "`echo $host | grep apple-darwin`" != "" ; then
AC_MSG_WARN([*** Using ptmalloc with OS X will result in failure.])
AC_MSG_ERROR([*** Aborting to save you the effort])
fi
memory_ptmalloc2_happy="yes"
memory_ptmalloc2_should_use=1],
[memory_ptmalloc2_should_use=0
AS_IF([test "$with_memory_manager" = ""],
[memory_ptmalloc2_happy="yes"],
[memory_ptmalloc2_happy="no"])])
# Per ticket #227, Intel 9.0 v20051201 on ia64 with optimization
# of -O2 or higher will bork ptmalloc2 in strange in mysterious
# ways. Doh! So if the compiler vendor is intel and we're on an
# ia64 box, run "icc --version" and snarf the version string. If
# it's 9.0 and the version is <= 20051201, then disable ptmalloc2.
# Executive decision: ignore optimization levels (even though -O1
# and -O0 seem to work). The upgrade to 9.1 is free, so that's a
# better path than trying to make a much more complicated test
# here.
case $host in
ia64-*)
AS_IF([test "$ompi_c_vendor" = "intel"],
[# check for v9.0 <= 20051201
icc_major_ver="`$CC --version | head -n 1 | awk '{ print [$]3 }'`"
icc_minor_ver="`$CC --version | head -n 1 | awk '{ print [$]4 }'`"
AS_IF([test "$icc_major_ver" = "9.0" -a "`expr $icc_minor_ver \<= 20051201`" = "1"],
[memory_ptmalloc2_happy="no"
AC_MSG_WARN([*** Detected Intel C compiler v9.0 <= 20051201 on ia64])
AC_MSG_WARN([*** This compiler/platform combination has known problems with ptmalloc2])
AC_MSG_WARN([*** Automatically disabling ptmalloc2])])
unset icc_major_ver icc_minor_ver])
;;
esac
AS_IF([test "$memory_ptmalloc2_happy" = "yes"],
[# check for malloc.h
AC_CHECK_HEADER([malloc.h],
[memory_ptmalloc2_happy="yes"],
[memory_ptmalloc2_happy="no"])])
AS_IF([test "$memory_ptmalloc2_happy" = "yes"],
[# check for init hook symbol
AC_CHECK_DECL([__malloc_initialize_hook],
[memory_ptmalloc2_happy="yes"],
[memory_ptmalloc2_happy="no"],
[AC_INCLUDES_DEFAULT
#include <malloc.h>])])
#
# See if we have sbrk prototyped
#
AS_IF([test "$memory_ptmalloc2_happy" = "yes"], [AC_CHECK_DECLS([sbrk])])
#
# Figure out how we're going to call mmap/munmap for real
#
AS_IF([test "$memory_ptmalloc2_happy" = "yes"],
[memory_ptmalloc2_mmap=0
memory_ptmalloc2_munmap=1
# it's nearly impossible to call mmap from syscall(), so
# only go this route if we can't get at munmap any other
# way.
AC_CHECK_HEADER([syscall.h],
[AC_CHECK_FUNCS([syscall], [], [memory_ptmalloc2_munmap=0])])
# Always look for __munmap and __mmap
AC_CHECK_FUNCS([__munmap], [memory_ptmalloc2_mmap=1])
AC_CHECK_FUNCS([__mmap])
# only allow dlsym (and therefore add -ldl) if we
# really need to
AS_IF([test "$memory_ptmalloc2_mmap" = "0"],
[memory_ptmalloc2_LIBS_SAVE="$LIBS"
AC_CHECK_LIB([dl],
[dlsym],
[LIBS="$LIBS -ldl"
memory_ptmalloc2_LIBS="-ldl"
memory_ptmalloc2_mmap=1])
AC_CHECK_FUNCS([dlsym])
LIBS="$memory_ptmalloc2_LIBS_SAVE"])
AS_IF([test "$memory_ptmalloc2_mmap" = "0" -a "$memory_ptmalloc2_munmap" = "0"],
[memory_ptmalloc2_happy="no"])])
AS_IF([test "$memory_ptmalloc2_happy" = "yes"],
[memory_ptmalloc2_WRAPPER_EXTRA_LIBS="$memory_ptmalloc2_LIBS"])
AS_IF([test "$memory_ptmalloc2_happy" = "no" -a \
"$memory_malloc_hoooks_should_use" = "1"],
[AC_MSG_ERROR([ptmalloc2 memory management requested but not available. Aborting.])])
AC_SUBST([memory_ptmalloc2_LIBS])
AS_IF([test "$memory_ptmalloc2_happy" = "yes"],
[memory_base_found=1
$1], [$2])
])

Просмотреть файл

@ -1,23 +0,0 @@
# -*- text -*-
#
# Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English help file for Open MPI's ptmalloc2 component.
#
[disable incorrectly set]
WARNING: The special MCA parameter "%s" was set in
an unexpected way, and is likely not working the way you want it to.
Specifically, this MCA parameter is "special" in that it can *only* be
set in the environment. Setting this value in a file -- and sometimes
even on the command line -- will not work as intended. The *only* way
to set this value is to set "OMPI_MCA_%s" in the environment before
starting your job.
Value: %d
Source: %s

Просмотреть файл

@ -1,150 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/* The goal of this component is to wholly replace the underlying
allocator with our internal ptmalloc2 allocator.
See the file README-open-mpi.txt for details of how it works. */
#include "opal_config.h"
#include <malloc.h>
#include <sys/mman.h>
#include "opal/constants.h"
#include "opal/mca/memory/memory.h"
#include "opal/memoryhooks/memory.h"
/* Need to call a function in hooks.c to ensure that all those symbols
get pulled in at link time (e.g., when building libmpi.a, so that
those symbols end up in the final executable -- especially if we
use --disable-dlopen and therefore -Wl,--export-dynamic isn't used
when we build OMPI). */
extern void *opal_memory_ptmalloc2_hook_pull(void);
static int ptmalloc2_open(void);
const opal_memory_base_component_2_0_0_t mca_memory_ptmalloc2_component = {
/* First, the mca_component_t struct containing meta information
about the component itself */
{
OPAL_MEMORY_BASE_VERSION_2_0_0,
/* Component name and version */
"ptmalloc2",
OPAL_MAJOR_VERSION,
OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
/* Component open and close functions */
ptmalloc2_open,
NULL
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
},
};
/* Public symbols */
bool opal_memory_ptmalloc2_free_invoked = false;
bool opal_memory_ptmalloc2_malloc_invoked = false;
bool opal_memory_ptmalloc2_realloc_invoked = false;
bool opal_memory_ptmalloc2_memalign_invoked = false;
bool opal_memory_ptmalloc2_munmap_invoked = false;
static int ptmalloc2_open(void)
{
void *p;
int val = 0;
/* Call a dummy function in hooks.c. ***Do not remove this
call!*** See comment at the beginning of this file explaining
why it is here. */
p = opal_memory_ptmalloc2_hook_pull();
/* We will also provide malloc/free support if we've been
activated. We don't rely on the __malloc_initialize_hook()
previously being called because it's possible that our hook was
called, but then someone else reset the hooks to point to
something else (i.e., before MPI_INIT). So explicitly test
here if our hooks are still in place. If they are, then enable
FREE|CUNK_SUPPORT. If not, then don't enable that support --
just leave it at MUNMAP_SUPPORT.
(Look in hooks.c for the __malloc_initialize_hook setup) */
/* Do a simple set of tests to see if our hooks are still the ones
installed. Explicitly reset the flags indicating that our
functions were invoked */
opal_memory_ptmalloc2_malloc_invoked = false;
opal_memory_ptmalloc2_realloc_invoked = false;
opal_memory_ptmalloc2_memalign_invoked = false;
opal_memory_ptmalloc2_free_invoked = false;
opal_memory_ptmalloc2_munmap_invoked = false;
p = malloc(1024 * 1024 * 4);
if (NULL == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
p = realloc(p, 1024 * 1024 * 4 + 32);
if (NULL == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
free(p);
p = memalign(4, 1024 * 1024);
if (NULL == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
free(p);
#if HAVE_POSIX_MEMALIGN
/* Double check for posix_memalign, too */
if (opal_memory_ptmalloc2_memalign_invoked) {
opal_memory_ptmalloc2_memalign_invoked = false;
if (0 != posix_memalign(&p, sizeof(void*), 1024 * 1024)) {
return OPAL_ERR_IN_ERRNO;
}
free(p);
}
#endif
if (opal_memory_ptmalloc2_malloc_invoked &&
opal_memory_ptmalloc2_realloc_invoked &&
opal_memory_ptmalloc2_memalign_invoked &&
opal_memory_ptmalloc2_free_invoked) {
/* Happiness; our functions were invoked */
val |= OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_CHUNK_SUPPORT;
}
p = mmap(NULL, 4096, PROT_READ, (MAP_ANONYMOUS | MAP_PRIVATE), -1, 0);
if (MAP_FAILED == p) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
munmap(p, 4096);
if (opal_memory_ptmalloc2_munmap_invoked) {
val |= OPAL_MEMORY_MUNMAP_SUPPORT;
}
/* Set the support level */
opal_mem_hooks_set_support(val);
return OPAL_SUCCESS;
}