1
1

Adding memkind component to use MPI_Alloc_mem through memkind

Этот коммит содержится в:
Vishwanath Venkatesan 2015-07-01 23:18:26 -07:00 коммит произвёл Nathan Hjelm
родитель d4afb16f5a
Коммит 3d98a1a01e
5 изменённых файлов: 525 добавлений и 0 удалений

41
opal/mca/mpool/memkind/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,41 @@
# -*- indent-tabs-mode:nil -*-
#
# Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights
# reserved.
#
# Additional copyrights may follow
#
# $HEADER$
#
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
AM_CPPFLAGS = $(mpool_memkind_CPPFLAGS)
if MCA_BUILD_opal_mpool_memkind_DSO
component_noinst =
component_install = mca_mpool_memkind.la
else
component_noinst = libmca_mpool_memkind.la
component_install =
endif
memkind_SOURCES = \
mpool_memkind_component.c \
mpool_memkind_module.c \
mpool_memkind.h
mcacomponentdir = $(opallibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mpool_memkind_la_SOURCES = $(memkind_SOURCES)
nodist_mca_mpool_memkind_la_SOURCES = $(memkind_nodist_SOURCES)
mca_mpool_memkind_la_LIBADD = $(mpool_memkind_LIBS)
mca_mpool_memkind_la_LDFLAGS = -module -avoid-version $(mpool_memkind_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_mpool_memkind_la_SOURCES = $(memkind_SOURCES)
nodist_libmca_mpool_memkind_la_SOURCES = $(memkind_nodist_SOURCES)
libmca_mpool_memkind_la_LIBADD = $(mpool_memkind_LIBS)
libmca_mpool_memkind_la_LDFLAGS = -module -avoid-version $(mpool_memkind_LDFLAGS)

42
opal/mca/mpool/memkind/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,42 @@
# -*- shell-script -*-
#
# Copyright (c) 2013-2014 Los Alamos National Security, LLC. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AC_DEFUN([MCA_opal_mpool_memkind_CONFIG],[
OPAL_VAR_SCOPE_PUSH([opal_mpool_memkind_happy])
AC_CONFIG_FILES([opal/mca/mpool/memkind/Makefile])
AC_ARG_WITH([memkind], [AC_HELP_STRING([--with-memkind(=DIR)]),
[Build with MEMKIND, searching for headers in DIR])])
OPAL_CHECK_WITHDIR([memkind], [$with_memkind], [include/memkind.h])
opal_mpool_memkind_happy="no"
if test "$with_memkind" != "no" ; then
if test -n "$with_memkind" -a "$with_memkind" != "yes" ; then
opal_check_memkind_dir=$with_memkind
fi
OPAL_CHECK_PACKAGE([mpool_memkind], [memkind.h], [memkind], [memkind_malloc], [-ljemalloc, -lnuma],
[$opal_check_memkind_dir], [], [opal_mpool_memkind_happy="yes"], [])
if test "$opal_mpool_memkind_happy" != "yes" -a -n "$with_memkind" ; then
AC_MSG_ERROR([MEMKIND support requested but not found. Aborting])
fi
fi
AS_IF([test "$opal_mpool_memkind_happy" = "yes"], [$1], [$2])
# substitute in the things needed to build memkind
AC_SUBST([mpool_memkind_CPPFLAGS])
AC_SUBST([mpool_memkind_LDFLAGS])
AC_SUBST([mpool_memkind_LIBS])
OPAL_VAR_SCOPE_POP
])dnl

103
opal/mca/mpool/memkind/mpool_memkind.h Обычный файл
Просмотреть файл

@ -0,0 +1,103 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2012 Los Alamos National Security, LLC.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_MPOOL_MEMKIND_H
#define MCA_MPOOL_MEMKIND_H
#include "opal_config.h"
#include "opal/mca/event/event.h"
#include "opal/mca/mpool/mpool.h"
#include "opal/mca/allocator/allocator.h"
#include "memkind.h"
BEGIN_C_DECLS
struct mca_mpool_base_resources_t {
char *pool_name;
void *reg_data;
size_t size;
};
static const int mca_mpool_memkind_default_pagesize = 4096;
typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;
struct mca_mpool_memkind_module_t {
mca_mpool_base_module_t super;
size_t alloc_size;
struct mca_mpool_base_resources_t resources;
};
typedef struct mca_mpool_memkind_module_t mca_mpool_memkind_module_t;
struct mca_mpool_memkind_component_t {
mca_mpool_base_component_t super;
int hbw;
int pagesize;
int bind;
memkind_t kind;
char *memkind_name;
char *memkind_file;
int verbose;
};
typedef struct mca_mpool_memkind_component_t mca_mpool_memkind_component_t;
OPAL_MODULE_DECLSPEC extern mca_mpool_memkind_component_t mca_mpool_memkind_component;
/*
* Initializes the mpool module.
*/
void mca_mpool_memkind_module_init(mca_mpool_memkind_module_t *mpool);
/**
* Allocate block of high bandwidth memory.
*/
void* mca_mpool_memkind_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration);
/**
* realloc function typedef
*/
void* mca_mpool_memkind_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration);
/**
* free function typedef
*/
void mca_mpool_memkind_free(
mca_mpool_base_module_t* mpool,
void * addr,
mca_mpool_base_registration_t* registration);
END_C_DECLS
#endif

Просмотреть файл

@ -0,0 +1,256 @@
/*
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2013 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2014 NVIDIA Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif /* HAVE_UNISTD_H*/
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif /* HAVE_STDLIB_H */
#include <errno.h>
#include <memkind.h>
#include "opal/mca/base/base.h"
#include "opal/mca/allocator/base/base.h"
#include "mpool_memkind.h"
/*
* Local functions
*/
static int
mca_mpool_memkind_register(void);
static int
mca_mpool_memkind_open(void);
static int
mca_mpool_memkind_close(void);
static mca_mpool_base_module_t*
mca_mpool_memkind_init(struct mca_mpool_base_resources_t* resources);
mca_mpool_memkind_component_t mca_mpool_memkind_component = {
{
/* First, the mca_base_component_t struct containing meta
information about the component itself */
{
MCA_MPOOL_BASE_VERSION_2_0_0,
"memkind", /* MCA component name */
OPAL_MAJOR_VERSION, /* MCA component major version */
OPAL_MINOR_VERSION, /* MCA component minor version */
OPAL_RELEASE_VERSION, /* MCA component release version */
mca_mpool_memkind_open, /* component open */
mca_mpool_memkind_close,
NULL,
mca_mpool_memkind_register
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
},
mca_mpool_memkind_init
}
};
static int opal_mpool_memkind_verbose;
static int mca_mpool_memkind_register(void)
{
char *memkind_names;
/* register MEMKIND component parameters */
mca_mpool_memkind_component.hbw = 1;
mca_mpool_memkind_component.bind = 0;
mca_mpool_memkind_component.pagesize = mca_mpool_memkind_default_pagesize;
memkind_names = (char *) malloc (2048 * sizeof(char));
if (NULL == memkind_names){
return OPAL_ERROR;
}
mca_mpool_memkind_component.memkind_name = (char *) malloc (4096 * sizeof(char));
if (NULL == mca_mpool_memkind_component.memkind_name){
return OPAL_ERROR;
}
mca_mpool_memkind_component.memkind_file = (char *) malloc (4096 * sizeof(char));
if (NULL ==mca_mpool_memkind_component.memkind_file){
return OPAL_ERROR;
}
strncpy (mca_mpool_memkind_component.memkind_name, "memkind_default", 17);
sprintf(memkind_names, "Use a specific kind of memory from (memkind_default, memkind_hugetlb, memkind_hbw, memkind_hbw_preferred, memkind_hbw_hugetlb, memkind_hbw_preferred_hugetlb, memkind_hbw_gbtlb memkind_hbw_preferred_gbtlb, memkind_gbtlb)");
(void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,"high_bandwidth",
"Allocate in high bandwidth node (0-> no, 1 -> yes)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_mpool_memkind_component.hbw);
(void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,"page_size",
"Allocate with different page size (4096, 2097152, 1073741824)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_mpool_memkind_component.pagesize);
(void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version, "bind",
"Bind allocations to specific nodes (0->preferred, 1-> bind)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_mpool_memkind_component.bind);
(void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,
"verbose", "Enable verbose output for mpool memkind component",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&opal_mpool_memkind_verbose);
(void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,
"name",
memkind_names,
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_mpool_memkind_component.memkind_name);
(void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,
"config",
"Config file user defined hints",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_mpool_memkind_component.memkind_file);
opal_output(mca_mpool_memkind_component.verbose,
"mca_mpool_memkind_register: Allocating with hbw: %d, pagesize: %d",
mca_mpool_memkind_component.hbw, mca_mpool_memkind_component.pagesize);
/*Setting the appropriate based on mca parameters
default allocates with 4K pages on high bandwidth memory on preferred mode.
For more information about the kinds, (refer to the manpage of memkind)
www.github.com/memkind
*/
if (!mca_mpool_memkind_component.hbw) {
switch (mca_mpool_memkind_component.pagesize) {
case 4096:
mca_mpool_memkind_component.kind = MEMKIND_DEFAULT;
break;
case 2097152:
mca_mpool_memkind_component.kind = MEMKIND_HUGETLB;
break;
case 1073741824:
mca_mpool_memkind_component.kind = MEMKIND_GBTLB;
break;
}
}
else if (1 == mca_mpool_memkind_component.hbw){
switch(mca_mpool_memkind_component.pagesize) {
case 4096:
mca_mpool_memkind_component.kind = MEMKIND_HBW;
break;
case 2097152:
mca_mpool_memkind_component.kind = MEMKIND_HBW_HUGETLB;
break;
case 1073741824:
mca_mpool_memkind_component.kind = MEMKIND_HBW_GBTLB;
break;
}
}
else {
switch (mca_mpool_memkind_component.pagesize) {
case 4096:
mca_mpool_memkind_component.kind = MEMKIND_HBW_PREFERRED;
break;
case 2097152:
mca_mpool_memkind_component.kind = MEMKIND_HBW_PREFERRED_HUGETLB;
break;
case 1073741824:
mca_mpool_memkind_component.kind = MEMKIND_HBW_PREFERRED_GBTLB;
break;
}
}
if(NULL != memkind_names){
free (memkind_names);
memkind_names = NULL;
}
return OPAL_SUCCESS;
}
/**
* component open/close/init function
*/
static int mca_mpool_memkind_open(void)
{
if (opal_mpool_memkind_verbose != 0) {
mca_mpool_memkind_component.verbose = opal_output_open(NULL);
}
else {
mca_mpool_memkind_component.verbose = -1;
}
return OPAL_SUCCESS;
}
static int mca_mpool_memkind_close(void)
{
return OPAL_SUCCESS;
}
static mca_mpool_base_module_t*
mca_mpool_memkind_init(struct mca_mpool_base_resources_t* resources)
{
mca_mpool_memkind_module_t* mpool_module;
mca_mpool_base_module_t* ret_val;
/*Check if the high bandwidth node is available ?*/
if (!memkind_check_available(MEMKIND_HBW)) {
opal_output(mca_mpool_memkind_component.verbose,
"mca_mpool_memkind_init: High bandwidth node not available");
ret_val = NULL;
}
else {
mpool_module =
(mca_mpool_memkind_module_t*)malloc(sizeof(mca_mpool_memkind_module_t));
mca_mpool_memkind_module_init(mpool_module);
mpool_module->alloc_size = resources->size;
opal_output(mca_mpool_memkind_component.verbose,
"mca_mpool_memkind_init: allocation size requested: (%ld)",
mpool_module->alloc_size);
ret_val = &mpool_module->super;
}
return ret_val;
}

Просмотреть файл

@ -0,0 +1,83 @@
/*
* Copyright (c) 2004-2011 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2011-2014 NVIDIA Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <string.h>
#include "opal/mca/mpool/memkind/mpool_memkind.h"
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "opal/mca/mpool/base/base.h"
/*
* Initializes the mpool module.
*/
void mca_mpool_memkind_module_init(mca_mpool_memkind_module_t* mpool)
{
mpool->super.mpool_component = &mca_mpool_memkind_component.super;
mpool->super.mpool_alloc = mca_mpool_memkind_alloc;
mpool->super.mpool_realloc = mca_mpool_memkind_realloc;
mpool->super.mpool_free = mca_mpool_memkind_free;
mpool->super.mpool_find = NULL;
mpool->super.mpool_register = NULL;
mpool->super.mpool_deregister = NULL;
mpool->super.mpool_release_memory = NULL;
mpool->super.flags = 0;
}
void* mca_mpool_memkind_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
void *addr;
if(0 == align)
align = mca_mpool_base_page_size;
if ((errno = memkind_posix_memalign(mca_mpool_memkind_component.kind,
&addr, align, size))!= 0){
return NULL;
}
return addr;
}
void* mca_mpool_memkind_realloc(mca_mpool_base_module_t *mpool, void *addr,
size_t size, mca_mpool_base_registration_t **registration)
{
return memkind_realloc(mca_mpool_memkind_component.kind,
addr, size);
}
void mca_mpool_memkind_free(mca_mpool_base_module_t *mpool, void *addr,
mca_mpool_base_registration_t *registration)
{
memkind_free(mca_mpool_memkind_component.kind,
addr);
}