ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Remove some now-obsolete mpool's (all have been replaced by RDMA).

Просмотр исходного кода 
These components were already .ompi_ignored -- this removal should not
trigger the need for an autogen.

This commit was SVN r15607.
Этот коммит содержится в:
Jeff Squyres 2007-07-25 12:37:09 +00:00
родитель f4b117957d
Коммит 4d53b2f2a7
30 изменённых файлов: 0 добавлений и 2637 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
ompi/mca/mpool/gm/.ompi_ignore
Просмотреть файл

@ -1 +0,0 @@
quilt

57
ompi/mca/mpool/gm/Makefile.am
Просмотреть файл

@ -1,57 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
AM_CPPFLAGS = $(mpool_gm_CPPFLAGS)
sources = \
mpool_gm.h \
mpool_gm_module.c \
mpool_gm_component.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_mpool_gm_DSO
component_noinst =
component_install = mca_mpool_gm.la
else
component_noinst = libmca_mpool_gm.la
component_install =
endif
# See src/mca/ptl/gm/Makefile.am for an explanation of
# libmca_common_gm.la.
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mpool_gm_la_SOURCES = $(sources)
mca_mpool_gm_la_LIBADD = \
$(mpool_gm_LIBS) \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/libopen-rte.la \
$(top_ompi_builddir)/opal/libopen-pal.la
mca_mpool_gm_la_LDFLAGS = -module -avoid-version $(mpool_gm_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_mpool_gm_la_SOURCES = $(sources)
libmca_mpool_gm_la_LIBADD = $(mpool_gm_LIBS)
libmca_mpool_gm_la_LDFLAGS = -module -avoid-version $(mpool_gm_LDFLAGS)

40
ompi/mca/mpool/gm/configure.m4
Просмотреть файл

@ -1,40 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_mpool_gm_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_mpool_gm_CONFIG],[
OMPI_CHECK_GM([mpool_gm],
[mpool_gm_happy="yes"],
[mpool_gm_happy="no"])
AS_IF([test "$mpool_gm_happy" = "yes"],
[mpool_gm_WRAPPER_EXTRA_LDFLAGS="$mpool_gm_LDFLAGS"
mpool_gm_WRAPPER_EXTRA_LIBS="$mpool_gm_LIBS"
$1],
[$2])
# substitute in the things needed to build gm
AC_SUBST([mpool_gm_CFLAGS])
AC_SUBST([mpool_gm_CPPFLAGS])
AC_SUBST([mpool_gm_LDFLAGS])
AC_SUBST([mpool_gm_LIBS])
])dnl

34
ompi/mca/mpool/gm/configure.params
Просмотреть файл

@ -1,34 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007 Los Alamos National Security, LLC. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_CONFIG_FILES="Makefile"

149
ompi/mca/mpool/gm/mpool_gm.h
Просмотреть файл

@ -1,149 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_MPOOL_GM_H
#define MCA_MPOOL_GM_H
#include "ompi_config.h"
#include <gm.h>
#include "opal/class/opal_list.h"
#include "ompi/class/ompi_free_list.h"
#include "opal/event/event.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/mca/allocator/allocator.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
#define MCA_MPOOL_GM_HAVE_REGISTER
struct mca_mpool_gm_component_t {
mca_mpool_base_component_t super;
char* rcache_name;
};
typedef struct mca_mpool_gm_component_t mca_mpool_gm_component_t;
OMPI_DECLSPEC extern mca_mpool_gm_component_t mca_mpool_gm_component;
struct mca_mpool_base_resources_t {
struct gm_port *port; /* the gm NIC handle */
};
typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;
struct mca_mpool_gm_module_t {
mca_mpool_base_module_t super;
struct gm_port* port;
};
typedef struct mca_mpool_gm_module_t mca_mpool_gm_module_t;
/*
* Initializes the mpool module.
*/
void mca_mpool_gm_module_init(mca_mpool_gm_module_t* mpool);
/*
* Returns base address of shared memory mapping.
*/
void* mca_mpool_gm_base(mca_mpool_base_module_t*);
/**
* Allocate block of shared memory.
*/
void* mca_mpool_gm_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration);
/**
* realloc function typedef
*/
void* mca_mpool_gm_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration);
/**
* register function typedef
*/
int mca_mpool_gm_register(
mca_mpool_base_module_t* mpool,
void *addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration);
int mca_mpool_gm_deregister(
mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t*);
/**
* free function typedef
*/
void mca_mpool_gm_free(
mca_mpool_base_module_t* mpool,
void * addr,
mca_mpool_base_registration_t* registration);
int mca_mpool_gm_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t *cnt
);
int mca_mpool_gm_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registraion
);
int mca_mpool_gm_retain(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registraion
);
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae
* @return OMPI_SUCCESS or failure status
*/
int mca_mpool_gm_ft_event(int state);
void* mca_common_gm_segment_alloc(
struct mca_mpool_base_module_t* module,
size_t* size,
mca_mpool_base_registration_t** registration);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

97
ompi/mca/mpool/gm/mpool_gm_component.c
Просмотреть файл

@ -1,97 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/mca/allocator/base/base.h"
#include "mpool_gm.h"
#include "orte/util/proc_info.h"
#include "orte/util/sys_info.h"
#include <unistd.h>
/*
* Local functions
*/
static int mca_mpool_gm_open(void);
static mca_mpool_base_module_t* mca_mpool_gm_init(
struct mca_mpool_base_resources_t* resources);
mca_mpool_gm_component_t mca_mpool_gm_component = {
{
/* First, the mca_base_component_t struct containing meta
information about the component itself */
{
/* Indicate that we are a mpool v1.0.0 component (which also
implies a specific MCA version) */
MCA_MPOOL_BASE_VERSION_1_0_0,
"gm", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_mpool_gm_open, /* component open */
NULL
},
/* Next the MCA v1.0.0 component meta data */
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_mpool_gm_init
}
};
/**
* component open/close/init function
*/
static int mca_mpool_gm_open(void)
{
return OMPI_SUCCESS;
}
/* Allocates a segment of memory and registers with GM, user_out returns the memory handle. */
static mca_mpool_base_module_t* mca_mpool_gm_init(
struct mca_mpool_base_resources_t* resources)
{
mca_mpool_gm_module_t* gm_mpool;
mca_base_param_reg_string(&mca_mpool_gm_component.super.mpool_version,
"rcache_name",
"The name of the registration cache the mpool should use",
false,
false,
"rb",
&(mca_mpool_gm_component.rcache_name));
gm_mpool = (mca_mpool_gm_module_t*)malloc(sizeof(mca_mpool_gm_module_t));
mca_mpool_gm_module_init(gm_mpool);
gm_mpool->port = resources->port;
return &gm_mpool->super;
}

238
ompi/mca/mpool/gm/mpool_gm_module.c
Просмотреть файл

@ -1,238 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "opal/util/output.h"
#include "mpool_gm.h"
#include "ompi/mca/rcache/rcache.h"
#include "ompi/mca/rcache/base/base.h"
#include "ompi/mca/mpool/base/base.h"
uint64_t mca_mpool_gm_mem_registered;
/*
* Initializes the mpool module.
*/
void mca_mpool_gm_module_init(mca_mpool_gm_module_t* mpool)
{
mpool->super.mpool_component = &mca_mpool_gm_component.super;
mpool->super.mpool_base = NULL; /* no base .. */
mpool->super.mpool_alloc = mca_mpool_gm_alloc;
mpool->super.mpool_realloc = mca_mpool_gm_realloc;
mpool->super.mpool_free = mca_mpool_gm_free;
mpool->super.mpool_register = mca_mpool_gm_register;
mpool->super.mpool_deregister = mca_mpool_gm_deregister;
mpool->super.mpool_find = mca_mpool_gm_find;
mpool->super.mpool_retain = mca_mpool_gm_retain;
mpool->super.mpool_release = mca_mpool_gm_release;
mpool->super.mpool_finalize = NULL;
mpool->super.mpool_ft_event = mca_mpool_gm_ft_event;
mpool->super.rcache =
mca_rcache_base_module_create(mca_mpool_gm_component.rcache_name);
mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM;
mca_mpool_gm_mem_registered = 0;
}
/**
* allocate function
*/
void* mca_mpool_gm_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
void *addr = malloc(size+align);
if(NULL == addr)
return NULL;
if(OMPI_SUCCESS != mca_mpool_gm_register(mpool,addr,size+align, flags, registration)) {
free(addr);
return NULL;
}
return addr;
}
/*
* register memory
*/
int mca_mpool_gm_register(
mca_mpool_base_module_t* mpool,
void *addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
mca_mpool_gm_module_t * gm_mpool = (mca_mpool_gm_module_t*) mpool;
mca_mpool_base_registration_t* reg = OBJ_NEW(mca_mpool_base_registration_t);
int rc;
if(NULL == reg) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
reg->mpool = mpool;
reg->base = down_align_addr(addr, mca_mpool_base_page_size_log);
reg->flags = flags;
reg->bound = up_align_addr((void*) ((unsigned long) addr + size -1)
, mca_mpool_base_page_size_log);
if((rc = gm_register_memory(gm_mpool->port, reg->base, reg->bound - reg->base + 1)) != GM_SUCCESS) {
opal_output(0, "[%s:%d] error(%d) registering gm memory\n", __FILE__, __LINE__, rc);
assert(0);
OBJ_RELEASE(reg);
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_mpool_gm_mem_registered += reg->bound - reg->base + 1;
OPAL_THREAD_ADD32(&reg->ref_count,1);
if(flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_insert(mpool->rcache,
(mca_mpool_base_registration_t*) reg,
flags);
}
*registration = reg;
/* opal_output(0,"registered memory from %p to %p\n", reg->base, reg->bound); */
return OMPI_SUCCESS;
}
/*
* deregister memory
*/
int mca_mpool_gm_deregister(mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* reg)
{
int rc;
if(reg->flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_delete(mpool->rcache,
reg,
reg->flags);
}
if((rc = mca_mpool_gm_release(mpool, reg)) != GM_SUCCESS) {
opal_output(0, "[%s:%d] error(%d) deregistering gm memory\n", __FILE__, __LINE__, rc);
assert(0);
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
/**
* realloc function
*/
void* mca_mpool_gm_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration)
{
void *new_addr = mca_mpool_gm_alloc(mpool,size,0, (*registration)->flags, registration);
if(new_addr == NULL) {
return NULL;
}
memcpy(new_addr,addr,size);
mca_mpool_gm_free(mpool,addr,*registration);
return new_addr;
}
/**
* free function
*/
void mca_mpool_gm_free(mca_mpool_base_module_t* mpool, void * addr,
mca_mpool_base_registration_t* registration)
{
if(registration){
mpool->mpool_deregister(mpool, registration);
}
free(addr);
}
int mca_mpool_gm_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t *cnt)
{
return mpool->rcache->rcache_find(mpool->rcache,
addr,
size,
regs,
cnt);
}
int mca_mpool_gm_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* reg)
{
mca_mpool_gm_module_t * mpool_gm = (mca_mpool_gm_module_t*) mpool;
if(0 == OPAL_THREAD_ADD32(&reg->ref_count, -1)) {
int rc = gm_deregister_memory(
mpool_gm->port,
reg->base,
reg->bound - reg->base + 1);
if(GM_SUCCESS != rc) {
opal_output(0, "[%s:%d] error(%d) deregistering gm memory\n", __FILE__, __LINE__, rc);
return OMPI_ERROR;
}
/* opal_output(0,"deregistering gm memory\n"); */
mca_mpool_gm_mem_registered -= reg->bound - reg->base + 1;
OBJ_RELEASE(reg);
}
else {
/* opal_output(0, "release says ref_count is %d\n", reg->ref_count); */
}
return OMPI_SUCCESS;
}
int mca_mpool_gm_retain(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration)
{
OPAL_THREAD_ADD32(&registration->ref_count, 1);
return OMPI_SUCCESS;
}
int mca_mpool_gm_ft_event(int state) {
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
;
}
else if(OPAL_CRS_RESTART == state) {
;
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}

1
ompi/mca/mpool/mvapi/.ompi_ignore
Просмотреть файл

@ -1 +0,0 @@
quilt

61
ompi/mca/mpool/mvapi/Makefile.am
Просмотреть файл

@ -1,61 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
CFLAGS = $(mpool_mvapi_CFLAGS)
AM_CPPFLAGS = $(mpool_mvapi_CPPFLAGS)
sources = \
mpool_mvapi.h \
mpool_mvapi_module.c \
mpool_mvapi_component.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_mpool_mvapi_DSO
component_noinst =
component_install = mca_mpool_mvapi.la
else
component_noinst = libmca_mpool_mvapi.la
component_install =
endif
# See src/mca/ptl/mvapi/Makefile.am for an explanation of
# libmca_common_mvapi.la.
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mpool_mvapi_la_SOURCES = $(sources)
mca_mpool_mvapi_la_LDFLAGS = -module -avoid-version $(mpool_mvapi_LDFLAGS)
mca_mpool_mvapi_la_LIBADD = \
$(mpool_mvapi_LIBS) \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/libopen-rte.la \
$(top_ompi_builddir)/opal/libopen-pal.la
noinst_LTLIBRARIES = $(component_noinst)
libmca_mpool_mvapi_la_SOURCES = $(sources)
libmca_mpool_mvapi_la_LDFLAGS = -module -avoid-version $(mpool_mvapi_LDFLAGS)
libmca_mpool_mvapi_la_LIBADD = $(mpool_mvapi_LIBS)

51
ompi/mca/mpool/mvapi/configure.m4
Просмотреть файл

@ -1,51 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_mpool_mvapi_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_mpool_mvapi_CONFIG],[
OMPI_CHECK_MVAPI([mpool_mvapi],
[mpool_mvapi_happy="yes"],
[mpool_mvapi_happy="no"])
AS_IF([test "$mpool_mvapi_happy" = "yes"],
[mpool_mvapi_WRAPPER_EXTRA_LDFLAGS="$mpool_mvapi_LDFLAGS"
mpool_mvapi_WRAPPER_EXTRA_LIBS="$mpool_mvapi_LIBS"
$1],
[$2])
# Many of the vapi.h files floating around don't obey ISO99 C
# standard, so cause oodles of warnings with -pedantic and
# -Wundef. Remove them from CFLAGS, which is then used to
# forcefully override CFLAGS in the makefile for MVAPI
# components
mpool_mvapi_CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
mpool_mvapi_CFLAGS="`echo $mpool_mvapi_CFLAGS | sed 's/-Wundef//g'`"
AS_IF([test "$mpool_mvapi_CFLAGS" != "$CFLAGS" -a "$mpool_mvapi_happy" = "yes"],
[AC_MSG_WARN([Removed -pedantic and -Wundef from CFLAGS for
mvapi component because some vapi.h files are not really ANSI C])])
# substitute in the things needed to build Portals
AC_SUBST([mpool_mvapi_CFLAGS])
AC_SUBST([mpool_mvapi_CPPFLAGS])
AC_SUBST([mpool_mvapi_LDFLAGS])
AC_SUBST([mpool_mvapi_LIBS])
])dnl

24
ompi/mca/mpool/mvapi/configure.params
Просмотреть файл

@ -1,24 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007 Los Alamos National Security, LLC. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_CONFIG_FILES="Makefile"

158
ompi/mca/mpool/mvapi/mpool_mvapi.h
Просмотреть файл

@ -1,158 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_MPOOL_VAPI_H
#define MCA_MPOOL_VAPI_H
#include "opal/class/opal_list.h"
#include "ompi/class/ompi_free_list.h"
#include "opal/event/event.h"
#include "ompi/mca/mpool/mpool.h"
#include <vapi.h>
#include <vapi_types.h>
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_mpool_mvapi_component_t {
mca_mpool_base_component_t super;
char* rcache_name;
};
typedef struct mca_mpool_mvapi_component_t mca_mpool_mvapi_component_t;
OMPI_DECLSPEC extern mca_mpool_mvapi_component_t mca_mpool_mvapi_component;
struct mca_mpool_base_resources_t {
VAPI_hca_hndl_t hca; /* the hca (nic) */
VAPI_pd_hndl_t pd_tag; /* the protection domain */
};
typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;
struct mca_mpool_mvapi_module_t {
mca_mpool_base_module_t super;
struct mca_mpool_base_resources_t hca_pd;
ompi_free_list_t reg_list;
}; typedef struct mca_mpool_mvapi_module_t mca_mpool_mvapi_module_t;
struct mca_mpool_mvapi_registration_t {
mca_mpool_base_registration_t base_reg;
VAPI_mr_hndl_t hndl;
/* Memory region handle */
VAPI_lkey_t l_key;
/* Local key to registered memory, needed for
* posting send/recv requests */
VAPI_rkey_t r_key;
/* Remote key to registered memory, need to send this
* to remote processes for incoming RDMA ops */
};
typedef struct mca_mpool_mvapi_registration_t mca_mpool_mvapi_registration_t;
OBJ_CLASS_DECLARATION(mca_mpool_mvapi_registration_t);
/*
* Initializes the mpool module.
*/
void mca_mpool_mvapi_module_init(mca_mpool_mvapi_module_t* mpool);
/*
* Returns base address of shared memory mapping.
*/
void* mca_mpool_mvapi_base(mca_mpool_base_module_t*);
/**
* Allocate block of shared memory.
*/
void* mca_mpool_mvapi_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration);
/**
* realloc function typedef
*/
void* mca_mpool_mvapi_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration);
/**
* register function typedef
*/
int mca_mpool_mvapi_register(
mca_mpool_base_module_t* mpool,
void *addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration);
int mca_mpool_mvapi_deregister(
mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* );
/**
* free function typedef
*/
void mca_mpool_mvapi_free(mca_mpool_base_module_t* mpool,
void * addr,
mca_mpool_base_registration_t* registration);
int mca_mpool_mvapi_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t *cnt
);
int mca_mpool_mvapi_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registraion
);
int mca_mpool_mvapi_retain(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registraion
);
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae
* @return OMPI_SUCCESS or failure status
*/
int mca_mpool_mvapi_ft_event(int state);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

124
ompi/mca/mpool/mvapi/mpool_mvapi_component.c
Просмотреть файл

@ -1,124 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "orte/util/proc_info.h"
#include "orte/util/sys_info.h"
#include "mpool_mvapi.h"
/*
* Local functions
*/
static int mca_mpool_mvapi_open(void);
static mca_mpool_base_module_t* mca_mpool_mvapi_init(
struct mca_mpool_base_resources_t* resources);
mca_mpool_mvapi_component_t mca_mpool_mvapi_component = {
{
/* First, the mca_base_component_t struct containing meta
information about the component itself */
{
/* Indicate that we are a mpool v1.0.0 component (which also
implies a specific MCA version) */
MCA_MPOOL_BASE_VERSION_1_0_0,
"mvapi", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_mpool_mvapi_open, /* component open */
NULL
},
/* Next the MCA v1.0.0 component meta data */
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_mpool_mvapi_init
}
};
static void mca_mpool_mvapi_registration_constructor( mca_mpool_mvapi_registration_t * registration )
{
registration->base_reg.base = NULL;
registration->base_reg.bound = NULL;
registration->base_reg.flags = 0;
}
static void mca_mpool_mvapi_registration_destructor( mca_mpool_mvapi_registration_t * registration )
{
registration->base_reg.base = NULL;
registration->base_reg.bound = NULL;
registration->base_reg.flags = 0;
}
OBJ_CLASS_INSTANCE(
mca_mpool_mvapi_registration_t,
mca_mpool_base_registration_t,
mca_mpool_mvapi_registration_constructor,
mca_mpool_mvapi_registration_destructor
);
/**
* component open/close/init function
*/
static int mca_mpool_mvapi_open(void)
{
/* register VAPI component parameters */
return OMPI_SUCCESS;
}
static mca_mpool_base_module_t* mca_mpool_mvapi_init(
struct mca_mpool_base_resources_t* resources)
{
mca_mpool_mvapi_module_t* mpool_module;
mca_base_param_reg_string(&mca_mpool_mvapi_component.super.mpool_version,
"rcache_name",
"The name of the registration cache the mpool should use",
false,
false,
"rb",
&(mca_mpool_mvapi_component.rcache_name));
mpool_module = (mca_mpool_mvapi_module_t*)malloc(sizeof(mca_mpool_mvapi_module_t));
mca_mpool_mvapi_module_init(mpool_module);
mpool_module->hca_pd = *resources;
return &mpool_module->super;
}

271
ompi/mca/mpool/mvapi/mpool_mvapi_module.c
Просмотреть файл

@ -1,271 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "opal/util/output.h"
#include "opal/memoryhooks/memory.h"
#include "ompi/mca/mpool/mvapi/mpool_mvapi.h"
#include <vapi.h>
#include <vapi_types.h>
#include <vapi_common.h>
#include "ompi/mca/rcache/rcache.h"
#include "ompi/mca/rcache/base/base.h"
#include "ompi/mca/mpool/base/base.h"
#ifdef HAVE_MALLOC_H
#include <malloc.h>
#endif
/*
* Initializes the mpool module.
*/
void mca_mpool_mvapi_module_init(mca_mpool_mvapi_module_t* mpool)
{
mpool->super.mpool_component = &mca_mpool_mvapi_component.super;
mpool->super.mpool_base = NULL; /* no base .. */
mpool->super.mpool_alloc = mca_mpool_mvapi_alloc;
mpool->super.mpool_realloc = mca_mpool_mvapi_realloc;
mpool->super.mpool_free = mca_mpool_mvapi_free;
mpool->super.mpool_register = mca_mpool_mvapi_register;
mpool->super.mpool_deregister = mca_mpool_mvapi_deregister;
mpool->super.mpool_find = mca_mpool_mvapi_find;
mpool->super.mpool_retain = mca_mpool_mvapi_retain;
mpool->super.mpool_release = mca_mpool_mvapi_release;
mpool->super.mpool_finalize = NULL;
mpool->super.mpool_ft_event = mca_mpool_mvapi_ft_event;
mpool->super.rcache =
mca_rcache_base_module_create(mca_mpool_mvapi_component.rcache_name);
mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM;
OBJ_CONSTRUCT(&mpool->reg_list, ompi_free_list_t);
ompi_free_list_init(&mpool->reg_list, sizeof(mca_mpool_mvapi_registration_t),
OBJ_CLASS(mca_mpool_mvapi_registration_t), 0, -1, 32, NULL);
}
/**
* allocate function
*/
void* mca_mpool_mvapi_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
void* addr_malloc = (void*)malloc(size + mca_mpool_base_page_size);
void* addr = (void*) up_align_addr(addr_malloc, mca_mpool_base_page_size_log);
if(OMPI_SUCCESS != mpool->mpool_register(mpool, addr, size, flags, registration)) {
free(addr_malloc);
return NULL;
}
(*registration)->alloc_base = addr_malloc;
return addr;
}
/*
* register memory
*/
int mca_mpool_mvapi_register(
mca_mpool_base_module_t* mpool,
void *addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
mca_mpool_mvapi_module_t * mpool_module = (mca_mpool_mvapi_module_t*) mpool;
mca_mpool_mvapi_registration_t * vapi_reg;
VAPI_mrw_t mr_in, mr_out;
VAPI_ret_t ret;
ompi_free_list_item_t *item;
int rc;
assert(size > 0);
memset(&mr_in, 0, sizeof(VAPI_mrw_t));
memset(&mr_out, 0, sizeof(VAPI_mrw_t));
OMPI_FREE_LIST_GET(&mpool_module->reg_list, item, rc);
if(OMPI_SUCCESS != rc) {
return rc;
}
vapi_reg = (mca_mpool_mvapi_registration_t*) item;
vapi_reg->base_reg.mpool = mpool;
vapi_reg->base_reg.flags = flags;
vapi_reg->hndl = VAPI_INVAL_HNDL;
*registration = &vapi_reg->base_reg;
mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE | VAPI_EN_REMOTE_READ;
mr_in.l_key = 0;
mr_in.r_key = 0;
mr_in.pd_hndl = mpool_module->hca_pd.pd_tag;
vapi_reg->base_reg.base = down_align_addr(addr, mca_mpool_base_page_size_log);
vapi_reg->base_reg.bound = up_align_addr((void*) ((unsigned long) addr + size - 1)
, mca_mpool_base_page_size_log);
mr_in.size = vapi_reg->base_reg.bound - vapi_reg->base_reg.base + 1;
mr_in.start = (VAPI_virt_addr_t) (MT_virt_addr_t) vapi_reg->base_reg.base;
mr_in.type = VAPI_MR;
/* printf("registering addr %p size %d base %p bound %p\n", */
/* addr, size, vapi_reg->base_reg.base, vapi_reg->base_reg.bound); */
ret = VAPI_register_mr(
mpool_module->hca_pd.hca,
&mr_in,
&vapi_reg->hndl,
&mr_out
);
if(VAPI_OK != ret){
opal_output(0, "error registering memory of size %d: %s ", size, VAPI_strerror(ret));
OMPI_FREE_LIST_RETURN(&mpool_module->reg_list, item);
return OMPI_ERROR;
}
vapi_reg->l_key = mr_out.l_key;
vapi_reg->r_key = mr_out.r_key;
assert(vapi_reg->base_reg.bound - vapi_reg->base_reg.base >= 0);
if(flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_insert(mpool->rcache,
(mca_mpool_base_registration_t*) vapi_reg,
flags);
}
OPAL_THREAD_ADD32(&vapi_reg->base_reg.ref_count, 1);
return OMPI_SUCCESS;
}
/*
* Remove the registration from the rcache immediately and
* deregister the memory when the reference count goes to zero.
*/
int mca_mpool_mvapi_deregister(mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration)
{
if(registration->flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_delete(mpool->rcache,
registration,
registration->flags);
registration->flags = 0;
}
return mca_mpool_mvapi_release(mpool, registration);
}
/**
* realloc function
*/
void* mca_mpool_mvapi_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration)
{
mca_mpool_base_registration_t* old_reg = *registration;
void* new_mem = mpool->mpool_alloc(mpool, size, 0, old_reg->flags, registration);
memcpy(new_mem, addr, old_reg->bound - old_reg->base);
mpool->mpool_free(mpool, addr, old_reg);
return new_mem;
}
/**
* free function
*/
void mca_mpool_mvapi_free(mca_mpool_base_module_t* mpool, void * addr,
mca_mpool_base_registration_t* registration)
{
mpool->mpool_deregister(mpool, registration);
free(registration->alloc_base);
}
int mca_mpool_mvapi_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t *cnt
){
return mpool->rcache->rcache_find(mpool->rcache,
addr,
size,
regs,
cnt);
}
int mca_mpool_mvapi_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration
)
{
if(0 >= OPAL_THREAD_ADD32(&registration->ref_count, -1)) {
VAPI_ret_t ret;
mca_mpool_mvapi_module_t * mpool_mvapi = (mca_mpool_mvapi_module_t*) mpool;
mca_mpool_mvapi_registration_t * vapi_reg;
vapi_reg = (mca_mpool_mvapi_registration_t*) registration;
ret = VAPI_deregister_mr(
mpool_mvapi->hca_pd.hca,
vapi_reg->hndl
);
if(VAPI_OK != ret){
opal_output(0, "%s: error unpinning vapi memory\n", __func__);
return OMPI_ERROR;
}
OMPI_FREE_LIST_RETURN(&mpool_mvapi->reg_list, (ompi_free_list_item_t*) vapi_reg);
}
return OMPI_SUCCESS;
}
int mca_mpool_mvapi_retain(struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration
)
{
OPAL_THREAD_ADD32(&registration->ref_count, 1);
return OMPI_SUCCESS;
}
int mca_mpool_mvapi_ft_event(int state) {
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
;
}
else if(OPAL_CRS_RESTART == state) {
;
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}

1
ompi/mca/mpool/openib/.ompi_ignore
Просмотреть файл

@ -1 +0,0 @@
quilt

59
ompi/mca/mpool/openib/Makefile.am
Просмотреть файл

@ -1,59 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
AM_CPPFLAGS = $(mpool_openib_CPPFLAGS)
sources = \
mpool_openib.h \
mpool_openib_module.c \
mpool_openib_component.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_mpool_openib_DSO
component_noinst =
component_install = mca_mpool_openib.la
else
component_noinst = libmca_mpool_openib.la
component_install =
endif
# See src/mca/ptl/openib/Makefile.am for an explanation of
# libmca_common_openib.la.
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mpool_openib_la_SOURCES = $(sources)
mca_mpool_openib_la_LDFLAGS = -module -avoid-version $(mpool_openib_LDFLAGS)
mca_mpool_openib_la_LIBADD = \
$(mpool_openib_LIBS) \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/libopen-rte.la \
$(top_ompi_builddir)/opal/libopen-pal.la
noinst_LTLIBRARIES = $(component_noinst)
libmca_mpool_openib_la_SOURCES = $(sources)
libmca_mpool_openib_la_LDFLAGS = -module -avoid-version $(mpool_openib_LDFLAGS)
libmca_mpool_openib_la_LIBADD = $(mpool_openib_LIBS)

40
ompi/mca/mpool/openib/configure.m4
Просмотреть файл

@ -1,40 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_mpool_openib_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_mpool_openib_CONFIG],[
OMPI_CHECK_OPENIB([mpool_openib],
[mpool_openib_happy="yes"],
[mpool_openib_happy="no"])
AS_IF([test "$mpool_openib_happy" = "yes"],
[mpool_openib_WRAPPER_EXTRA_LDFLAGS="$mpool_openib_LDFLAGS"
mpool_openib_WRAPPER_EXTRA_LIBS="$mpool_openib_LIBS"
$1],
[$2])
# substitute in the things needed to build openib
AC_SUBST([mpool_openib_CFLAGS])
AC_SUBST([mpool_openib_CPPFLAGS])
AC_SUBST([mpool_openib_LDFLAGS])
AC_SUBST([mpool_openib_LIBS])
])dnl

24
ompi/mca/mpool/openib/configure.params
Просмотреть файл

@ -1,24 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007 Los Alamos National Security, LLC. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_CONFIG_FILES="Makefile"

156
ompi/mca/mpool/openib/mpool_openib.h
Просмотреть файл

@ -1,156 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_MPOOL_VAPI_H
#define MCA_MPOOL_VAPI_H
#include "opal/class/opal_list.h"
#include "ompi/class/ompi_free_list.h"
#include "opal/event/event.h"
#include "ompi/mca/mpool/mpool.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
static inline void * DOWN_ALIGN_ADDR(void * addr, uint32_t cnt) {
return (void*)((uintptr_t)((unsigned char*) addr) & (~((uintptr_t)0) << (cnt)));
}
static inline void* ALIGN_ADDR(void* addr, uint32_t cnt ) {
DOWN_ALIGN_ADDR((((unsigned char*) addr) + ~(~((uintptr_t)0) << (cnt))), (cnt));
return addr;
}
struct mca_mpool_openib_component_t {
mca_mpool_base_component_t super;
char* rcache_name;
};
typedef struct mca_mpool_openib_component_t mca_mpool_openib_component_t;
OMPI_DECLSPEC extern mca_mpool_openib_component_t mca_mpool_openib_component;
struct mca_mpool_base_resources_t {
struct ibv_pd* ib_pd;
};
typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;
struct mca_mpool_openib_module_t {
mca_mpool_base_module_t super;
struct mca_mpool_base_resources_t resources;
ompi_free_list_t reg_list;
}; typedef struct mca_mpool_openib_module_t mca_mpool_openib_module_t;
struct mca_mpool_openib_registration_t {
mca_mpool_base_registration_t base_reg;
struct ibv_mr *mr;
};
typedef struct mca_mpool_openib_registration_t mca_mpool_openib_registration_t;
OBJ_CLASS_DECLARATION(mca_mpool_openib_registration_t);
/*
* Initializes the mpool module.
*/
void mca_mpool_openib_module_init(mca_mpool_openib_module_t* mpool);
/*
* Returns base address of shared memory mapping.
*/
void* mca_mpool_openib_base(mca_mpool_base_module_t*);
/**
* Allocate block of shared memory.
*/
void* mca_mpool_openib_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration);
/**
* realloc function typedef
*/
void* mca_mpool_openib_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration);
/**
* register function typedef
*/
int mca_mpool_openib_register(
mca_mpool_base_module_t* mpool,
void *addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration);
int mca_mpool_openib_deregister(
mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* );
/**
* free function typedef
*/
void mca_mpool_openib_free(mca_mpool_base_module_t* mpool,
void * addr,
mca_mpool_base_registration_t* registration);
int mca_mpool_openib_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t *cnt
);
int mca_mpool_openib_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registraion
);
int mca_mpool_openib_retain(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registraion
);
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae
* @return OMPI_SUCCESS or failure status
*/
int mca_mpool_openib_ft_event(int state);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

125
ompi/mca/mpool/openib/mpool_openib_component.c
Просмотреть файл

@ -1,125 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "mpool_openib.h"
#include "orte/util/proc_info.h"
#include "orte/util/sys_info.h"
#include <unistd.h>
#include <malloc.h>
/*
* Local functions
*/
static int mca_mpool_openib_open(void);
static mca_mpool_base_module_t* mca_mpool_openib_init(
struct mca_mpool_base_resources_t* resources);
mca_mpool_openib_component_t mca_mpool_openib_component = {
{
/* First, the mca_base_component_t struct containing meta
information about the component itself */
{
/* Indicate that we are a mpool v1.0.0 component (which also
implies a specific MCA version) */
MCA_MPOOL_BASE_VERSION_1_0_0,
"openib", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_mpool_openib_open, /* component open */
NULL
},
/* Next the MCA v1.0.0 component meta data */
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_mpool_openib_init
}
};
static void mca_mpool_openib_registration_constructor( mca_mpool_openib_registration_t * registration )
{
registration->base_reg.base = NULL;
registration->base_reg.bound = NULL;
registration->base_reg.flags = 0;
}
static void mca_mpool_openib_registration_destructor( mca_mpool_openib_registration_t * registration )
{
registration->base_reg.base = NULL;
registration->base_reg.bound = NULL;
registration->base_reg.flags = 0;
}
OBJ_CLASS_INSTANCE(
mca_mpool_openib_registration_t,
mca_mpool_base_registration_t,
mca_mpool_openib_registration_constructor,
mca_mpool_openib_registration_destructor
);
/**
* component open/close/init function
*/
static int mca_mpool_openib_open(void)
{
mca_base_param_reg_string(&mca_mpool_openib_component.super.mpool_version,
"rcache_name",
"The name of the registration cache the mpool should use",
false,
false,
"rb",
&(mca_mpool_openib_component.rcache_name));
return OMPI_SUCCESS;
}
static mca_mpool_base_module_t* mca_mpool_openib_init(
struct mca_mpool_base_resources_t* resources)
{
mca_mpool_openib_module_t* mpool_module;
mpool_module = (mca_mpool_openib_module_t*)malloc(sizeof(mca_mpool_openib_module_t));
mca_mpool_openib_module_init(mpool_module);
mpool_module->resources = *resources;
return &mpool_module->super;
}

247
ompi/mca/mpool/openib/mpool_openib_module.c
Просмотреть файл

@ -1,247 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include "ompi/mca/mpool/openib/mpool_openib.h"
#include <infiniband/verbs.h>
#include <errno.h>
#include <string.h>
#include <malloc.h>
#include "ompi/mca/rcache/rcache.h"
#include "ompi/mca/rcache/base/base.h"
#include "ompi/mca/mpool/base/base.h"
/*
* Initializes the mpool module.
*/
void mca_mpool_openib_module_init(mca_mpool_openib_module_t* mpool)
{
mpool->super.mpool_component = &mca_mpool_openib_component.super;
mpool->super.mpool_base = NULL; /* no base .. */
mpool->super.mpool_alloc = mca_mpool_openib_alloc;
mpool->super.mpool_realloc = mca_mpool_openib_realloc;
mpool->super.mpool_free = mca_mpool_openib_free;
mpool->super.mpool_register = mca_mpool_openib_register;
mpool->super.mpool_deregister = mca_mpool_openib_deregister;
mpool->super.mpool_find = mca_mpool_openib_find;
mpool->super.mpool_retain = mca_mpool_openib_retain;
mpool->super.mpool_release = mca_mpool_openib_release;
mpool->super.mpool_finalize = NULL;
mpool->super.mpool_ft_event = mca_mpool_openib_ft_event;
mpool->super.rcache =
mca_rcache_base_module_create(mca_mpool_openib_component.rcache_name);
mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM;
OBJ_CONSTRUCT(&mpool->reg_list, ompi_free_list_t);
ompi_free_list_init(&mpool->reg_list, sizeof(mca_mpool_openib_registration_t),
OBJ_CLASS(mca_mpool_openib_registration_t), 0, -1, 32, NULL);
}
/**
* allocate function
*/
void* mca_mpool_openib_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
void* addr_malloc = (void*)memalign(mca_mpool_base_page_size, size);
void* addr = addr_malloc;
if(OMPI_SUCCESS != mpool->mpool_register(mpool, addr, size, flags, registration)) {
free(addr_malloc);
return NULL;
}
(*registration)->alloc_base = addr_malloc;
return addr;
}
/*
* register memory
*/
int mca_mpool_openib_register(mca_mpool_base_module_t* mpool,
void *addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration){
mca_mpool_openib_module_t * mpool_module = (mca_mpool_openib_module_t*) mpool;
mca_mpool_openib_registration_t * vapi_reg;
ompi_free_list_item_t *item;
int rc;
OMPI_FREE_LIST_GET(&mpool_module->reg_list, item, rc);
if(OMPI_SUCCESS != rc) {
return rc;
}
vapi_reg = (mca_mpool_openib_registration_t*) item;
*registration = &vapi_reg->base_reg;
vapi_reg->base_reg.mpool = mpool;
vapi_reg->base_reg.base = down_align_addr(addr, mca_mpool_base_page_size_log);
vapi_reg->base_reg.bound = up_align_addr( (void*) ((char*) addr + size - 1)
, mca_mpool_base_page_size_log);
size = vapi_reg->base_reg.bound -vapi_reg->base_reg.base + 1;
vapi_reg->mr = ibv_reg_mr(
mpool_module->resources.ib_pd,
vapi_reg->base_reg.base,
size,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ
/* IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE */
);
if(NULL == vapi_reg->mr){
return OMPI_ERROR;
}
if(flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_insert(mpool->rcache,
(mca_mpool_base_registration_t*) vapi_reg,
flags);
}
vapi_reg->base_reg.flags = flags;
mca_mpool_openib_retain(mpool,
(mca_mpool_base_registration_t*) vapi_reg);
return OMPI_SUCCESS;
}
/*
* deregister memory
*/
int mca_mpool_openib_deregister(mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration){
if(registration->flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_delete(mpool->rcache,
registration,
registration->flags);
registration->flags=0;
}
return mca_mpool_openib_release(mpool, registration);
}
/**
* realloc function
*/
void* mca_mpool_openib_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration)
{
mca_mpool_base_registration_t* old_reg = *registration;
void* new_mem = mpool->mpool_alloc(mpool, size, 0,0, registration);
memcpy(new_mem, addr, old_reg->bound - old_reg->base);
mpool->mpool_free(mpool, addr, old_reg);
return new_mem;
}
/**
* free function
*/
void mca_mpool_openib_free(mca_mpool_base_module_t* mpool, void * addr,
mca_mpool_base_registration_t* registration)
{
mpool->mpool_deregister(mpool, registration);
free(registration->alloc_base);
}
int mca_mpool_openib_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t *cnt
){
return mpool->rcache->rcache_find(mpool->rcache,
addr,
size,
regs,
cnt);
}
int mca_mpool_openib_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration
){
if(0 >= OPAL_THREAD_ADD32(&registration->ref_count, -1)) {
mca_mpool_openib_registration_t * openib_reg;
mca_mpool_openib_module_t* mpool_openib =
(mca_mpool_openib_module_t*) mpool;
openib_reg = (mca_mpool_openib_registration_t*) registration;
if(ibv_dereg_mr(openib_reg->mr)){
opal_output(0, "%s: error unpinning openib memory errno says %s\n", __func__, strerror(errno));
return OMPI_ERROR;
}
OMPI_FREE_LIST_RETURN(&mpool_openib->reg_list, (ompi_free_list_item_t*) openib_reg);
}
return OMPI_SUCCESS;
}
int mca_mpool_openib_retain(struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration
){
OPAL_THREAD_ADD32(&registration->ref_count, 1);
return OMPI_SUCCESS;
}
int mca_mpool_openib_ft_event(int state) {
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
;
}
else if(OPAL_CRS_RESTART == state) {
;
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}

0
ompi/mca/mpool/openib/mvapi_config.h
Просмотреть файл

0
ompi/mca/mpool/openib/mvapi_config.h.in
Просмотреть файл

1
ompi/mca/mpool/udapl/.ompi_ignore
Просмотреть файл

@ -1 +0,0 @@
quilt

53
ompi/mca/mpool/udapl/Makefile.am
Просмотреть файл

@ -1,53 +0,0 @@
#
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
CFLAGS = $(mpool_udapl_CFLAGS)
AM_CPPFLAGS = $(mpool_udapl_CPPFLAGS)
sources = \
mpool_udapl.h \
mpool_udapl_module.c \
mpool_udapl_component.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_mpool_udapl_DSO
component_noinst =
component_install = mca_mpool_udapl.la
else
component_noinst = libmca_mpool_udapl.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mpool_udapl_la_SOURCES = $(sources)
mca_mpool_udapl_la_LIBADD = \
$(mpool_udapl_LIBS) \
$(top_ompi_builddir)/ompi/libmpi.la \
$(top_ompi_builddir)/orte/libopen-rte.la \
$(top_ompi_builddir)/opal/libopen-pal.la
mca_mpool_udapl_la_LDFLAGS = -module -avoid-version $(mpool_udapl_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_mpool_udapl_la_SOURCES = $(sources)
libmca_mpool_udapl_la_LIBADD = $(mpool_udapl_LIBS)
libmca_mpool_udapl_la_LDFLAGS = -module -avoid-version $(mpool_udapl_LDFLAGS)

47
ompi/mca/mpool/udapl/configure.m4
Просмотреть файл

@ -1,47 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_mpool_udapl_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_mpool_udapl_CONFIG],[
OMPI_CHECK_UDAPL([mpool_udapl],
[mpool_udapl_happy="yes"],
[mpool_udapl_happy="no"])
AS_IF([test "$mpool_udapl_happy" = "yes"],
[mpool_udapl_WRAPPER_EXTRA_LDFLAGS="$mpool_udapl_LDFLAGS"
mpool_udapl_WRAPPER_EXTRA_LIBS="$mpool_udapl_LIBS"
$1],
[$2])
# A data structure in the uDAPL headers
# is not fully ISO C. Remove -pedantic to silence a warning.
mpool_udapl_CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
AS_IF([test "$mpool_udapl_CFLAGS" != "$CFLAGS" -a "$mpool_udapl_happy" = "yes"],
[AC_MSG_WARN([Removed -pedantic from CFLAGS for
uDAPL component because the uDAPL headers are not fully ISO C])])
# substitute in the things needed to build uDAPL
AC_SUBST([mpool_udapl_CFLAGS])
AC_SUBST([mpool_udapl_CPPFLAGS])
AC_SUBST([mpool_udapl_LDFLAGS])
AC_SUBST([mpool_udapl_LIBS])
])dnl

25
ompi/mca/mpool/udapl/configure.params
Просмотреть файл

@ -1,25 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2007 Los Alamos National Security, LLC. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_CONFIG_FILES="Makefile"

168
ompi/mca/mpool/udapl/mpool_udapl.h
Просмотреть файл

@ -1,168 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_MPOOL_UDAPL_H
#define MCA_MPOOL_UDAPL_H
#include "ompi_config.h"
#include <dat/udat.h>
#include "opal/class/opal_list.h"
#include "ompi/class/ompi_free_list.h"
#include "opal/event/event.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/mca/allocator/allocator.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_mpool_udapl_component_t {
mca_mpool_base_component_t super;
char* rcache_name;
};
typedef struct mca_mpool_udapl_component_t mca_mpool_udapl_component_t;
OMPI_DECLSPEC extern mca_mpool_udapl_component_t mca_mpool_udapl_component;
struct mca_mpool_base_resources_t {
DAT_IA_HANDLE udapl_ia; /* interface adapter */
DAT_PZ_HANDLE udapl_pz; /* protection zone */
};
typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;
struct mca_mpool_udapl_module_t {
mca_mpool_base_module_t super;
struct mca_mpool_base_resources_t udapl_res;
};
typedef struct mca_mpool_udapl_module_t mca_mpool_udapl_module_t;
struct mca_mpool_udapl_registration_t {
mca_mpool_base_registration_t base_reg;
DAT_LMR_HANDLE lmr; /* local memory region (LMR) */
DAT_LMR_TRIPLET lmr_triplet; /* LMR triplet - context, address, length */
DAT_RMR_CONTEXT rmr_context; /* remote memory region context handle */
};
typedef struct mca_mpool_udapl_registration_t mca_mpool_udapl_registration_t;
OBJ_CLASS_DECLARATION(mca_mpool_udapl_registration_t);
/**
* Report a uDAPL error - for debugging
*/
#if OMPI_ENABLE_DEBUG
extern void mca_mpool_udapl_error(DAT_RETURN ret, char* str);
#define MCA_MPOOL_UDAPL_ERROR(ret, str) \
mca_mpool_udapl_error((ret), (str));
#else
#define MCA_MPOOL_UDAPL_ERROR(ret, str)
#endif
/*
* Initializes the mpool module.
*/
void mca_mpool_udapl_module_init(mca_mpool_udapl_module_t* mpool);
/*
* Returns base address of shared memory mapping.
*/
void* mca_mpool_udapl_base(mca_mpool_base_module_t*);
/**
* Allocate block of shared memory.
*/
void* mca_mpool_udapl_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration);
/**
* realloc function typedef
*/
void* mca_mpool_udapl_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration);
/**
* register function typedef
*/
int mca_mpool_udapl_register(
mca_mpool_base_module_t* mpool,
void *addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration);
int mca_mpool_udapl_deregister(
mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t*);
/**
* free function typedef
*/
void mca_mpool_udapl_free(
mca_mpool_base_module_t* mpool,
void * addr,
mca_mpool_base_registration_t* registration);
int mca_mpool_udapl_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t *cnt
);
int mca_mpool_udapl_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration
);
int mca_mpool_udapl_retain(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration
);
void* mca_common_udapl_segment_alloc(
struct mca_mpool_base_module_t* module,
size_t* size,
mca_mpool_base_registration_t** registration);
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae
* @return OMPI_SUCCESS or failure status
*/
int mca_mpool_udapl_ft_event(int state);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

145
ompi/mca/mpool/udapl/mpool_udapl_component.c
Просмотреть файл

@ -1,145 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/mca/allocator/base/base.h"
#include "mpool_udapl.h"
#include "orte/util/proc_info.h"
#include "orte/util/sys_info.h"
#include <unistd.h>
/*
* Local functions
*/
static int mca_mpool_udapl_open(void);
static mca_mpool_base_module_t* mca_mpool_udapl_init(
struct mca_mpool_base_resources_t* resources);
mca_mpool_udapl_component_t mca_mpool_udapl_component = {
{
/* First, the mca_base_component_t struct containing meta
information about the component itself */
{
/* Indicate that we are a mpool v1.0.0 component (which also
implies a specific MCA version) */
MCA_MPOOL_BASE_VERSION_1_0_0,
"udapl", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_mpool_udapl_open, /* component open */
NULL /* component close */
},
/* Next the MCA v1.0.0 component meta data */
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_mpool_udapl_init
}
};
/**
* Report a uDAPL error - for debugging
*/
#if OMPI_ENABLE_DEBUG
void
mca_mpool_udapl_error(DAT_RETURN ret, char* str)
{
char* major;
char* minor;
if(DAT_SUCCESS != dat_strerror(ret,
(const char**)&major, (const char**)&minor))
{
printf("dat_strerror failed! ret is %d\n", ret);
exit(-1);
}
OPAL_OUTPUT((0, "ERROR: %s %s %s\n", str, major, minor));
}
#endif
/**
* udapl_registration_t constructor/destructor
*/
static void mca_mpool_udapl_registration_constructor(mca_mpool_udapl_registration_t *registration)
{
registration->base_reg.base = NULL;
registration->base_reg.bound = NULL;
registration->base_reg.flags = 0;
}
static void mca_mpool_udapl_registration_destructor(mca_mpool_udapl_registration_t *registration)
{
#if OMPI_ENABLE_DEBUG
registration->base_reg.base = NULL;
registration->base_reg.bound = NULL;
registration->base_reg.flags = 0;
#endif
}
OBJ_CLASS_INSTANCE(mca_mpool_udapl_registration_t,
mca_mpool_base_registration_t,
mca_mpool_udapl_registration_constructor,
mca_mpool_udapl_registration_destructor);
/**
* component open/init functions
*/
static int mca_mpool_udapl_open(void)
{
return OMPI_SUCCESS;
}
static mca_mpool_base_module_t* mca_mpool_udapl_init(
struct mca_mpool_base_resources_t* resources)
{
mca_mpool_udapl_module_t* udapl_mpool;
mca_base_param_reg_string(&mca_mpool_udapl_component.super.mpool_version,
"rcache_name", "The name of the registration cache the mpool should use",
false, false, "rb", &(mca_mpool_udapl_component.rcache_name));
udapl_mpool = (mca_mpool_udapl_module_t*)malloc(sizeof(mca_mpool_udapl_module_t));
if(NULL == udapl_mpool) {
return NULL;
}
mca_mpool_udapl_module_init(udapl_mpool);
udapl_mpool->udapl_res = *resources;
return &udapl_mpool->super;
}

240
ompi/mca/mpool/udapl/mpool_udapl_module.c
Просмотреть файл

@ -1,240 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006 Sandia National Laboratories. All rights
* reserved.
* Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "opal/util/output.h"
#include "mpool_udapl.h"
#include "ompi/mca/rcache/rcache.h"
#include "ompi/mca/rcache/base/base.h"
#include "ompi/mca/mpool/base/base.h"
uint64_t mca_mpool_udapl_mem_registered;
/*
* Initializes the mpool module.
*/
void mca_mpool_udapl_module_init(mca_mpool_udapl_module_t* mpool)
{
mpool->super.mpool_component = &mca_mpool_udapl_component.super;
mpool->super.mpool_base = NULL; /* no base .. */
mpool->super.mpool_alloc = mca_mpool_udapl_alloc;
mpool->super.mpool_realloc = mca_mpool_udapl_realloc;
mpool->super.mpool_free = mca_mpool_udapl_free;
mpool->super.mpool_register = mca_mpool_udapl_register;
mpool->super.mpool_deregister = mca_mpool_udapl_deregister;
mpool->super.mpool_find = mca_mpool_udapl_find;
mpool->super.mpool_retain = mca_mpool_udapl_retain;
mpool->super.mpool_release = mca_mpool_udapl_release;
mpool->super.mpool_finalize = NULL;
mpool->super.mpool_ft_event = mca_mpool_udapl_ft_event;
mpool->super.rcache =
mca_rcache_base_module_create(mca_mpool_udapl_component.rcache_name);
mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM;
mca_mpool_udapl_mem_registered = 0;
}
/**
* allocate function
*/
void* mca_mpool_udapl_alloc(
mca_mpool_base_module_t* mpool,
size_t size,
size_t align,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
void* addr = malloc(size+align);
if(NULL == addr)
return NULL;
if(OMPI_SUCCESS != mca_mpool_udapl_register(mpool,
addr, size + align, flags, registration)) {
free(addr);
return NULL;
}
(*registration)->alloc_base = addr;
return addr;
}
/*
* register memory
*/
int mca_mpool_udapl_register(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
uint32_t flags,
mca_mpool_base_registration_t** registration)
{
mca_mpool_udapl_module_t* udapl_mpool;
mca_mpool_udapl_registration_t* reg;
DAT_REGION_DESCRIPTION region;
DAT_VLEN dat_size;
DAT_VADDR dat_addr;
int rc;
udapl_mpool = (mca_mpool_udapl_module_t*)mpool;
reg = OBJ_NEW(mca_mpool_udapl_registration_t);
if(NULL == reg) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
reg->base_reg.mpool = mpool;
reg->base_reg.base = down_align_addr(addr, mca_mpool_base_page_size_log);
reg->base_reg.flags = flags;
reg->base_reg.bound = up_align_addr((void*)((unsigned long) addr + size -1),
mca_mpool_base_page_size_log);
region.for_va = addr;
reg->lmr_triplet.virtual_address = (DAT_VADDR)addr;
reg->lmr_triplet.segment_length = size;
rc = dat_lmr_create(udapl_mpool->udapl_res.udapl_ia,
DAT_MEM_TYPE_VIRTUAL, region, reg->lmr_triplet.segment_length,
udapl_mpool->udapl_res.udapl_pz, DAT_MEM_PRIV_ALL_FLAG,
&reg->lmr, &reg->lmr_triplet.lmr_context,
&reg->rmr_context, &dat_size, &dat_addr);
if(DAT_SUCCESS != rc) {
MCA_MPOOL_UDAPL_ERROR(rc, "dat_lmr_create");
OBJ_RELEASE(reg);
return OMPI_ERR_OUT_OF_RESOURCE;
}
OPAL_THREAD_ADD32(&reg->base_reg.ref_count,1);
if(flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_insert(mpool->rcache,
(mca_mpool_base_registration_t*) reg,
flags);
}
*registration = &reg->base_reg;
return OMPI_SUCCESS;
}
/*
* deregister memory
*/
int mca_mpool_udapl_deregister(mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* reg)
{
if(reg->flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) {
mpool->rcache->rcache_delete(mpool->rcache, reg, reg->flags);
reg->flags = 0;
}
return mca_mpool_udapl_release(mpool, reg);
}
/**
* realloc function
*/
void* mca_mpool_udapl_realloc(
mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
mca_mpool_base_registration_t** registration)
{
mca_mpool_base_registration_t* old_reg = *registration;
void *new_addr = mca_mpool_udapl_alloc(mpool,
size, 0, (*registration)->flags, registration);
if(new_addr == NULL) {
return NULL;
}
memcpy(new_addr, addr, size);
mca_mpool_udapl_free(mpool, addr, old_reg);
return new_addr;
}
/**
* free function
*/
void mca_mpool_udapl_free(mca_mpool_base_module_t* mpool, void * addr,
mca_mpool_base_registration_t* registration)
{
mpool->mpool_deregister(mpool, registration);
free(addr);
}
int mca_mpool_udapl_find(
struct mca_mpool_base_module_t* mpool,
void* addr,
size_t size,
ompi_pointer_array_t *regs,
uint32_t* cnt)
{
return mpool->rcache->rcache_find(mpool->rcache, addr, size, regs, cnt);
}
int mca_mpool_udapl_release(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* reg)
{
if(0 == OPAL_THREAD_ADD32(&reg->ref_count, -1)) {
mca_mpool_udapl_registration_t *udapl_reg =
(mca_mpool_udapl_registration_t*)reg;
int rc = dat_lmr_free(udapl_reg->lmr);
if(DAT_SUCCESS != rc) {
MCA_MPOOL_UDAPL_ERROR(rc, "dat_lmr_free");
return OMPI_ERROR;
}
OBJ_RELEASE(reg);
}
return OMPI_SUCCESS;
}
int mca_mpool_udapl_retain(
struct mca_mpool_base_module_t* mpool,
mca_mpool_base_registration_t* registration)
{
OPAL_THREAD_ADD32(&registration->ref_count, 1);
return OMPI_SUCCESS;
}
int mca_mpool_udapl_ft_event(int state) {
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
;
}
else if(OPAL_CRS_RESTART == state) {
;
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}