ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

start of a gm btl - not currently functional

Просмотр исходного кода 
This commit was SVN r6232.
Этот коммит содержится в:
Tim Woodall 2005-06-30 18:21:55 +00:00
родитель b12a3272e8
Коммит f7d2b120ef
15 изменённых файлов: 2353 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
src/mca/btl/gm/.ompi_ignore Обычный файл
Просмотреть файл

2
src/mca/btl/gm/.ompi_unignore Обычный файл
Просмотреть файл

@ -0,0 +1,2 @@
twoodall
gshipman

56
src/mca/btl/gm/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,56 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
include $(top_ompi_srcdir)/config/Makefile.options
sources = \
btl_gm.c \
btl_gm.h \
btl_gm_component.c \
btl_gm_endpoint.c \
btl_gm_endpoint.h \
btl_gm_frag.c \
btl_gm_frag.h \
btl_gm_proc.c \
btl_gm_proc.h \
btl_gm_error.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_btl_gm_DSO
lib =
lib_sources =
component = mca_btl_gm.la
component_sources = $(sources)
else
lib = libmca_btl_gm.la
lib_sources = $(sources)
component =
component_sources =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component)
mca_btl_gm_la_SOURCES = $(component_sources)
mca_btl_gm_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(lib)
libmca_btl_gm_la_SOURCES = $(lib_sources)
libmca_btl_gm_la_LDFLAGS = -module -avoid-version

645
src/mca/btl/gm/btl_gm.c Обычный файл
Просмотреть файл

@ -0,0 +1,645 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "util/output.h"
#include "util/if.h"
#include "mca/pml/pml.h"
#include "mca/btl/btl.h"
#include "btl_gm.h"
#include "btl_gm_frag.h"
#include "btl_gm_proc.h"
#include "btl_gm_endpoint.h"
#include "datatype/convertor.h"
#include "mca/mpool/base/base.h"
#include "mca/mpool/mpool.h"
mca_btl_gm_module_t mca_btl_gm_module = {
{
&mca_btl_gm_component.super,
0, /* max size of first fragment */
0, /* min send fragment size */
0, /* max send fragment size */
0, /* min rdma fragment size */
0, /* max rdma fragment size */
0, /* exclusivity */
0, /* latency */
0, /* bandwidth */
0, /* flags */
mca_btl_gm_add_procs,
mca_btl_gm_del_procs,
mca_btl_gm_register,
mca_btl_gm_finalize,
mca_btl_gm_alloc,
mca_btl_gm_free,
mca_btl_gm_prepare_src,
mca_btl_gm_prepare_dst,
mca_btl_gm_send,
mca_btl_gm_put,
NULL /* get */
}
};
/**
*
*/
int mca_btl_gm_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **ompi_procs,
struct mca_btl_base_endpoint_t** peers,
ompi_bitmap_t* reachable)
{
mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*)btl;
int i, rc;
for(i = 0; i < (int) nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_btl_gm_proc_t* gm_proc;
mca_btl_base_endpoint_t* gm_endpoint;
if(NULL == (gm_proc = mca_btl_gm_proc_create(ompi_proc))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*
* Check to make sure that the peer has at least as many interface
* addresses exported as we are trying to use. If not, then
* don't bind this PTL instance to the proc.
*/
OMPI_THREAD_LOCK(&gm_proc->proc_lock);
/* The btl_proc datastructure is shared by all GM PTL
* instances that are trying to reach this destination.
* Cache the peer instance on the btl_proc.
*/
gm_endpoint = OBJ_NEW(mca_btl_gm_endpoint_t);
if(NULL == gm_endpoint) {
OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
gm_endpoint->endpoint_btl = gm_btl;
rc = mca_btl_gm_proc_insert(gm_proc, gm_endpoint);
if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(gm_endpoint);
OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
continue;
}
ompi_bitmap_set_bit(reachable, i);
OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
peers[i] = gm_endpoint;
}
return OMPI_SUCCESS;
}
int mca_btl_gm_del_procs(struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t ** peers)
{
/* TODO */
return OMPI_SUCCESS;
}
/**
* Register callback function to support send/recv semantics
*/
int mca_btl_gm_register(
struct mca_btl_base_module_t* btl,
mca_btl_base_tag_t tag,
mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata)
{
mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*) btl;
gm_btl->gm_reg[tag].cbfunc = cbfunc;
gm_btl->gm_reg[tag].cbdata = cbdata;
return OMPI_SUCCESS;
}
/**
* Allocate a segment.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*/
mca_btl_base_descriptor_t* mca_btl_gm_alloc(
struct mca_btl_base_module_t* btl,
size_t size)
{
mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*) btl;
mca_btl_gm_frag_t* frag;
int rc;
if(size <= btl->btl_eager_limit){
MCA_BTL_GM_FRAG_ALLOC_EAGER(gm_btl, frag, rc);
frag->segment.seg_len =
size <= btl->btl_eager_limit ?
size : btl->btl_eager_limit ;
} else {
MCA_BTL_GM_FRAG_ALLOC_MAX(gm_btl, frag, rc);
frag->segment.seg_len =
size <= btl->btl_max_send_size ?
size : btl->btl_max_send_size ;
}
frag->base.des_flags = 0;
return (mca_btl_base_descriptor_t*)frag;
}
/**
* Return a segment
*/
int mca_btl_gm_free(
struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des)
{
mca_btl_gm_frag_t* frag = (mca_btl_gm_frag_t*)des;
if(frag->size == 0) {
OBJ_RELEASE(frag->registration);
MCA_BTL_GM_FRAG_RETURN_USER(btl, frag);
} else if(frag->size == btl->btl_eager_limit){
MCA_BTL_GM_FRAG_RETURN_EAGER(btl, frag);
} else if(frag->size == btl->btl_max_send_size) {
MCA_BTL_GM_FRAG_RETURN_EAGER(btl, frag);
} else {
return OMPI_ERR_BAD_PARAM;
}
return OMPI_SUCCESS;
}
/**
* Pack data and return a descriptor that can be
* used for send/put.
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
*/
mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor,
size_t reserve,
size_t* size
)
{
mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*)btl;
mca_btl_gm_frag_t* frag;
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data = *size;
int32_t free_after;
int rc;
/*
* If the data has already been pinned and is contigous than we can
* use it in place.
*/
if (NULL != registration && 0 == ompi_convertor_need_buffers(convertor)) {
size_t reg_len;
MCA_BTL_GM_FRAG_ALLOC_USER(gm_btl, frag, rc);
if(NULL == frag){
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
reg_len = (unsigned char*)registration->bound - (unsigned char*)iov.iov_base + 1;
if(frag->segment.seg_len > reg_len) {
mca_mpool_base_module_t* mpool = gm_btl->gm_mpool;
size_t new_len = (unsigned char*)iov.iov_base - registration->base + max_data;
void* base_addr = registration->base;
/* remove old registration from tree and decrement reference count */
mca_mpool_base_remove(base_addr);
OBJ_RELEASE(registration);
/* re-register at new size */
rc = mpool->mpool_register(
mpool,
base_addr,
new_len,
&registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
return NULL;
}
/* re-insert into tree with new registration */
rc = mca_mpool_base_insert(
base_addr,
new_len,
mpool,
btl,
registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
OBJ_RELEASE(registration);
return NULL;
}
}
/* bump reference count as so that the registration
* doesn't go away when the operation completes
*/
OBJ_RETAIN(registration);
frag->registration = registration;
/*
* if the data is not already pinned - but the leave pinned option is set,
* then go ahead and pin contigous data. however, if a reserve is required
* then we must allocated a fragment w/ buffer space
*/
} else if ((mca_btl_gm_component.leave_pinned || max_data > btl->btl_max_send_size) &&
ompi_convertor_need_buffers(convertor) == 0 &&
reserve == 0) {
mca_mpool_base_module_t* mpool = gm_btl->gm_mpool;
MCA_BTL_GM_FRAG_ALLOC_USER(gm_btl, frag, rc);
if(NULL == frag){
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
rc = mpool->mpool_register(
mpool,
iov.iov_base,
max_data,
&registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
return NULL;
}
if(mca_btl_gm_component.leave_pinned) {
/*
* insert the registration into the tree and bump the reference
* count so that it doesn't go away on completion.
*/
rc = mca_mpool_base_insert(
iov.iov_base,
iov.iov_len,
mpool,
btl,
registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
OBJ_RELEASE(registration);
return NULL;
}
OBJ_RETAIN(registration);
}
frag->registration = registration;
}
/*
* if we aren't pinning the data and the requested size is less
* than the eager limit pack into a fragment from the eager pool
*/
else if (max_data+reserve <= btl->btl_eager_limit) {
MCA_BTL_GM_FRAG_ALLOC_EAGER(btl, frag, rc);
if(NULL == frag) {
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
*size = max_data;
if( rc < 0 ) {
MCA_BTL_GM_FRAG_RETURN_EAGER(btl, frag);
return NULL;
}
frag->segment.seg_len = max_data + reserve;
}
/*
* otherwise pack as much data as we can into a fragment
* that is the max send size.
*/
else {
MCA_BTL_GM_FRAG_ALLOC_MAX(btl, frag, rc);
if(NULL == frag) {
return NULL;
}
if(max_data + reserve > frag->size){
max_data = frag->size - reserve;
}
iov.iov_len = max_data;
iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
*size = max_data;
if( rc < 0 ) {
MCA_BTL_GM_FRAG_RETURN_MAX(btl, frag);
return NULL;
}
frag->segment.seg_len = max_data + reserve;
}
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
return &frag->base;
}
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
mca_btl_base_descriptor_t* mca_btl_gm_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor,
size_t reserve,
size_t* size)
{
mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*) btl;
mca_btl_gm_frag_t* frag;
int rc;
MCA_BTL_GM_FRAG_ALLOC_USER(btl, frag, rc);
if(NULL == frag) {
return NULL;
}
frag->segment.seg_len = *size;
frag->segment.seg_addr.pval = convertor->pBaseBuf + convertor->bConverted;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = &frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_flags = 0;
if(NULL != registration) {
size_t reg_len = (unsigned char*)registration->bound - (unsigned char*)frag->segment.seg_addr.pval + 1;
if(frag->segment.seg_len > reg_len) {
mca_mpool_base_module_t* mpool = gm_btl->gm_mpool;
size_t new_len = (unsigned char*)frag->segment.seg_addr.pval - registration->base + frag->segment.seg_len;
void* base_addr = registration->base;
/* remove old registration from tree and decrement reference count */
mca_mpool_base_remove(base_addr);
OBJ_RELEASE(registration);
/* re-register at new size */
rc = mpool->mpool_register(
mpool,
base_addr,
new_len,
&registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
return NULL;
}
/* re-insert into tree with new registration */
rc = mca_mpool_base_insert(
base_addr,
new_len,
mpool,
btl,
registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
OBJ_RELEASE(registration);
return NULL;
}
}
/* bump reference count as so that the registration
* doesn't go away when the operation completes
*/
OBJ_RETAIN(registration);
frag->registration = registration;
} else {
mca_mpool_base_module_t* mpool = gm_btl->gm_mpool;
rc = mpool->mpool_register(
mpool,
frag->segment.seg_addr.pval,
frag->segment.seg_len,
&registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
return NULL;
}
if(mca_btl_gm_component.leave_pinned) {
/*
* insert the registration into the tree and bump the reference
* count so that it doesn't go away on completion.
*/
rc = mca_mpool_base_insert(
frag->segment.seg_addr.pval,
frag->segment.seg_len,
mpool,
btl,
registration);
if(rc != OMPI_SUCCESS) {
MCA_BTL_GM_FRAG_RETURN_USER(btl,frag);
OBJ_RELEASE(registration);
return NULL;
}
OBJ_RETAIN(registration);
}
frag->registration = registration;
}
return &frag->base;
}
/**
*
*/
static void mca_btl_gm_send_callback( struct gm_port* port, void* context, gm_status_t status )
{
mca_btl_gm_frag_t* frag = (mca_btl_gm_frag_t*)context;
mca_btl_gm_module_t* btl = frag->btl;
/* release the send token */
OMPI_THREAD_ADD32( &btl->gm_num_send_tokens, 1 );
/* call the completion callback */
frag->base.des_cbfunc(&btl->super, frag->endpoint, &frag->base, OMPI_SUCCESS);
}
/**
* Initiate an asynchronous send.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transfered
* @param tag (IN) The tag value used to notify the peer.
*/
int mca_btl_gm_send(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des,
mca_btl_base_tag_t tag)
{
mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*) btl;
mca_btl_gm_frag_t* frag = (mca_btl_gm_frag_t*)des;
frag->btl = gm_btl;
frag->endpoint = endpoint;
/* TODO - queue the descriptor if there are no send tokens */
assert(OMPI_THREAD_ADD32( &gm_btl->gm_num_send_tokens, -1 ) >= 0);
gm_send_with_callback(
gm_btl->gm_port,
frag->hdr,
frag->size,
frag->segment.seg_len + sizeof(mca_btl_base_header_t),
(des->des_flags & MCA_BTL_DES_FLAGS_PRIORITY) ? GM_HIGH_PRIORITY : GM_LOW_PRIORITY,
endpoint->endpoint_addr.local_id,
endpoint->endpoint_addr.port_id,
mca_btl_gm_send_callback,
frag);
return OMPI_SUCCESS;
}
/**
* Initiate an asynchronous put.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
int mca_btl_gm_put(
mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* descriptor)
{
/* mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*) btl; */
mca_btl_gm_frag_t* frag = (mca_btl_gm_frag_t*) descriptor;
frag->endpoint = endpoint;
/* TODO */
return OMPI_ERR_NOT_IMPLEMENTED;
}
/**
* Initiate an asynchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*
*/
int mca_btl_gm_get(
mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* descriptor)
{
/* mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*) btl; */
mca_btl_gm_frag_t* frag = (mca_btl_gm_frag_t*) descriptor;
frag->endpoint = endpoint;
/* TODO */
return OMPI_ERR_NOT_IMPLEMENTED;
}
/*
* Cleanup/release module resources.
*/
int mca_btl_gm_finalize(struct mca_btl_base_module_t* btl)
{
mca_btl_gm_module_t* gm_btl = (mca_btl_gm_module_t*) btl;
if(gm_btl->gm_frag_eager.fl_num_allocated !=
gm_btl->gm_frag_eager.super.ompi_list_length){
ompi_output(0, "btl gm_frag_eager: %d allocated %d returned \n",
gm_btl->gm_frag_eager.fl_num_allocated,
gm_btl->gm_frag_eager.super.ompi_list_length);
}
if(gm_btl->gm_frag_max.fl_num_allocated !=
gm_btl->gm_frag_max.super.ompi_list_length) {
ompi_output(0, "btl gm_frag_max: %d allocated %d returned \n",
gm_btl->gm_frag_max.fl_num_allocated,
gm_btl->gm_frag_max.super.ompi_list_length);
}
if(gm_btl->gm_frag_user.fl_num_allocated !=
gm_btl->gm_frag_user.super.ompi_list_length){
ompi_output(0, "btl gm_frag_user: %d allocated %d returned \n",
gm_btl->gm_frag_user.fl_num_allocated,
gm_btl->gm_frag_user.super.ompi_list_length);
}
OBJ_DESTRUCT(&gm_btl->gm_lock);
OBJ_DESTRUCT(&gm_btl->gm_frag_eager);
OBJ_DESTRUCT(&gm_btl->gm_frag_max);
OBJ_DESTRUCT(&gm_btl->gm_frag_user);
free(gm_btl);
return OMPI_SUCCESS;
}

310
src/mca/btl/gm/btl_gm.h Обычный файл
Просмотреть файл

@ -0,0 +1,310 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_BTL_GM_H
#define MCA_BTL_GM_H
/* Standard system includes */
#include <sys/types.h>
#include <string.h>
#include <gm.h>
/* Open MPI includes */
#include "class/ompi_free_list.h"
#include "class/ompi_bitmap.h"
#include "event/event.h"
#include "mca/pml/pml.h"
#include "mca/btl/btl.h"
#include "mca/btl/base/base.h"
#include "util/output.h"
#include "mca/mpool/mpool.h"
#include "mca/btl/btl.h"
#include "btl_gm_endpoint.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Infiniband (GM) BTL component.
*/
struct mca_btl_gm_component_t {
mca_btl_base_component_1_0_0_t super; /**< base BTL component */
size_t gm_num_btls; /**< number of hcas available to the GM component */
size_t gm_max_btls; /**< maximum number of supported hcas */
struct mca_btl_gm_module_t **gm_btls; /**< array of available BTL modules */
size_t gm_max_ports; /**< maximum number of ports per board */
size_t gm_max_boards; /**< maximum number of boards */
size_t gm_num_high_priority; /**< number of receive descriptors at high priority */
char* gm_port_name;
int gm_free_list_num; /**< initial size of free lists */
int gm_free_list_max; /**< maximum size of free lists */
int gm_free_list_inc; /**< number of elements to alloc when growing free lists */
ompi_list_t gm_procs; /**< list of gm proc structures */
ompi_mutex_t gm_lock; /**< lock for accessing module state */
char* gm_mpool_name; /**< name of memory pool */
bool leave_pinned;
/**< pin memory on first use and leave pinned */
};
typedef struct mca_btl_gm_component_t mca_btl_gm_component_t;
extern mca_btl_gm_component_t mca_btl_gm_component;
/**
* BTL Module Interface
*/
struct mca_btl_gm_module_t {
mca_btl_base_module_t super; /**< base BTL interface */
mca_btl_base_recv_reg_t gm_reg[256];
/* local port handle/address */
struct gm_port *gm_port;
mca_btl_gm_addr_t gm_addr;
/* free list of fragment descriptors */
ompi_free_list_t gm_frag_eager;
ompi_free_list_t gm_frag_max;
ompi_free_list_t gm_frag_user;
/* number of send/recv tokens */
unsigned int gm_num_send_tokens;
unsigned int gm_max_send_tokens;
unsigned int gm_num_recv_tokens;
unsigned int gm_max_recv_tokens;
/* lock for accessing module state */
ompi_mutex_t gm_lock;
struct mca_mpool_base_module_t* gm_mpool;
};
typedef struct mca_btl_gm_module_t mca_btl_gm_module_t;
extern mca_btl_gm_module_t mca_btl_gm_module;
/**
* Register GM component parameters with the MCA framework
*/
extern int mca_btl_gm_component_open(void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_btl_gm_component_close(void);
/**
* GM component initialization.
*
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
*/
extern mca_btl_base_module_t** mca_btl_gm_component_init(
int *num_btl_modules,
bool allow_multi_user_threads,
bool have_hidden_threads
);
/**
* GM component progress.
*/
extern int mca_btl_gm_component_progress(void);
/**
* Cleanup any resources held by the BTL.
*
* @param btl BTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_btl_gm_finalize(
struct mca_btl_base_module_t* btl
);
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN)
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info.
* @param peers (IN/OUT) Set of processes that are reachable via this BTL.
* @return OMPI_SUCCESS or error status on failure.
*
*/
extern int mca_btl_gm_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers,
ompi_bitmap_t* reachable
);
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN) BTL instance
* @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful
*
*/
extern int mca_btl_gm_del_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers
);
/**
* Initiate an asynchronous send.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transfered
* @param tag (IN) The tag value used to notify the peer.
*/
extern int mca_btl_gm_send(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag
);
/**
* Initiate an asynchronous put.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
extern int mca_btl_gm_put(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* decriptor
);
/**
* Initiate an asynchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
extern int mca_btl_gm_get(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* decriptor
);
/**
* Register a callback function that is called on receipt
* of a fragment.
*
* @param btl (IN) BTL module
* @return Status indicating if registration was successful
*
*/
extern int mca_btl_gm_register(
struct mca_btl_base_module_t* btl,
mca_btl_base_tag_t tag,
mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata);
/**
* Allocate a descriptor with a segment of the requested size.
* Note that the BTL layer may choose to return a smaller size
* if it cannot support the request.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*/
extern mca_btl_base_descriptor_t* mca_btl_gm_alloc(
struct mca_btl_base_module_t* btl,
size_t size);
/**
* Return a segment allocated by this BTL.
*
* @param btl (IN) BTL module
* @param descriptor (IN) Allocated descriptor.
*/
extern int mca_btl_gm_free(
struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des);
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct ompi_convertor_t* convertor,
size_t reserve,
size_t* size
);
extern mca_btl_base_descriptor_t* mca_btl_gm_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct ompi_convertor_t* convertor,
size_t reserve,
size_t* size);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

462
src/mca/btl/gm/btl_gm_component.c Обычный файл
Просмотреть файл

@ -0,0 +1,462 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "include/constants.h"
#include "event/event.h"
#include "util/if.h"
#include "util/argv.h"
#include "util/output.h"
#include "mca/pml/pml.h"
#include "mca/btl/btl.h"
#include "mca/base/mca_base_param.h"
#include "mca/base/mca_base_module_exchange.h"
#include "mca/errmgr/errmgr.h"
#include "mca/mpool/base/base.h"
#include "mca/mpool/gm/mpool_gm.h"
#include "btl_gm.h"
#include "btl_gm_frag.h"
#include "btl_gm_endpoint.h"
#include "mca/btl/base/base.h"
#include "datatype/convertor.h"
#include "btl_gm_endpoint.h"
mca_btl_gm_component_t mca_btl_gm_component = {
{
/* First, the mca_base_component_t struct containing meta information
about the component itself */
{
/* Indicate that we are a pml v1.0.0 component (which also implies a
specific MCA version) */
MCA_BTL_BASE_VERSION_1_0_0,
"ib", /* MCA component name */
1, /* MCA component major version */
0, /* MCA component minor version */
0, /* MCA component release version */
mca_btl_gm_component_open, /* component open */
mca_btl_gm_component_close /* component close */
},
/* Next the MCA v1.0.0 component meta data */
{
/* Whether the component is checkpointable or not */
false
},
mca_btl_gm_component_init,
mca_btl_gm_component_progress,
}
};
/*
* utility routines for parameter registration
*/
static inline char* mca_btl_gm_param_register_string(
const char* param_name,
const char* default_value)
{
char *param_value;
int id = mca_base_param_register_string("btl","ib",param_name,NULL,default_value);
mca_base_param_lookup_string(id, &param_value);
return param_value;
}
static inline int mca_btl_gm_param_register_int(
const char* param_name,
int default_value)
{
int id = mca_base_param_register_int("btl","ib",param_name,NULL,default_value);
int param_value = default_value;
mca_base_param_lookup_int(id,&param_value);
return param_value;
}
/*
* Called by MCA framework to open the component, registers
* component parameters.
*/
int mca_btl_gm_component_open(void)
{
/* initialize state */
mca_btl_gm_component.gm_num_btls=0;
mca_btl_gm_component.gm_btls=NULL;
/* initialize objects */
OBJ_CONSTRUCT(&mca_btl_gm_component.gm_procs, ompi_list_t);
/* register GM component parameters */
mca_btl_gm_component.gm_free_list_num =
mca_btl_gm_param_register_int ("free_list_num", 8);
mca_btl_gm_component.gm_free_list_max =
mca_btl_gm_param_register_int ("free_list_max", 1024);
mca_btl_gm_component.gm_free_list_inc =
mca_btl_gm_param_register_int ("free_list_inc", 32);
mca_btl_gm_component.gm_mpool_name =
mca_btl_gm_param_register_string("mpool", "gm");
/* register gm module parameters */
mca_btl_gm_module.super.btl_exclusivity =
mca_btl_gm_param_register_int ("exclusivity", 0);
mca_btl_gm_module.super.btl_eager_limit =
mca_btl_gm_param_register_int ("first_frag_size", 64*1024) - sizeof(mca_btl_base_header_t);
mca_btl_gm_module.super.btl_min_send_size =
mca_btl_gm_param_register_int ("min_send_size", 64*1024) - sizeof(mca_btl_base_header_t);
mca_btl_gm_module.super.btl_max_send_size =
mca_btl_gm_param_register_int ("max_send_size", 128*1024) - sizeof(mca_btl_base_header_t);
mca_btl_gm_module.super.btl_min_rdma_size =
mca_btl_gm_param_register_int("min_rdma_size", 1024*1024);
mca_btl_gm_module.super.btl_max_rdma_size =
mca_btl_gm_param_register_int("max_rdma_size", 1024*1024);
mca_btl_gm_module.super.btl_flags =
mca_btl_gm_param_register_int("flags", MCA_BTL_FLAGS_RDMA);
return OMPI_SUCCESS;
}
/*
* component cleanup - sanity checking of queue lengths
*/
int mca_btl_gm_component_close(void)
{
return OMPI_SUCCESS;
}
/**
* Initialize module instance
*/
static int
mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
{
mca_mpool_base_resources_t resources;
size_t i;
int rc;
/* initialize objects */
OBJ_CONSTRUCT(&btl->gm_frag_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->gm_frag_max, ompi_free_list_t);
OBJ_CONSTRUCT(&btl->gm_frag_user, ompi_list_t);
OBJ_CONSTRUCT(&btl->gm_lock, ompi_mutex_t);
/* query nic tokens */
btl->gm_num_send_tokens = gm_num_send_tokens (btl->gm_port);
btl->gm_max_send_tokens = btl->gm_num_send_tokens;
btl->gm_num_recv_tokens = gm_num_receive_tokens (btl->gm_port);
btl->gm_max_recv_tokens = btl->gm_num_recv_tokens;
/* initialize memory pool */
resources.gm_port = btl->gm_port;
btl->gm_mpool = mca_mpool_base_module_create(
mca_btl_gm_component.gm_mpool_name,
&btl->super,
&resources);
if(NULL == btl->gm_mpool) {
ompi_output (0, "[%s:%d] unable to initialize mpool", __FILE__, __LINE__);
return OMPI_ERROR;
}
/* initialize free lists */
ompi_free_list_init( &btl->gm_frag_eager,
sizeof (mca_btl_gm_frag_eager_t),
OBJ_CLASS (mca_btl_gm_frag_eager_t),
mca_btl_gm_component.gm_free_list_num,
mca_btl_gm_component.gm_free_list_max,
mca_btl_gm_component.gm_free_list_inc,
btl->gm_mpool );
ompi_free_list_init( &btl->gm_frag_max,
sizeof (mca_btl_gm_frag_max_t),
OBJ_CLASS (mca_btl_gm_frag_max_t),
mca_btl_gm_component.gm_free_list_num,
mca_btl_gm_component.gm_free_list_max,
mca_btl_gm_component.gm_free_list_inc,
btl->gm_mpool );
ompi_free_list_init( &btl->gm_frag_user,
sizeof (mca_btl_gm_frag_user_t),
OBJ_CLASS (mca_btl_gm_frag_user_t),
mca_btl_gm_component.gm_free_list_num,
mca_btl_gm_component.gm_free_list_max,
mca_btl_gm_component.gm_free_list_inc,
btl->gm_mpool );
/* post receive buffers */
for(i=0; i<mca_btl_gm_component.gm_num_high_priority; i++) {
mca_btl_gm_frag_t* frag;
MCA_BTL_GM_FRAG_ALLOC_EAGER(btl, frag, rc);
if(NULL == frag) {
return rc;
}
gm_provide_receive_buffer(btl->gm_port, frag->hdr, frag->size, GM_HIGH_PRIORITY);
}
for(i=mca_btl_gm_component.gm_num_high_priority; i<btl->gm_max_recv_tokens; i++) {
mca_btl_gm_frag_t* frag;
MCA_BTL_GM_FRAG_ALLOC_MAX(btl, frag, rc);
if(NULL == frag) {
return rc;
}
gm_provide_receive_buffer(btl->gm_port, frag->hdr, frag->size, GM_LOW_PRIORITY);
}
/* enable rdma */
if( GM_SUCCESS != gm_allow_remote_memory_access (btl->gm_port) ) {
ompi_output (0, "[%s:%d] unable to allow remote memory access", __FILE__, __LINE__);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/*
* Scan all ports on the boards. As it's difficult to find the total number of boards
* so we use a predefined maximum.
*/
static int mca_btl_gm_discover( void )
{
uint32_t board_no;
uint32_t port_no;
uint32_t local_id;
struct gm_port* gm_port;
#if GM_API_VERSION > 0x200
uint32_t global_id;
#else
char global_id[GM_MAX_HOST_NAME_LEN];
#endif /* GM_API_VERSION > 0x200 */
int rc;
for( board_no = 0; board_no < mca_btl_gm_component.gm_max_boards; board_no++ ) {
mca_btl_gm_module_t *btl;
/* open the first available gm port for this board */
for( port_no = 2; port_no < mca_btl_gm_component.gm_max_ports; port_no++ ) {
if (3 == port_no) {
continue; /* port 0,1,3 reserved */
} else if (GM_SUCCESS ==
gm_open(&gm_port, board_no, port_no,
mca_btl_gm_component.gm_port_name,
GM_API_VERSION) ) {
break;
}
}
if( port_no == mca_btl_gm_component.gm_max_ports ) {
continue;
}
/* Get node local Id */
if( GM_SUCCESS != gm_get_node_id( gm_port, &local_id) ) {
ompi_output (0, " failure to get local_id \n");
continue;
}
/* Gather an unique id for the node */
#if GM_API_VERSION > 0x200
if (GM_SUCCESS != gm_node_id_to_global_id( gm_port, local_id, &global_id) ) {
ompi_output (0, "[%s:%d] Unable to get my GM global unique id", __FILE__, __LINE__);
continue;
}
#else
if( GM_SUCCESS != gm_get_host_name( gm_port, global_id ) ) {
ompi_output( 0, "[%s:%d] Unable to get the GM host name\n", __FILE__, __LINE__);
continue;
}
#endif /* GM_API_VERSION > 0x200 */
/* create the btl module */
btl = (mca_btl_gm_module_t *)malloc( sizeof(mca_btl_gm_module_t) );
if (NULL == btl) {
ompi_output( 0, "[%s:%d] out of resources", __FILE__, __LINE__);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* copy the basic informations in the new PTL */
memcpy (btl, &mca_btl_gm_module, sizeof(mca_btl_gm_module_t) );
/* setup local address */
btl->gm_port = gm_port;
btl->gm_addr.port_id = port_no;
btl->gm_addr.local_id = local_id;
#if GM_API_VERSION > 0x200
btl->gm_addr.global_id = global_id;
#else
strncpy( btl->gm_addr.global_id, global_id, GM_MAX_HOST_NAME_LEN );
#endif /* GM_API_VERSION > 0x200 */
if((rc = mca_btl_gm_module_init(btl)) != OMPI_SUCCESS) {
ompi_output(0, "[%s:%d] unable to initialze gm port", __FILE__, __LINE__);
return rc;
}
/* everything is OK let's mark it as usable and go to the next one */
mca_btl_gm_component.gm_btls[mca_btl_gm_component.gm_num_btls++];
if(mca_btl_gm_component.gm_num_btls >= mca_btl_gm_component.gm_max_btls ) {
break;
}
}
return OMPI_SUCCESS;
}
/*
* Register GM component addressing information. The MCA framework
* will make this available to all peers.
*/
static int
mca_btl_gm_modex_send(void)
{
int rc;
size_t i;
size_t size;
mca_btl_gm_addr_t *addrs;
size = mca_btl_gm_component.gm_num_btls * sizeof (mca_btl_gm_addr_t);
addrs = (mca_btl_gm_addr_t *)malloc (size);
if (NULL == addrs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (i = 0; i < mca_btl_gm_component.gm_num_btls; i++) {
mca_btl_gm_module_t *btl = mca_btl_gm_component.gm_btls[i];
addrs[i] = btl->gm_addr;
}
rc = mca_base_modex_send (&mca_btl_gm_component.super.btl_version, addrs, size);
free (addrs);
return rc;
}
/*
* Initialize the GM component,
* check how many boards are available and open ports on them.
*/
mca_btl_base_module_t **
mca_btl_gm_component_init (int *num_btl_modules,
bool enable_progress_threads,
bool enable_mpi_threads)
{
mca_btl_base_module_t **btls;
*num_btl_modules = 0;
/* s try to initialize GM */
if( GM_SUCCESS != gm_init() ) {
ompi_output( 0, "[%s:%d] error in initializing the gm library\n", __FILE__, __LINE__ );
return NULL;
}
/* First discover all available boards. For each board we create a unique BTL */
mca_btl_gm_component.gm_btls = malloc( mca_btl_gm_component.gm_max_btls * sizeof (mca_btl_gm_module_t *));
if (NULL == mca_btl_gm_component.gm_btls) {
ompi_output( 0, "[%s:%d] out of resources.", __FILE__, __LINE__ );
return NULL;
}
mca_btl_gm_component.gm_num_btls = mca_btl_gm_discover( );
/* initialize gm */
if (OMPI_SUCCESS != mca_btl_gm_discover()) {
return NULL;
}
/* publish GM parameters with the MCA framework */
if (OMPI_SUCCESS != mca_btl_gm_modex_send()) {
return NULL;
}
/* return array of BTLs */
btls = (mca_btl_base_module_t**) malloc (
mca_btl_gm_component.gm_num_btls * sizeof(mca_btl_base_module_t *));
if (NULL == btls) {
return NULL;
}
memcpy(btls, mca_btl_gm_component.gm_btls,
mca_btl_gm_component.gm_num_btls * sizeof(mca_btl_gm_module_t *));
*num_btl_modules = mca_btl_gm_component.gm_num_btls;
return btls;
}
/*
* GM component progress.
*/
int mca_btl_gm_component_progress()
{
int count = 0;
size_t i;
for( i = 0; i < mca_btl_gm_component.gm_num_btls; ) {
mca_btl_gm_module_t* btl = mca_btl_gm_component.gm_btls[i];
gm_recv_event_t* event = gm_receive(btl->gm_port);
unsigned char* buffer = (unsigned char*)gm_ntohp(event->recv.buffer);
mca_btl_gm_frag_t* frag = (mca_btl_gm_frag_t*)(buffer - sizeof(mca_btl_gm_frag_t));
mca_btl_base_header_t* hdr;
uint32_t priority = GM_HIGH_PRIORITY;
/* If there are no receive events just skip the function call */
switch(gm_ntohc(event->recv.type)) {
case GM_FAST_RECV_EVENT:
case GM_FAST_PEER_RECV_EVENT:
priority = GM_LOW_PRIORITY;
case GM_FAST_HIGH_RECV_EVENT:
case GM_FAST_HIGH_PEER_RECV_EVENT:
{
mca_btl_base_recv_reg_t* reg;
hdr = (mca_btl_base_header_t *)gm_ntohp(event->recv.message);
reg = &btl->gm_reg[hdr->tag];
reg->cbfunc(&btl->super, hdr->tag, &frag->base, reg->cbdata);
count++;
break;
}
case GM_RECV_EVENT:
case GM_PEER_RECV_EVENT:
priority = GM_LOW_PRIORITY;
case GM_HIGH_RECV_EVENT:
case GM_HIGH_PEER_RECV_EVENT:
{
mca_btl_base_recv_reg_t* reg;
hdr = (mca_btl_base_header_t*)buffer;
reg = &btl->gm_reg[hdr->tag];
reg->cbfunc(&btl->super, hdr->tag, &frag->base, reg->cbdata);
count++;
break;
}
case GM_NO_RECV_EVENT:
i++;
break;
default:
gm_unknown(btl->gm_port, event);
break;
}
gm_provide_receive_buffer(btl->gm_port, buffer, frag->size, priority);
}
return count;
}

59
src/mca/btl/gm/btl_gm_endpoint.c Обычный файл
Просмотреть файл

@ -0,0 +1,59 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/time.h>
#include <time.h>
#include "include/types.h"
#include "mca/ns/base/base.h"
#include "mca/oob/base/base.h"
#include "mca/rml/rml.h"
#include "mca/errmgr/errmgr.h"
#include "dps/dps.h"
#include "btl_gm.h"
#include "btl_gm_endpoint.h"
#include "btl_gm_proc.h"
#include "btl_gm_frag.h"
/*
* Initialize state of the endpoint instance.
*
*/
static void mca_btl_gm_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
{
endpoint->endpoint_btl = 0;
endpoint->endpoint_proc = 0;
}
/*
* Destroy a endpoint
*
*/
static void mca_btl_gm_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
{
}
OBJ_CLASS_INSTANCE(
mca_btl_gm_endpoint_t,
ompi_list_item_t,
mca_btl_gm_endpoint_construct,
mca_btl_gm_endpoint_destruct);

73
src/mca/btl/gm/btl_gm_endpoint.h Обычный файл
Просмотреть файл

@ -0,0 +1,73 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_GM_ENDPOINT_H
#define MCA_BTL_GM_ENDPOINT_H
#include "class/ompi_list.h"
#include "event/event.h"
#include "mca/pml/pml.h"
#include "mca/btl/btl.h"
#include "btl_gm_frag.h"
#include "btl_gm.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Structure used to publish GM id information to peers.
*/
struct mca_btl_gm_addr_t {
#if GM_API_VERSION > 0x200
unsigned int global_id;
#else
char global_id[GM_MAX_HOST_NAME_LEN];
#endif /* GM_API_VERSION > 0x200 */
unsigned int local_id;
unsigned int port_id;
};
typedef struct mca_btl_gm_addr_t mca_btl_gm_addr_t;
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_btl_base_endpoint_t {
ompi_list_item_t super;
struct mca_btl_gm_module_t* endpoint_btl;
/**< BTL instance that created this connection */
struct mca_btl_gm_proc_t* endpoint_proc;
/**< proc structure corresponding to endpoint */
mca_btl_gm_addr_t endpoint_addr;
};
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
typedef mca_btl_base_endpoint_t mca_btl_gm_endpoint_t;
OBJ_CLASS_DECLARATION(mca_btl_gm_endpoint_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

64
src/mca/btl/gm/btl_gm_frag.c Обычный файл
Просмотреть файл

@ -0,0 +1,64 @@
#include "btl_gm_frag.h"
static void mca_btl_gm_frag_common_constructor(mca_btl_gm_frag_t* frag)
{
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
}
static void mca_btl_gm_frag_eager_constructor(mca_btl_gm_frag_t* frag)
{
frag->hdr = (mca_btl_base_header_t*)(frag + 1);
frag->segment.seg_addr.pval = (unsigned char*)(frag->hdr + 1);
frag->segment.seg_len = frag->size;
frag->registration = NULL;
frag->size = mca_btl_gm_module.super.btl_eager_limit;
mca_btl_gm_frag_common_constructor(frag);
}
static void mca_btl_gm_frag_max_constructor(mca_btl_gm_frag_t* frag)
{
frag->hdr = (mca_btl_base_header_t*)(frag + 1);
frag->segment.seg_addr.pval = (unsigned char*)(frag->hdr + 1);
frag->segment.seg_len = frag->size;
frag->registration = NULL;
frag->size = mca_btl_gm_module.super.btl_max_send_size;
mca_btl_gm_frag_common_constructor(frag);
}
static void mca_btl_gm_frag_user_constructor(mca_btl_gm_frag_t* frag)
{
frag->hdr = NULL;
frag->size = 0;
mca_btl_gm_frag_common_constructor(frag);
}
OBJ_CLASS_INSTANCE(
mca_btl_gm_frag_t,
mca_btl_base_descriptor_t,
NULL,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_gm_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_gm_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_gm_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_gm_frag_max_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_gm_frag_user_t,
mca_btl_base_descriptor_t,
mca_btl_gm_frag_user_constructor,
NULL);

113
src/mca/btl/gm/btl_gm_frag.h Обычный файл
Просмотреть файл

@ -0,0 +1,113 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_GM_FRAG_H
#define MCA_BTL_GM_FRAG_H
#define MCA_BTL_GM_FRAG_ALIGN (8)
#include "ompi_config.h"
#include "btl_gm.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_gm_frag_t);
/**
* GM send fragment derived type.
*/
struct mca_btl_gm_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_base_segment_t segment;
struct mca_btl_gm_module_t* btl;
struct mca_btl_base_endpoint_t *endpoint;
mca_btl_base_header_t *hdr;
size_t size;
struct mca_mpool_base_registration_t* registration;
};
typedef struct mca_btl_gm_frag_t mca_btl_gm_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_gm_frag_t);
typedef struct mca_btl_gm_frag_t mca_btl_gm_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_btl_gm_frag_eager_t);
typedef struct mca_btl_gm_frag_t mca_btl_gm_frag_max_t;
OBJ_CLASS_DECLARATION(mca_btl_gm_frag_max_t);
typedef struct mca_btl_gm_frag_t mca_btl_gm_frag_user_t;
OBJ_CLASS_DECLARATION(mca_btl_gm_frag_user_t);
/*
* Macros to allocate/return descriptors from module specific
* free list(s).
*/
#define MCA_BTL_GM_FRAG_ALLOC_EAGER(btl, frag, rc) \
{ \
\
ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_btl_gm_module_t*)btl)->gm_frag_eager, item, rc); \
frag = (mca_btl_gm_frag_t*) item; \
}
#define MCA_BTL_GM_FRAG_RETURN_EAGER(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_btl_gm_module_t*)btl)->gm_frag_eager, \
(ompi_list_item_t*)(frag)); \
}
#define MCA_BTL_GM_FRAG_ALLOC_MAX(btl, frag, rc) \
{ \
\
ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_btl_gm_module_t*)btl)->gm_frag_max, item, rc); \
frag = (mca_btl_gm_frag_t*) item; \
}
#define MCA_BTL_GM_FRAG_RETURN_MAX(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_btl_gm_module_t*)btl)->gm_frag_max, \
(ompi_list_item_t*)(frag)); \
}
#define MCA_BTL_GM_FRAG_ALLOC_USER(btl, frag, rc) \
{ \
ompi_list_item_t *item; \
OMPI_FREE_LIST_WAIT(&((mca_btl_gm_module_t*)btl)->gm_frag_user, item, rc); \
frag = (mca_btl_gm_frag_t*) item; \
}
#define MCA_BTL_GM_FRAG_RETURN_USER(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_btl_gm_module_t*)btl)->gm_frag_user, \
(ompi_list_item_t*)(frag)); \
}
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

167
src/mca/btl/gm/btl_gm_proc.c Обычный файл
Просмотреть файл

@ -0,0 +1,167 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "class/ompi_hash_table.h"
#include "mca/base/mca_base_module_exchange.h"
#include "btl_gm.h"
#include "btl_gm_proc.h"
static void mca_btl_gm_proc_construct(mca_btl_gm_proc_t* proc);
static void mca_btl_gm_proc_destruct(mca_btl_gm_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_btl_gm_proc_t,
ompi_list_item_t, mca_btl_gm_proc_construct,
mca_btl_gm_proc_destruct);
void mca_btl_gm_proc_construct(mca_btl_gm_proc_t* proc)
{
proc->proc_ompi = 0;
proc->proc_addr_count = 0;
proc->proc_endpoints = 0;
proc->proc_endpoint_count = 0;
OBJ_CONSTRUCT(&proc->proc_lock, ompi_mutex_t);
/* add to list of all proc instance */
OMPI_THREAD_LOCK(&mca_btl_gm_component.gm_lock);
ompi_list_append(&mca_btl_gm_component.gm_procs, &proc->super);
OMPI_THREAD_UNLOCK(&mca_btl_gm_component.gm_lock);
}
/*
* Cleanup ib proc instance
*/
void mca_btl_gm_proc_destruct(mca_btl_gm_proc_t* proc)
{
/* remove from list of all proc instances */
OMPI_THREAD_LOCK(&mca_btl_gm_component.gm_lock);
ompi_list_remove_item(&mca_btl_gm_component.gm_procs, &proc->super);
OMPI_THREAD_UNLOCK(&mca_btl_gm_component.gm_lock);
/* release resources */
if(NULL != proc->proc_endpoints) {
free(proc->proc_endpoints);
}
}
/*
* Look for an existing GM process instances based on the associated
* ompi_proc_t instance.
*/
static mca_btl_gm_proc_t* mca_btl_gm_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_btl_gm_proc_t* gm_proc;
OMPI_THREAD_LOCK(&mca_btl_gm_component.gm_lock);
for(gm_proc = (mca_btl_gm_proc_t*)
ompi_list_get_first(&mca_btl_gm_component.gm_procs);
gm_proc != (mca_btl_gm_proc_t*)
ompi_list_get_end(&mca_btl_gm_component.gm_procs);
gm_proc = (mca_btl_gm_proc_t*)ompi_list_get_next(gm_proc)) {
if(gm_proc->proc_ompi == ompi_proc) {
OMPI_THREAD_UNLOCK(&mca_btl_gm_component.gm_lock);
return gm_proc;
}
}
OMPI_THREAD_UNLOCK(&mca_btl_gm_component.gm_lock);
return NULL;
}
/*
* Create a GM process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_btl_gm_proc_t instance. We cache
* additional data (specifically the list of mca_btl_gm_endpoint_t instances,
* and published addresses) associated w/ a given destination on this
* datastructure.
*/
mca_btl_gm_proc_t* mca_btl_gm_proc_create(ompi_proc_t* ompi_proc)
{
mca_btl_gm_proc_t* gm_proc = NULL;
size_t size;
int rc;
/* Check if we have already created a GM proc
* structure for this ompi process */
gm_proc = mca_btl_gm_proc_lookup_ompi(ompi_proc);
if(gm_proc != NULL) {
return gm_proc;
}
/* create a new gm proc out of the ompi_proc ... */
gm_proc = OBJ_NEW(mca_btl_gm_proc_t);
gm_proc->proc_endpoint_count = 0;
gm_proc->proc_ompi = ompi_proc;
gm_proc->proc_guid = ompi_proc->proc_name;
/* query for the peer address info */
rc = mca_base_modex_recv(
&mca_btl_gm_component.super.btl_version,
ompi_proc,
(void*)&gm_proc->proc_addrs,
&size);
if(OMPI_SUCCESS != rc) {
ompi_output(0, "[%s:%d] mca_base_modex_recv failed for peer [%d,%d,%d]",
__FILE__,__LINE__,ORTE_NAME_ARGS(&ompi_proc->proc_name));
OBJ_RELEASE(gm_proc);
return NULL;
}
if((size % sizeof(mca_btl_gm_addr_t)) != 0) {
ompi_output(0, "[%s:%d] invalid gm address for peer [%d,%d,%d]",
__FILE__,__LINE__,ORTE_NAME_ARGS(&ompi_proc->proc_name));
OBJ_RELEASE(gm_proc);
return NULL;
}
gm_proc->proc_addr_count = sizeof(mca_btl_gm_addr_t)/size;
gm_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc(gm_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
if(NULL == gm_proc->proc_endpoints) {
OBJ_RELEASE(gm_proc);
return NULL;
}
return gm_proc;
}
/*
* Note that this routine must be called with the lock on the process
* already held. Insert a btl instance into the proc array and assign
* it an address.
*/
int mca_btl_gm_proc_insert(mca_btl_gm_proc_t* gm_proc,
mca_btl_base_endpoint_t* gm_endpoint)
{
/* insert into endpoint array */
if(gm_proc->proc_addr_count >= gm_proc->proc_endpoint_count)
return OMPI_ERR_OUT_OF_RESOURCE;
gm_endpoint->endpoint_proc = gm_proc;
gm_endpoint->endpoint_addr = gm_proc->proc_addrs[gm_proc->proc_endpoint_count];
gm_proc->proc_endpoints[gm_proc->proc_endpoint_count] = gm_endpoint;
gm_proc->proc_endpoint_count++;
return OMPI_SUCCESS;
}

70
src/mca/btl/gm/btl_gm_proc.h Обычный файл
Просмотреть файл

@ -0,0 +1,70 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_GM_PROC_H
#define MCA_BTL_GM_PROC_H
#include "mca/ns/ns.h"
#include "class/ompi_object.h"
#include "proc/proc.h"
#include "btl_gm.h"
#include "btl_gm_endpoint.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
OBJ_CLASS_DECLARATION(mca_btl_gm_proc_t);
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each
* BTL instance that attempts to open a connection to the process.
*/
struct mca_btl_gm_proc_t {
ompi_list_item_t super;
/**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
orte_process_name_t proc_guid;
/**< globally unique identifier for the process */
struct mca_btl_gm_addr_t* proc_addrs;
/**< array of addresses exported by peer */
size_t proc_addr_count;
/**< number of addresses published by peer */
struct mca_btl_base_endpoint_t **proc_endpoints;
/**< array of endpoints that have been created to access this proc */
size_t proc_endpoint_count;
/**< number of endpoints */
ompi_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_btl_gm_proc_t mca_btl_gm_proc_t;
mca_btl_gm_proc_t* mca_btl_gm_proc_create(ompi_proc_t* ompi_proc);
int mca_btl_gm_proc_insert(mca_btl_gm_proc_t*, mca_btl_base_endpoint_t*);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

22
src/mca/btl/gm/configure.params Обычный файл
Просмотреть файл

@ -0,0 +1,22 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=btl_gm.c
PARAM_CONFIG_HEADER_FILE="gm_config.h"
PARAM_CONFIG_FILES="Makefile"

196
src/mca/btl/gm/configure.stub Обычный файл
Просмотреть файл

@ -0,0 +1,196 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004 The Ohio State University.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
#
# Main function. This will be invoked in the middle of the templated
# configure script.
#
AC_DEFUN([MCA_CONFIGURE_STUB],[
# Additional --with flags that can be specified
AC_ARG_WITH(btl-gm,
AC_HELP_STRING([--with-btl-gm=DIR],
[Specify the installation directory of GM]))
AC_ARG_WITH(btl-gm-libdir,
AC_HELP_STRING([--with-btl-gm-libdir=DIR],
[directory where the GM library can be found, if it is not in \$GMDIR/lib or \$GMDIR/binary/lib]))
AC_ARG_ENABLE(gm-ptl-cache,
AC_HELP_STRING([--enable-gm-ptl-cache],[Enable/disable the internal registered memory cache (default=no)]),
[enable_gm_cache=$(enable_gm_ptl_cache)],
[enable_gm_cache="no"])
if test "$enable_gm_cache" = "yes"; then
AC_DEFINE_UNQUOTED( OMPI_MCA_BTL_GM_CACHE_ENABLE, 1,
[Enable GM BTL internal cache for registered memory])
else
AC_DEFINE_UNQUOTED( OMPI_MCA_BTL_GM_CACHE_ENABLE, 0,
[Disable GM BTL internal cache for registered memory])
fi
AM_CONDITIONAL( OMPI_ENABLE_GM_CACHE, test "$enable_gm_cache" = "yes" )
# Add to CPPFLAGS if necessary
EXTRA_CPPFLAGS=
if test -n "$with_btl_gm"; then
if test -d "$with_btl_gm/include"; then
EXTRA_CPPFLAGS="-I$with_btl_gm/include"
else
AC_MSG_WARN([*** Warning: cannot find $with_btl_gm/include])
AC_MSG_WARN([*** Will still try to configure gm ptl anyway...])
fi
fi
# See if we can find gm.h
CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS"
AC_CHECK_HEADERS(gm.h,,
AC_MSG_ERROR([*** Cannot find working gm.h.]))
# Add to LDFLAGS if necessary
EXTRA_LDFLAGS=
if test -n "$with_btl_gm_libdir"; then
if test -d "$with_btl_gm_libdir"; then
EXTRA_LDFLAGS="-L$with_btl_gm_libdir"
else
AC_MSG_WARN([*** Warning: cannot find $with_btl_gm_libdir])
AC_MSG_WARN([*** Will still try to configure gm ptl anyway...])
fi
elif test -n "$with_btl_gm"; then
if test -d "$with_btl_gm/lib"; then
EXTRA_LDFLAGS="-L$with_btl_gm/lib"
elif test -d "$with_btl_gm/binary/lib"; then
EXTRA_LDFLAGS="-L$with_btl_gm/binary/lib"
else
AC_MSG_WARN([*** Warning: cannot find $with_btl_gm/lib])
AC_MSG_WARN([*** or $with_btl_gm/binary/lib])
AC_MSG_WARN([*** Will still try to configure gm ptl anyway...])
fi
fi
# Try to find libgm
LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
AC_CHECK_LIB([gm], [main], [],
AC_MSG_ERROR([*** Cannot find libgm]))
#
# See if we have GM_API_VERSION. If we do, use it. If not, find the
# highest one available. It seems that GM_API_VERSION was introduced
# somewhere after 1.3 but before 1.6. :-\
#
AC_MSG_CHECKING(for GM_API_VERSION)
AC_TRY_COMPILE([#include<gm.h>],
[int foo = GM_API_VERSION;],
have_gm_api_ver_msg=yes gm_api_ver=GM_API_VERSION,
have_gm_api_ver_msg=no gm_api_ver="")
AC_MSG_RESULT([$have_gm_api_ver_msg])
if test "$gm_api_ver" = ""; then
found=0
for val in 5 4 3; do
if test "$found" = "0"; then
var="GM_API_VERSION_1_$val"
AC_MSG_CHECKING(for $var)
AC_TRY_COMPILE([#include<gm.h>],
[int foo = $var;],
msg=yes found=1 gm_api_ver=$var,
msg=no found=0 gm_api_ver="")
AC_MSG_RESULT($msg)
fi
done
fi
if test "$gm_api_ver" = ""; then
AC_MSG_WARN([*** Could not find a supported GM_API_VERSION])
AC_MSG_ERROR([*** Cannot continue])
fi
AC_DEFINE_UNQUOTED(OMPI_MCA_BTL_GM_API_VERSION, $gm_api_ver,
[Version of the GM API to use])
unset gm_api_ver have_gm_api_ver_msg found val msg
#
# Do we have gm_put()?
# gm_put() was introduced in gm 2.0, and is exactly identical to gm
# 1.6's gm_directed_send_with_callback(). The name was simply changed
# for consistency/symmtery with gm_get().
#
AC_MSG_CHECKING([for gm_put()])
AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include<gm.h>
]],
[[gm_put(0, 0, 0, 0, 0, 0, 0, 0, 0);]])],
[HAVE_RDMA_PUT=1 MSG=yes],
[HAVE_RDMA_PUT=0 MSG="no, use gm_directed_send_with_callback()"])
AC_DEFINE_UNQUOTED(OMPI_MCA_BTL_GM_HAVE_RDMA_PUT, $HAVE_RDMA_PUT,
[Whether we have gm_put() or gm_directed_send_with_callback()])
AC_MSG_RESULT([$MSG])
#
# Do we have gm_get()?
# gm_get() was introduced in gm 2.0.
#
AC_MSG_CHECKING([for gm_get()])
AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include<gm.h>
]],
[[gm_get(0, 0, 0, 0, 0, 0, 0, 0, 0);]])],
[HAVE_RDMA_GET=1 MSG=yes],
[HAVE_RDMA_GET=0 MSG=no])
AC_DEFINE_UNQUOTED(OMPI_MCA_BTL_GM_HAVE_RDMA_GET, $HAVE_RDMA_GET,
[Whether we have get_get() or not])
AC_MSG_RESULT([$MSG])
# Now test to see if the targetted GM is a broken one broken gm builds
AC_MSG_CHECKING([for broken GM 2.x RDMA gets build])
AC_TRY_COMPILE([
#include <gm.h>
], [
#if GM_API_VERSION_2_0 && \
((GM_API_VERSION_2_1_0 && GM_API_VERSION < 0x20102) || \
GM_API_VERSION < 0x2000c)
#error GM build is broken
#endif ],
[ mca_btl_gm_broken=0 gm_build_broken=no ],
[ mca_btl_gm_broken=1 gm_build_broken=yes ])
AC_MSG_RESULT( [$gm_build_broken] )
AC_DEFINE_UNQUOTED( OMPI_MCA_BTL_GM_GET_BROKEN, $mca_btl_gm_broken,
[The GM build has or not a broker gm_get function] )
unset gm_build_broken mca_btl_gm_broken
AC_DEFINE_UNQUOTED( OMPI_MCA_BTL_GM_SUPPORT_REGISTERING, 1,
[The OS support or not the virtal page registration] )
#
# Save extra compiler/linker flags so that they can be added in
# the wrapper compilers, if necessary
#
WRAPPER_EXTRA_LDFLAGS="$EXTRA_LDFLAGS"
WRAPPER_EXTRA_LIBS="-lgm"
])dnl
#
# For dist
#
AC_DEFUN([MCA_CONFIGURE_DIST_STUB],[
AM_CONDITIONAL( OMPI_ENABLE_GM_CACHE, test "yes" = "yes" )
])

114
src/mca/btl/gm/gm_config.h Обычный файл
Просмотреть файл

@ -0,0 +1,114 @@
/* gm_config.h. Generated by configure. */
/* gm_config.h.in. Generated from configure.ac by autoheader. */
/* -*- c -*-
*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
* Open MPI configuation header file.
* MCA btl: gm component
*/
#ifndef MCA_btl_gm_CONFIG_H
#define MCA_btl_gm_CONFIG_H
/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1
/* Define to 1 if you have the <gm.h> header file. */
#define HAVE_GM_H 1
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
/* Define to 1 if you have the `gm' library (-lgm). */
#define HAVE_LIBGM 1
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1
/* Whether the gm btl is the default btl or not */
#define MCA_btl_gm_DEFAULT 0
/* OMPI architecture string */
#define OMPI_ARCH "i686-pc-linux-gnu"
/* OMPI underlying C compiler */
#define OMPI_CC "gcc"
/* Version of the GM API to use */
#define OMPI_MCA_BTL_GM_API_VERSION GM_API_VERSION
/* Disable GM BTL internal cache for registered memory */
#define OMPI_MCA_BTL_GM_CACHE_ENABLE 0
/* The GM build has or not a broker gm_get function */
#define OMPI_MCA_BTL_GM_GET_BROKEN 0
/* Whether we have get_get() or not */
#define OMPI_MCA_BTL_GM_HAVE_RDMA_GET 1
/* Whether we have gm_put() or gm_directed_send_with_callback() */
#define OMPI_MCA_BTL_GM_HAVE_RDMA_PUT 1
/* The OS support or not the virtal page registration */
#define OMPI_MCA_BTL_GM_SUPPORT_REGISTERING 1
/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT ""
/* Define to the full name of this package. */
#define PACKAGE_NAME ""
/* Define to the full name and version of this package. */
#define PACKAGE_STRING ""
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME ""
/* Define to the version of this package. */
#define PACKAGE_VERSION ""
/* Define to 1 if you have the ANSI C header files. */
/* #undef STDC_HEADERS */
/* Enable GNU extensions on systems that have them. */
#ifndef _GNU_SOURCE
# define _GNU_SOURCE 1
#endif
#endif /* _MCA_btl_gm_CONFIG_H */