ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

remove the elan btl

Просмотр исходного кода 
This commit was SVN r26798.
Этот коммит содержится в:
Nathan Hjelm 2012-07-17 14:51:41 +00:00
родитель fc1b295606
Коммит 35de50b823
11 изменённых файлов: 0 добавлений и 1743 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

39
ompi/mca/btl/elan/Makefile.am
Просмотреть файл

@ -1,39 +0,0 @@
#
# Copyright (c) 2004-2007 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CPPFLAGS = $(btl_elan_CPPFLAGS)
btl_elan_sources = btl_elan.c btl_elan.h btl_elan_component.c btl_elan_endpoint.c \
btl_elan_endpoint.h btl_elan_frag.c btl_elan_frag.h btl_elan_proc.c btl_elan_proc.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_ompi_btl_elan_DSO
component_noinst =
component_install = mca_btl_elan.la
else
component_noinst = libmca_btl_elan.la
component_install =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component_install)
mca_btl_elan_la_SOURCES = $(btl_elan_sources)
mca_btl_elan_la_LIBADD = $(btl_elan_LIBS)
mca_btl_elan_la_LDFLAGS = -module -avoid-version $(btl_elan_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_btl_elan_la_SOURCES = $(btl_elan_sources)
libmca_btl_elan_la_LIBADD = $(btl_elan_LIBS)
libmca_btl_elan_la_LDFLAGS = -module -avoid-version $(btl_elan_LDFLAGS)

662
ompi/mca/btl/elan/btl_elan.c
Просмотреть файл

@ -1,662 +0,0 @@
/*
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/class/opal_bitmap.h"
#include "opal/util/output.h"
#include "ompi/mca/btl/btl.h"
#include "btl_elan.h"
#include "btl_elan_frag.h"
#include "btl_elan_proc.h"
#include "btl_elan_endpoint.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/btl/base/base.h"
#include <stdio.h>
#include "elan/elan.h"
#include "opal/util/os_path.h"
#include "opal/util/opal_environ.h"
#include "orte/util/proc_info.h"
/**
*
*/
extern char** environ;
/**
* Reduce function that compute the MAX over an array of integers.
*/
static void __reduce_max_fn( void *vin, void* vinout, int* count, void* handle )
{
int *in = (int*)vin, *inout = (int*)vinout;
int i;
for( i = 0; i < (*count); i++ ) {
if( in[i] > inout[i] ) inout[i] = in[i];
}
}
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN)
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info.
* @param peers (IN/OUT) Set of processes that are reachable via this BTL.
* @return OMPI_SUCCESS or error status on failure.
*
*/
static int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **ompi_procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable )
{
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*)btl;
int i, j, rc;
char* filename;
FILE* file;
ELAN_BASE* base;
filename = opal_os_path( false, orte_process_info.proc_session_dir, "ELAN_ID", NULL );
file = fopen( filename, "w" );
fprintf( file, "%s %d\n", ompi_proc_local_proc->proc_hostname, elan_btl->elan_position );
for(i = 0; i < (int)nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_btl_elan_proc_t* elan_proc;
mca_btl_base_endpoint_t* elan_endpoint;
/* Don't use Elan for local communications */
if( ompi_proc_local_proc == ompi_proc )
continue;
if(NULL == (elan_proc = mca_btl_elan_proc_create(ompi_proc))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
elan_endpoint = OBJ_NEW(mca_btl_elan_endpoint_t);
if(NULL == elan_endpoint) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
elan_endpoint->endpoint_btl = elan_btl;
OPAL_THREAD_LOCK(&elan_proc->proc_lock);
rc = mca_btl_elan_proc_insert(elan_proc, elan_endpoint);
OPAL_THREAD_UNLOCK(&elan_proc->proc_lock);
if( OMPI_SUCCESS != rc ) {
OBJ_RELEASE(elan_endpoint);
OBJ_RELEASE(elan_proc);
continue;
}
for( j = 0; j < (int)elan_proc->proc_rail_count; j++ ) {
fprintf( file, "%s %d\n", ompi_proc->proc_hostname,
elan_proc->position_id_array[j] );
}
opal_bitmap_set_bit(reachable, i);
peers[i] = elan_endpoint;
}
fclose(file);
/* Set the environment before firing up the Elan library */
opal_setenv( "LIBELAN_MACHINES_FILE", filename, true, &environ );
opal_setenv( "MPIRUN_ELANIDMAP_FILE", mca_btl_elan_component.elanidmap_file,
false, &environ );
base = elan_baseInit(0);
if( NULL == base )
return OMPI_ERR_OUT_OF_RESOURCE;
if( NULL == base->state ) {
mca_btl_base_error_no_nics( "ELAN", "Quadrics" );
return OMPI_ERR_OUT_OF_RESOURCE;
}
elan_btl->base = base;
elan_btl->elan_vp = base->state->vp;
{
unsigned int* vp_array = (unsigned int*)calloc( nprocs, sizeof(unsigned int) );
/* Set my position in the array with the Elan vp */
vp_array[(int)ompi_proc_local_proc->proc_name.vpid] = elan_btl->elan_vp;
/* Do a reduce with the previously defined MAX function. The outcome will be
* that at each process vpid index we will have their Elan vp. With this Elan
* vp we can therefore communicate with the process.
*/
elan_reduce( base->allGroup, vp_array, vp_array, sizeof(unsigned int), (int)nprocs,
__reduce_max_fn, NULL, 0, 0,
ELAN_REDUCE_COMMUTE | ELAN_RESULT_ALL | base->group_flags, 0);
for(i = 0; i < (int)nprocs; i++) {
if(NULL == peers[i])
continue;
peers[i]->elan_vp = vp_array[(int)ompi_procs[i]->proc_name.vpid];
}
free(vp_array);
}
/* Create the tport global queue */
if( (elan_btl->tport_queue = elan_gallocQueue(base, base->allGroup)) == NULL ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Create and initialize the tport */
if( !(elan_btl->tport = elan_tportInit(base->state,
elan_btl->tport_queue,
mca_btl_elan_component.elan_max_posted_recv,
base->tport_smallmsg,
mca_btl_elan_module.super.btl_eager_limit,
base->tport_stripemsg,
ELAN_POLL_EVENT,
base->retryCount,
&base->shm_key,
base->shm_fifodepth,
base->shm_fragsize,
0))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Create the receive queue */
if( (elan_btl->global_queue = elan_gallocQueue(base, base->allGroup)) == NULL ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_btl_elan_component.queue_max_size = elan_queueMaxSlotSize( base->state )
- sizeof(mca_btl_elan_hdr_t);
elan_btl->rx_queue =
elan_queueRxInit( base->state, /* ELAN_STATE *state */
elan_btl->global_queue, /* ELAN_QUEUE *queue */
mca_btl_elan_component.elan_max_posted_recv, /* int nSlots */
(int)mca_btl_elan_component.queue_max_size, /* int slotSize */
ELAN_RAIL_ALL, /* int rail */
0 ); /* ELAN_FLAGS flags */
elan_btl->tx_queue =
elan_queueTxInit( base->state, /* ELAN_STATE *state */
elan_btl->global_queue, /* ELAN_QUEUE *q */
ELAN_RAIL_ALL, /* int rail */
0 ); /* ELAN_FLAGS flags */
for( i = 0; i < mca_btl_elan_component.elan_max_posted_recv; i++ ) {
mca_btl_elan_frag_t* frag;
MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc );
if( OPAL_UNLIKELY(NULL == frag) ) {
return OMPI_ERROR;
}
frag->segment.seg_addr.pval = (void*)(frag + 1);
frag->base.des_dst = &(frag->segment);
frag->base.des_dst_cnt = 1;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->type = MCA_BTL_ELAN_HDR_TYPE_RECV;
frag->elan_event = elan_tportRxStart( elan_btl->tport,
ELAN_TPORT_RXBUF | ELAN_TPORT_RXANY,
0, 0, 0, 0,
frag->segment.seg_addr.pval,
mca_btl_elan_module.super.btl_eager_limit );
opal_list_append( &(elan_btl->recv_list), (opal_list_item_t*)frag );
}
/* enable the network */
elan_enable_network( elan_btl->base->state );
return OMPI_SUCCESS;
}
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN) BTL instance
* @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful
*
*/
static int mca_btl_elan_del_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t ** endpoints )
{
return OMPI_SUCCESS;
}
/**
* Allocate a descriptor with a segment of the requested size.
* Note that the BTL layer may choose to return a smaller size
* if it cannot support the request.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*/
static mca_btl_base_descriptor_t*
mca_btl_elan_alloc( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
uint8_t order,
size_t size,
uint32_t flags )
{
mca_btl_elan_frag_t* frag = NULL;
ptrdiff_t hdr_skip = 0;
int rc;
if( size <= btl->btl_eager_limit ) {
MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc);
if( size <= mca_btl_elan_component.queue_max_size ) { /* This will be going over the queue */
hdr_skip = sizeof(mca_btl_elan_hdr_t);
}
} else if( size <= btl->btl_max_send_size ) {
MCA_BTL_ELAN_FRAG_ALLOC_MAX(frag, rc);
}
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->segment.seg_addr.pval = (void*)((char*)(frag + 1) + hdr_skip);
frag->segment.seg_len = size;
frag->base.des_src = &(frag->segment);
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = flags;
frag->btl = (mca_btl_elan_module_t*)btl;
frag->endpoint = peer;
frag->base.order = MCA_BTL_NO_ORDER;
return (mca_btl_base_descriptor_t*)frag;
}
/**
* Return a segment allocated by this BTL.
*
* @param btl (IN) BTL module
* @param descriptor (IN) Allocated descriptor.
*/
static int mca_btl_elan_free( struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des )
{
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)des;
MCA_BTL_ELAN_FRAG_RETURN(frag);
return OMPI_SUCCESS;
}
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
static mca_btl_base_descriptor_t*
mca_btl_elan_prepare_src( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
mca_btl_elan_frag_t* frag;
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data = *size, skip = 0;
int rc;
if( OPAL_UNLIKELY(max_data > UINT32_MAX) ) {
max_data = (size_t)UINT32_MAX;
}
if( 0 != reserve ) {
if( max_data + reserve <= btl->btl_eager_limit ) {
MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc);
if( (max_data + reserve) <= mca_btl_elan_component.queue_max_size ) {
skip = sizeof(mca_btl_elan_hdr_t);
}
} else {
MCA_BTL_ELAN_FRAG_ALLOC_MAX(frag, rc);
if( (max_data + reserve) > btl->btl_max_send_size ) {
max_data = btl->btl_max_send_size - reserve;
}
}
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->segment.seg_addr.pval = (void*)((unsigned char*)(frag + 1) + skip);
iov.iov_len = max_data;
iov.iov_base = (unsigned char*)frag->segment.seg_addr.pval + reserve;
rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
*size = max_data;
if( rc < 0 ) {
MCA_BTL_ELAN_FRAG_RETURN(frag);
return NULL;
}
frag->segment.seg_len = max_data + reserve;
} else { /* this is a real RDMA operation */
MCA_BTL_ELAN_FRAG_ALLOC_USER(frag, rc);
if(OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT;
iov.iov_len = max_data;
iov.iov_base = NULL;
opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
*size = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
frag->segment.seg_len = max_data;
}
frag->base.des_src = &(frag->segment);
frag->base.des_src_cnt = 1;
frag->base.order = MCA_BTL_NO_ORDER;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = flags;
return &frag->base;
}
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
static mca_btl_base_descriptor_t*
mca_btl_elan_prepare_dst( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags )
{
mca_btl_elan_frag_t* frag;
size_t origin, position = *size;
int rc;
if( OPAL_UNLIKELY((*size) > UINT32_MAX) ) {
*size = (size_t)UINT32_MAX;
}
MCA_BTL_ELAN_FRAG_ALLOC_USER(frag, rc);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
origin = convertor->bConverted;
position += origin;
opal_convertor_set_position( convertor, &position );
*size = position - origin;
frag->segment.seg_len = *size;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_flags = flags;
frag->base.des_dst = &(frag->segment);
frag->base.des_dst_cnt = 1;
frag->base.order = MCA_BTL_NO_ORDER;
return &frag->base;
}
/**
* Initiate an asynchronous send.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transfered
* @param tag (IN) The tag value used to notify the peer.
*/
static int mca_btl_elan_send( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag )
{
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl;
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)descriptor;
mca_btl_elan_hdr_t* elan_hdr = (mca_btl_elan_hdr_t*)(frag+1);
int send_len;
frag->btl = elan_btl;
frag->endpoint = endpoint;
frag->tag = tag;
frag->type = MCA_BTL_ELAN_HDR_TYPE_SEND;
if( frag->segment.seg_len <= mca_btl_elan_component.queue_max_size ) {
elan_hdr->tag = (int)tag;
elan_hdr->length = (int)frag->segment.seg_len;
send_len = frag->segment.seg_len + sizeof(mca_btl_elan_hdr_t);
frag->elan_event = elan_queueTx( elan_btl->tx_queue,
endpoint->elan_vp,
(void*)elan_hdr,
send_len, ELAN_RAIL_ALL );
if( OPAL_UNLIKELY(NULL == frag->elan_event) ) {
opal_output( 0, "elan_queueTx failed for destination %d\n", endpoint->elan_vp );
return OMPI_ERROR;
}
} else {
frag->elan_event = elan_tportTxStart( elan_btl->tport, 0, endpoint->elan_vp,
elan_btl->elan_vp, frag->tag,
(void*)elan_hdr, frag->segment.seg_len );
if( OPAL_UNLIKELY(NULL == frag->elan_event) ) {
opal_output( 0, "elan_tportTxStart failed for destination %d\n", endpoint->elan_vp );
return OMPI_ERROR;
}
}
if( elan_poll( frag->elan_event, 0 ) ) {
int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP );
frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS );
if( btl_ownership ) {
MCA_BTL_ELAN_FRAG_RETURN(frag);
}
return OMPI_SUCCESS;
}
/* Add the fragment to the pending send list */
OPAL_THREAD_LOCK( &elan_btl->elan_lock );
opal_list_append( &(elan_btl->send_list), (opal_list_item_t*)frag );
OPAL_THREAD_UNLOCK( &elan_btl->elan_lock );
return OMPI_ERR_RESOURCE_BUSY;
}
/**
* Initiate an asynchronous put.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
static int mca_btl_elan_put( mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* des )
{
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl;
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) des;
int peer = endpoint->elan_vp;
mca_btl_base_segment_t* src = des->des_src;
mca_btl_base_segment_t* dst = des->des_dst;
unsigned char* src_addr = (unsigned char*)src->seg_addr.pval;
size_t src_len = src->seg_len;
unsigned char* dst_addr = (unsigned char*)dst->seg_addr.pval;
frag->endpoint = endpoint;
frag->btl = elan_btl;
frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT;
frag->elan_event = elan_put(elan_btl->base->state, src_addr, dst_addr, src_len, peer);
/* Add the fragment to the pending RDMA list */
OPAL_THREAD_LOCK( &elan_btl->elan_lock );
opal_list_append( &(elan_btl->rdma_list), (opal_list_item_t*)frag );
OPAL_THREAD_UNLOCK( &elan_btl->elan_lock );
return OMPI_SUCCESS;
}
/**
* Initiate an asynchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*
*/
static int mca_btl_elan_get( mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* des )
{
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl;
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) des;
int peer = endpoint->elan_vp;
mca_btl_base_segment_t* src = des->des_src;
mca_btl_base_segment_t* dst = des->des_dst;
unsigned char* src_addr = (unsigned char*)src->seg_addr.pval;
size_t src_len = src->seg_len;
unsigned char* dst_addr = (unsigned char*)dst->seg_addr.lval;
frag->endpoint = endpoint;
frag->btl = elan_btl;
frag->type = MCA_BTL_ELAN_HDR_TYPE_GET;
opal_output( 0, "elan_get( remote %p, local %p, length %d, peer %d )\n",
(void*)src_addr, (void*)dst_addr, (int)src_len, peer );
frag->elan_event = elan_get(elan_btl->base->state, src_addr, dst_addr, src_len, peer);
/* Add the fragment to the pending RDMA list */
OPAL_THREAD_LOCK( &elan_btl->elan_lock );
opal_list_append( &(elan_btl->rdma_list), (opal_list_item_t*)frag );
OPAL_THREAD_UNLOCK( &elan_btl->elan_lock );
return OMPI_SUCCESS;
}
int mca_btl_elan_finalize( struct mca_btl_base_module_t* btl )
{
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl;
int i, num_btls;
/* First find the correct BTL in the list attached to the component */
num_btls = mca_btl_elan_component.elan_num_btls;
for( i = 0; i < num_btls; i++ ) {
if( elan_btl == mca_btl_elan_component.elan_btls[i] ) {
/* disable the network */
elan_disable_network( elan_btl->base->state );
/* Remove it from the list attached to the component */
if( i == (num_btls-1) ) {
mca_btl_elan_component.elan_btls[i] = NULL;
} else {
mca_btl_elan_component.elan_btls[i] = mca_btl_elan_component.elan_btls[num_btls-1];
}
mca_btl_elan_component.elan_num_btls--;
/* Cancel all posted receives */
/* Release the internal structures */
OBJ_DESTRUCT(&elan_btl->recv_list);
OBJ_DESTRUCT(&elan_btl->send_list);
OBJ_DESTRUCT(&elan_btl->rdma_list);
OBJ_DESTRUCT(&elan_btl->elan_lock);
/* The BTL is clean, remove it */
free(elan_btl);
return OMPI_SUCCESS;
}
}
/* This BTL is not present in the list attached to the communicator */
return OMPI_ERROR;
}
int mca_btl_elan_ft_event(int state)
{
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
;
}
else if(OPAL_CRS_RESTART == state) {
;
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}
static void mca_btl_elan_dump( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
int verbose )
{
}
static int
mca_btl_elan_register_error( struct mca_btl_base_module_t* btl,
mca_btl_base_module_error_cb_fn_t cbfunc )
{
return OMPI_SUCCESS;
}
mca_btl_elan_module_t mca_btl_elan_module = {
{
&mca_btl_elan_component.super,
0, /* max size of first fragment */
0, /* min send fragment size */
0, /* max send fragment size */
0, /* btl_rdma_pipeline_offset */
0, /* btl_rdma_pipeline_frag_size */
0, /* btl_min_rdma_pipeline_size */
0, /* exclusivity */
0, /* latency */
0, /* bandwidth */
0, /* flags */
0, /* segment size */
mca_btl_elan_add_procs,
mca_btl_elan_del_procs,
NULL, /* btl_register */
mca_btl_elan_finalize,
mca_btl_elan_alloc,
mca_btl_elan_free,
mca_btl_elan_prepare_src,
mca_btl_elan_prepare_dst,
mca_btl_elan_send,
NULL, /* send immediate */
mca_btl_elan_put,
mca_btl_elan_get,
mca_btl_elan_dump,
NULL, /* mpool */
mca_btl_elan_register_error, /* register error cb */
mca_btl_elan_ft_event /* mca_btl_elan_ft_event*/
}
};

141
ompi/mca/btl/elan/btl_elan.h
Просмотреть файл

@ -1,141 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_BTL_ELAN_H
#define MCA_BTL_ELAN_H
#include "ompi_config.h"
#include "opal/mca/event/event.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "btl_elan_endpoint.h"
#include "btl_elan_frag.h"
#include "elan/elan.h"
BEGIN_C_DECLS
/**
* ELAN BTL component.
*/
struct mca_btl_elan_component_t {
mca_btl_base_component_2_0_0_t super; /**< base BTL component */
size_t queue_max_size;
/**< maximum amount of data transfered using the queues */
uint32_t elan_num_btls;
/**< number of hcas available to the ELAN component */
struct mca_btl_elan_module_t **elan_btls;
/**< array of available BTL modules */
int elan_free_list_num;
/**< initial size of free lists */
int elan_free_list_max;
/**< maximum size of free lists */
int elan_free_list_inc;
/**< number of elements to alloc when growing free lists */
int elan_max_posted_recv;
/**< number of pre-posted receives */
/* free list of fragment descriptors */
ompi_free_list_t elan_frag_eager;
ompi_free_list_t elan_frag_max;
ompi_free_list_t elan_frag_user;
opal_list_t elan_procs;
/**< list of elan proc structures */
opal_mutex_t elan_lock;
/**< lock for accessing module state */
char* elanidmap_file; /**< name of the ELANIDMAP file */
};
typedef struct mca_btl_elan_component_t mca_btl_elan_component_t;
OMPI_MODULE_DECLSPEC extern mca_btl_elan_component_t mca_btl_elan_component;
/**
* BTL Module Interface
*/
struct mca_btl_elan_module_t {
mca_btl_base_module_t super; /**< base BTL interface */
int expect_tport_recv;
int elan_vp;
int elan_position; /**< position of this elan interface */
ELAN_BASE* base;
ELAN_TPORT* tport;
ELAN_QUEUE* global_queue; /**< The global queue */
ELAN_QUEUE* tport_queue;
ELAN_QUEUE_RX* rx_queue; /**< The local receive queue */
ELAN_QUEUE_TX* tx_queue; /**< The global send queue */
opal_mutex_t elan_lock;
opal_list_t send_list; /**< list of posted sends */
opal_list_t rdma_list; /**< list of posted receives */
opal_list_t recv_list;
};
typedef struct mca_btl_elan_module_t mca_btl_elan_module_t;
extern mca_btl_elan_module_t mca_btl_elan_module;
typedef struct mca_btl_elan_hdr_t {
int tag;
int length;
} mca_btl_elan_hdr_t;
/**
* Register ELAN component parameters with the MCA framework
*/
extern int mca_btl_elan_component_open(void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_btl_elan_component_close(void);
/**
* ELAN component initialization.
*
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
*/
extern mca_btl_base_module_t**
mca_btl_elan_component_init( int* num_btl_modules,
bool allow_multi_user_threads,
bool have_hidden_threads );
/**
* ELAN component progress.
*/
extern int mca_btl_elan_component_progress(void);
extern int mca_btl_elan_finalize( struct mca_btl_base_module_t* btl );
extern int mca_btl_elan_ft_event(int state);
END_C_DECLS
#endif /* MCA_BTL_ELAN_H */

370
ompi/mca/btl/elan/btl_elan_component.c
Просмотреть файл

@ -1,370 +0,0 @@
/*
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "ompi/mca/btl/btl.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "ompi/mca/mpool/base/base.h"
#include "btl_elan.h"
#include "btl_elan_frag.h"
#include "btl_elan_endpoint.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "elan/elan.h"
#include "opal/util/opal_environ.h"
#define ELAN_MAX_BTL 10
mca_btl_elan_component_t mca_btl_elan_component = {
{
/* First, the mca_base_component_t struct containing meta information
about the component itself */
{
MCA_BTL_BASE_VERSION_2_0_0,
"elan", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_btl_elan_component_open, /* component open */
mca_btl_elan_component_close /* component close */
},
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_btl_elan_component_init,
mca_btl_elan_component_progress,
}
};
/*
* utility routines for parameter registration
*/
static inline int
mca_btl_elan_param_register_int( const char* param_name,
int default_value )
{
int id = mca_base_param_register_int("btl","elan",param_name,NULL,default_value);
int param_value = default_value;
mca_base_param_lookup_int(id,&param_value);
return param_value;
}
/*
* Called by MCA framework to open the component, registers
* component parameters.
*/
int mca_btl_elan_component_open(void)
{
/* initialize state */
mca_btl_elan_component.elan_num_btls = 0;
mca_btl_elan_component.elan_btls = NULL;
mca_btl_elan_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
mca_btl_elan_module.super.btl_eager_limit = 32*1024;
mca_btl_elan_module.super.btl_rndv_eager_limit = mca_btl_elan_module.super.btl_eager_limit;
mca_btl_elan_module.super.btl_max_send_size = 64*1024; /*64*1024;*/
mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 * 1024;
mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128 * 1024;
mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND;
mca_btl_elan_module.super.btl_seg_size = sizeof (mca_btl_base_segment_t);
mca_btl_elan_module.super.btl_bandwidth = 1959;
mca_btl_elan_module.super.btl_latency = 4;
mca_btl_base_param_register(&mca_btl_elan_component.super.btl_version,
&mca_btl_elan_module.super);
mca_base_param_reg_string( (mca_base_component_t*)&mca_btl_elan_component, "elanidmap",
"System-wide configuration file for the Quadrics network (elanidmap)",
false, false, "/etc/elanidmap", &mca_btl_elan_component.elanidmap_file );
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_elan_component, "max_posted_recv",
"Number of received posted in advance. Increasing this number for"
" communication bound application can lead to visible improvement"
" in performances",
false, false, 128, &mca_btl_elan_component.elan_max_posted_recv );
/* register Elan4 component parameters */
mca_btl_elan_component.elan_free_list_num =
mca_btl_elan_param_register_int( "free_list_num", 8 );
mca_btl_elan_component.elan_free_list_max =
mca_btl_elan_param_register_int( "free_list_max",
(mca_btl_elan_component.elan_free_list_num +
mca_btl_elan_component.elan_max_posted_recv) );
mca_btl_elan_component.elan_free_list_inc =
mca_btl_elan_param_register_int( "free_list_inc", 32 );
return OMPI_SUCCESS;
}
/*
* component cleanup - sanity checking of queue lengths
*/
int mca_btl_elan_component_close(void)
{
if( NULL != mca_btl_elan_component.elan_btls ) {
free( mca_btl_elan_component.elan_btls );
mca_btl_elan_component.elan_btls = NULL;
mca_btl_elan_component.elan_num_btls = 0;
/* release resources */
OBJ_DESTRUCT(&mca_btl_elan_component.elan_procs);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_eager);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_user);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_max);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_lock);
}
return OMPI_SUCCESS;
}
/*
* Elan4 component initialization:
* (1) read interface list from kernel and compare against component parameters
* then create a BTL instance for selected interfaces
* (2) setup Elan4 listen socket for incoming connection attempts
* (3) register BTL parameters with the MCA
*/
mca_btl_base_module_t**
mca_btl_elan_component_init( int *num_btl_modules,
bool enable_progress_threads,
bool enable_mpi_threads )
{
mca_btl_base_module_t** btls = NULL;
*num_btl_modules = 0;
/* There is no support for a progress thread yet. */
if (enable_progress_threads) {
ompi_modex_send(&mca_btl_elan_component.super.btl_version, NULL, 0);
return NULL;
}
OBJ_CONSTRUCT (&mca_btl_elan_component.elan_lock, opal_mutex_t);
OBJ_CONSTRUCT (&mca_btl_elan_component.elan_frag_eager, ompi_free_list_t);
OBJ_CONSTRUCT (&mca_btl_elan_component.elan_frag_max, ompi_free_list_t);
OBJ_CONSTRUCT (&mca_btl_elan_component.elan_frag_user, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_elan_component.elan_procs, opal_list_t);
ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_eager,
sizeof(mca_btl_elan_frag_t) + mca_btl_elan_module.super.btl_eager_limit,
opal_cache_line_size,
OBJ_CLASS(mca_btl_elan_frag_t),
0,opal_cache_line_size,
mca_btl_elan_component.elan_free_list_num,
mca_btl_elan_component.elan_free_list_max,
mca_btl_elan_component.elan_free_list_inc,
NULL ); /* use default allocator */
ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_user,
sizeof(mca_btl_elan_frag_t),
opal_cache_line_size,
OBJ_CLASS(mca_btl_elan_frag_t),
0,opal_cache_line_size,
mca_btl_elan_component.elan_free_list_num,
mca_btl_elan_component.elan_free_list_max,
mca_btl_elan_component.elan_free_list_inc,
NULL ); /* use default allocator */
ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_max,
sizeof(mca_btl_elan_frag_t)+mca_btl_elan_module.super.btl_max_send_size,
opal_cache_line_size,
OBJ_CLASS(mca_btl_elan_frag_t),
0,opal_cache_line_size,
mca_btl_elan_component.elan_free_list_num,
mca_btl_elan_component.elan_free_list_max,
mca_btl_elan_component.elan_free_list_inc,
NULL ); /* use default allocator */
mca_btl_elan_component.elan_num_btls = ELAN_MAX_BTL; /* no more than that */
mca_btl_elan_component.elan_btls = calloc( mca_btl_elan_component.elan_num_btls,
sizeof(mca_btl_base_module_t*) );
/* Retrieve the positions of the node in the elan network */
{
FILE* position;
char filename[255], file_line[255];
int index, count = 0, positions[ELAN_MAX_BTL];
mca_btl_elan_module_t* btl;
for( index = 0; index < ELAN_MAX_BTL; index++ ) {
snprintf( filename, 255, "/proc/qsnet/elan4/device%d/position", index );
position = fopen( filename, "r" );
if( NULL == position ) {
continue;
}
if( 0 == fscanf( position, "%s%i", file_line, &positions[count] ) ) {
opal_output( 0, "Unable to read the network position" );
continue;
}
fclose(position);
btl = (mca_btl_elan_module_t*)malloc (sizeof (mca_btl_elan_module_t));
if(NULL == btl) {
opal_output( 0, "No enough memory to allocate the Elan internal structures" );
return NULL;
}
memcpy( btl, &mca_btl_elan_module, sizeof(mca_btl_elan_module_t) );
OBJ_CONSTRUCT( &btl->elan_lock, opal_mutex_t );
OBJ_CONSTRUCT( &btl->send_list, opal_list_t );
OBJ_CONSTRUCT( &btl->rdma_list, opal_list_t );
OBJ_CONSTRUCT( &btl->recv_list, opal_list_t );
btl->expect_tport_recv = 1;
btl->elan_position = positions[count];
mca_btl_elan_component.elan_btls[count++] = btl;
}
mca_btl_elan_component.elan_num_btls = count;
/* Publish the network positions for the current node */
ompi_modex_send( &mca_btl_elan_component.super.btl_version, positions,
count * sizeof(int));
}
if(mca_btl_elan_component.elan_num_btls) {
btls = (mca_btl_base_module_t**)malloc( mca_btl_elan_component.elan_num_btls *
sizeof(mca_btl_base_module_t*) );
if( NULL == btls ) {
free( mca_btl_elan_component.elan_btls );
mca_btl_elan_component.elan_num_btls = 0; /* no active BTL modules */
return NULL;
}
memcpy( btls, mca_btl_elan_component.elan_btls,
mca_btl_elan_component.elan_num_btls * sizeof(mca_btl_elan_module_t*) );
}
*num_btl_modules = mca_btl_elan_component.elan_num_btls;
return btls;
}
/*
* Elan4 component progress.
*/
int mca_btl_elan_component_progress( void )
{
int num_progressed = 0, i;
for( i = 0; i < (int)mca_btl_elan_component.elan_num_btls; i++ ) {
mca_btl_elan_module_t* elan_btl = mca_btl_elan_component.elan_btls[i];
/* This is a fast receive over the queue */
if( elan_queueRxPoll( elan_btl->rx_queue, 0 ) ) {
mca_btl_active_message_callback_t* reg;
mca_btl_elan_hdr_t* elan_hdr = NULL;
mca_btl_elan_frag_t frag;
elan_hdr = (mca_btl_elan_hdr_t*)elan_queueRxWait( elan_btl->rx_queue, NULL, 0 );
frag.base.des_dst = &frag.segment;
frag.base.des_dst->seg_addr.pval = (void*)(elan_hdr+1);
frag.base.des_dst->seg_len = (size_t)elan_hdr->length;
frag.base.des_dst_cnt = 1;
frag.tag = (mca_btl_base_tag_t)elan_hdr->tag;
frag.size = elan_hdr->length;
reg = mca_btl_base_active_message_trigger + frag.tag;
reg->cbfunc( &(elan_btl->super), frag.tag, &(frag.base), reg->cbdata );
elan_queueRxComplete( elan_btl->rx_queue );
num_progressed++;
}
/* This is the slower receive over the tport */
if(elan_btl->expect_tport_recv && !OPAL_THREAD_TRYLOCK(&elan_btl->elan_lock)) {
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)opal_list_get_first( &(elan_btl->recv_list) );
if( elan_done(frag->elan_event, 0) ) {
int tag;
size_t length;
mca_btl_active_message_callback_t* reg;
void* recv_buf;
recv_buf = (mca_btl_elan_hdr_t*)elan_tportRxWait( frag->elan_event,
NULL, &tag, &length );
num_progressed++;
/*elan_btl->expect_tport_recv--;*/
opal_list_remove_first( &(elan_btl->recv_list) );
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
frag->base.des_dst->seg_addr.pval = (void*)recv_buf;
frag->base.des_dst->seg_len = length;
frag->tag = (mca_btl_base_tag_t)tag;
reg = mca_btl_base_active_message_trigger + frag->tag;
reg->cbfunc( &(elan_btl->super), frag->tag, &(frag->base), reg->cbdata );
if( recv_buf != (void*)(frag+1) ) {
elan_tportBufFree( elan_btl->tport, recv_buf );
frag->base.des_dst->seg_addr.pval = (void*)(frag+1);
}
frag->elan_event = elan_tportRxStart( elan_btl->tport,
ELAN_TPORT_RXBUF | ELAN_TPORT_RXANY,
0, 0, 0, 0,
frag->base.des_dst->seg_addr.pval,
mca_btl_elan_module.super.btl_eager_limit );
OPAL_THREAD_LOCK(&elan_btl->elan_lock);
opal_list_append( &(elan_btl->recv_list), (opal_list_item_t*)frag );
}
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
}
/* If there are any pending sends check their completion */
recheck_send_list:
if( !opal_list_is_empty( &(elan_btl->send_list) ) && !OPAL_THREAD_TRYLOCK(&elan_btl->elan_lock) ) {
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)opal_list_get_first( &(elan_btl->send_list) );
if( (NULL != frag) && elan_poll(frag->elan_event, 0) ) {
int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP );
opal_list_remove_first( &(elan_btl->send_list) );
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
num_progressed++;
frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS );
if( btl_ownership ) {
MCA_BTL_ELAN_FRAG_RETURN(frag);
}
goto recheck_send_list;
} else {
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
}
}
recheck_rdma_list:
/* If any RDMA have been posted, check their status */
if( !opal_list_is_empty( &(elan_btl->rdma_list) ) && !OPAL_THREAD_TRYLOCK(&elan_btl->elan_lock) ) {
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)opal_list_get_first( &(elan_btl->rdma_list) );
if( (NULL != frag) && elan_poll(frag->elan_event, 0) ) {
int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP );
opal_list_remove_first( &(elan_btl->rdma_list) );
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
num_progressed++;
frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS );
if( btl_ownership ) {
MCA_BTL_ELAN_FRAG_RETURN(frag);
}
goto recheck_rdma_list;
} else {
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
}
}
}
return num_progressed;
}

48
ompi/mca/btl/elan/btl_elan_endpoint.c
Просмотреть файл

@ -1,48 +0,0 @@
/*
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/time.h>
#include <time.h>
#include "ompi/types.h"
#include "btl_elan.h"
#include "btl_elan_endpoint.h"
#include "btl_elan_proc.h"
#include "btl_elan_frag.h"
/*
* Initialize state of the endpoint instance.
*
*/
static void mca_btl_elan_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
{
endpoint->endpoint_btl = NULL;
endpoint->endpoint_proc = NULL;
}
/*
* Destroy a endpoint
*
*/
static void mca_btl_elan_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
{
}
OBJ_CLASS_INSTANCE(mca_btl_elan_endpoint_t,
opal_list_item_t,
mca_btl_elan_endpoint_construct,
mca_btl_elan_endpoint_destruct);

51
ompi/mca/btl/elan/btl_elan_endpoint.h
Просмотреть файл

@ -1,51 +0,0 @@
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_ELAN_ENDPOINT_H
#define MCA_BTL_ELAN_ENDPOINT_H
#include "opal/class/opal_list.h"
#include "opal/mca/event/event.h"
#include "btl_elan_frag.h"
#include "btl_elan.h"
BEGIN_C_DECLS
/**
* State of ELAN endpoint connection.
*/
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_btl_base_endpoint_t {
opal_list_item_t super;
struct mca_btl_elan_module_t* endpoint_btl;
/**< BTL instance that created this connection */
struct mca_btl_elan_proc_t* endpoint_proc;
/**< proc structure corresponding to endpoint */
unsigned int elan_vp;
};
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
typedef mca_btl_base_endpoint_t mca_btl_elan_endpoint_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_endpoint_t);
END_C_DECLS
#endif /* MCA_BTL_ELAN_ENDPOINT_H */

65
ompi/mca/btl/elan/btl_elan_frag.c
Просмотреть файл

@ -1,65 +0,0 @@
/*
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "btl_elan.h"
static void mca_btl_elan_frag_common_constructor(mca_btl_elan_frag_t* frag)
{
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->elan_event = NULL;
frag->base.des_dst = &(frag->segment);
frag->base.des_dst_cnt = 1;
}
static void mca_btl_elan_frag_eager_constructor(mca_btl_elan_frag_t* frag)
{
frag->size = mca_btl_elan_module.super.btl_eager_limit;
frag->my_list = &mca_btl_elan_component.elan_frag_eager;
mca_btl_elan_frag_common_constructor(frag);
}
static void mca_btl_elan_frag_max_constructor(mca_btl_elan_frag_t* frag)
{
frag->size = mca_btl_elan_module.super.btl_max_send_size;
frag->my_list = &mca_btl_elan_component.elan_frag_max;
mca_btl_elan_frag_common_constructor(frag);
}
static void mca_btl_elan_frag_user_constructor(mca_btl_elan_frag_t* frag)
{
frag->size = 0;
frag->my_list = &mca_btl_elan_component.elan_frag_user;
mca_btl_elan_frag_common_constructor(frag);
}
OBJ_CLASS_INSTANCE( mca_btl_elan_frag_t,
mca_btl_base_descriptor_t,
NULL,
NULL );
OBJ_CLASS_INSTANCE( mca_btl_elan_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_elan_frag_eager_constructor,
NULL );
OBJ_CLASS_INSTANCE( mca_btl_elan_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_elan_frag_max_constructor,
NULL );
OBJ_CLASS_INSTANCE( mca_btl_elan_frag_user_t,
mca_btl_base_descriptor_t,
mca_btl_elan_frag_user_constructor,
NULL );

85
ompi/mca/btl/elan/btl_elan_frag.h
Просмотреть файл

@ -1,85 +0,0 @@
/*
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_ELAN_FRAG_H
#define MCA_BTL_ELAN_FRAG_H
#include "ompi_config.h"
#include "btl_elan.h"
BEGIN_C_DECLS
#define MCA_BTL_ELAN_HDR_TYPE_SEND 1
#define MCA_BTL_ELAN_HDR_TYPE_PUT 2
#define MCA_BTL_ELAN_HDR_TYPE_GET 3
#define MCA_BTL_ELAN_HDR_TYPE_RECV 4
/**
* Elan send fragment derived type.
*/
struct mca_btl_elan_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_base_segment_t segment;
struct mca_btl_base_endpoint_t *endpoint;
struct mca_btl_elan_module_t* btl;
int type;
ompi_free_list_t* my_list;
mca_btl_base_tag_t tag;
struct ELAN_EVENT* elan_event;
size_t size;
};
typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_frag_t);
typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_frag_eager_t);
typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_max_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_frag_max_t);
typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_user_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_frag_user_t);
/*
* Macros to allocate/return descriptors from module specific
* free list(s).
*/
#define MCA_BTL_ELAN_FRAG_ALLOC_LIST( list, frag, rc ) \
{ \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET(&(list), item, rc); \
frag = (mca_btl_elan_frag_t*) item; \
}
#define MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc) \
MCA_BTL_ELAN_FRAG_ALLOC_LIST(mca_btl_elan_component.elan_frag_eager, frag, rc)
#define MCA_BTL_ELAN_FRAG_ALLOC_MAX(frag, rc) \
MCA_BTL_ELAN_FRAG_ALLOC_LIST(mca_btl_elan_component.elan_frag_max, frag, rc)
#define MCA_BTL_ELAN_FRAG_ALLOC_USER(frag, rc) \
MCA_BTL_ELAN_FRAG_ALLOC_LIST(mca_btl_elan_component.elan_frag_user, frag, rc)
#define MCA_BTL_ELAN_FRAG_RETURN(frag) \
{ \
OMPI_FREE_LIST_RETURN(frag->my_list, \
(ompi_free_list_item_t*)(frag)); \
}
END_C_DECLS
#endif /* MCA_BTL_ELAN_FRAG_H */

149
ompi/mca/btl/elan/btl_elan_proc.c
Просмотреть файл

@ -1,149 +0,0 @@
/*
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "btl_elan.h"
#include "btl_elan_proc.h"
static void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc);
static void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_btl_elan_proc_t,
opal_list_item_t,mca_btl_elan_proc_construct,
mca_btl_elan_proc_destruct);
void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* elan_proc)
{
elan_proc->proc_ompi = 0;
elan_proc->proc_rail_count = 0;
elan_proc->proc_endpoints = 0;
elan_proc->proc_endpoint_count = 0;
OBJ_CONSTRUCT(&elan_proc->proc_lock, opal_mutex_t);
/* add to list of all proc instance */
OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
opal_list_append(&mca_btl_elan_component.elan_procs, &elan_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
}
/*
* Cleanup elan proc instance
*/
void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* elan_proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
opal_list_remove_item(&mca_btl_elan_component.elan_procs, &elan_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
/* release resources */
if(NULL != elan_proc->proc_endpoints) {
free(elan_proc->proc_endpoints);
}
OBJ_DESTRUCT(&elan_proc->proc_lock);
}
/*
* Look for an existing Elan process instances based on the associated
* ompi_proc_t instance.
*/
static mca_btl_elan_proc_t* mca_btl_elan_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_btl_elan_proc_t* elan_proc;
OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
for(elan_proc = (mca_btl_elan_proc_t*)
opal_list_get_first(&mca_btl_elan_component.elan_procs);
elan_proc != (mca_btl_elan_proc_t*)
opal_list_get_end(&mca_btl_elan_component.elan_procs);
elan_proc = (mca_btl_elan_proc_t*)opal_list_get_next(elan_proc)) {
if(elan_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
return elan_proc;
}
}
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
return NULL;
}
/*
* Create a ELAN process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_btl_elan_proc_t instance. We cache
* additional data (specifically the list of mca_btl_elan_endpoint_t instances,
* and published addresses) associated w/ a given destination on this
* datastructure.
*/
mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
{
int rc;
size_t size;
mca_btl_elan_proc_t* module_proc = NULL;
/* Check if we have already created a Elan proc
* structure for this ompi process */
module_proc = mca_btl_elan_proc_lookup_ompi(ompi_proc);
if(module_proc != NULL) {
/* Gotcha! */
return module_proc;
}
/* Oops! First time, gotta create a new Elan proc
* out of the ompi_proc ... */
module_proc = OBJ_NEW(mca_btl_elan_proc_t);
if(NULL == module_proc)
return NULL;
/* Initialize number of peer */
module_proc->proc_endpoint_count = 0;
module_proc->proc_ompi = ompi_proc;
/* build a unique identifier (of arbitrary
* size) to represent the proc */
rc = ompi_modex_recv( &mca_btl_elan_component.super.btl_version,
ompi_proc,
(void**)&module_proc->position_id_array,
&size );
if(rc != OMPI_SUCCESS) {
BTL_ERROR(("mca_base_modex_recv: failed with return value=%d", rc));
OBJ_RELEASE(module_proc);
return NULL;
}
module_proc->proc_rail_count = size / sizeof(unsigned int);;
/* XXX: Right now, there can be only 1 peer associated
* with a proc. Needs a little bit change in
* mca_btl_elan_proc_t to allow on demand increasing of
* number of endpoints for this proc
*/
module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc((1+module_proc->proc_rail_count )* sizeof(mca_btl_base_endpoint_t*));
if(NULL == module_proc->proc_endpoints) {
OBJ_RELEASE(module_proc);
return NULL;
}
return module_proc;
}
/*
* Note that this routine must be called with the lock on the process
* already held. Insert a btl instance into the proc array and assign
* it an address.
*/
int mca_btl_elan_proc_insert( mca_btl_elan_proc_t* module_proc,
mca_btl_base_endpoint_t* module_endpoint )
{
/* insert into endpoint array */
module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint;
module_endpoint->endpoint_proc = module_proc;
module_endpoint->elan_vp = module_proc->proc_ompi->proc_name.vpid;
return OMPI_SUCCESS;
}

57
ompi/mca/btl/elan/btl_elan_proc.h
Просмотреть файл

@ -1,57 +0,0 @@
/*
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_ELAN_PROC_H
#define MCA_BTL_ELAN_PROC_H
#include "opal/class/opal_object.h"
#include "ompi/proc/proc.h"
#include "btl_elan.h"
#include "btl_elan_endpoint.h"
BEGIN_C_DECLS
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each
* BTL instance that attempts to open a connection to the process.
*/
struct mca_btl_elan_proc_t {
opal_list_item_t super;
/**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
unsigned int *position_id_array;
size_t proc_rail_count;
/**< number of addresses published by endpoint */
struct mca_btl_base_endpoint_t **proc_endpoints;
/**< array of endpoints that have been created to access this proc */
size_t proc_endpoint_count;
/**< number of endpoints */
opal_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_btl_elan_proc_t mca_btl_elan_proc_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_proc_t);
mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc);
int mca_btl_elan_proc_insert(mca_btl_elan_proc_t*, mca_btl_base_endpoint_t*);
END_C_DECLS
#endif /* MCA_BTL_ELAN_PROC_H */

76
ompi/mca/btl/elan/configure.m4
Просмотреть файл

@ -1,76 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2007 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# OMPI_CHECK_ELAN(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if Elan support can be found. sets prefix_{CPPFLAGS,
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
# support, otherwise executes action-if-not-found
AC_DEFUN([OMPI_CHECK_ELAN],[
AC_ARG_WITH([elan],
[AC_HELP_STRING([--with-elan(=DIR)],
[Build Elan (QsNet2) support, searching for libraries in DIR])])
OMPI_CHECK_WITHDIR([elan], [$with_elan], [include/elan/elan.h])
AC_ARG_WITH([elan-libdir],
[AC_HELP_STRING([--with-elan-libdir=DIR],
[Search for Elan (QsNet2) libraries in DIR])])
OMPI_CHECK_WITHDIR([elan-libdir], [$with_elan_libdir], [libelan.*])
AS_IF([test "$with_elan" != "no"],
[AS_IF([test ! -z "$with_elan" -a "$with_elan" != "yes"],
[ompi_check_elan_dir="$with_elan"])
AS_IF([test ! -z "$with_elan_libdir" -a "$with_elan_libdir" != "yes"],
[ompi_check_elan_libdir="$with_elan_libdir"])
OMPI_CHECK_PACKAGE([$1],
[elan/elan.h],
[elan],
[elan_init],
[],
[$ompi_check_elan_dir],
[$ompi_check_elan_libdir],
[ompi_check_elan_happy="yes"],
[ompi_check_elan_happy="no"])
],
[ompi_check_elan_happy="no"])
AS_IF([test "$ompi_check_elan_happy" = "yes"],
[$2],
[AS_IF([test ! -z "$with_elan" -a "$with_elan" != "no"],
[AC_MSG_ERROR([Elan (QsNet2) support requested but not found. Aborting])])
$3])
])
# MCA_btl_elan_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_btl_elan_CONFIG],[
AC_CONFIG_FILES([ompi/mca/btl/elan/Makefile])
OMPI_CHECK_ELAN([btl_elan],
[btl_elan_happy="yes"],
[btl_elan_happy="no"])
AS_IF([test "$btl_elan_happy" = "yes"],
[btl_elan_WRAPPER_EXTRA_LDFLAGS="$btl_elan_LDFLAGS"
btl_elan_WRAPPER_EXTRA_LIBS="$btl_elan_LIBS"
$1],
[$2])
# substitute in the things needed to build elan
AC_SUBST([btl_elan_CFLAGS])
AC_SUBST([btl_elan_CPPFLAGS])
AC_SUBST([btl_elan_LDFLAGS])
AC_SUBST([btl_elan_LIBS])
])dnl