ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Update the elan BTL. Now we support the following protocols: send, put

Просмотр исходного кода 
and partially get.

This commit was SVN r15564.
Этот коммит содержится в:
George Bosilca 2007-07-23 19:07:13 +00:00
родитель f219cc1e6e
Коммит 21a7670390
7 изменённых файлов: 637 добавлений и 623 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

168
ompi/mca/btl/elan/btl_elan.c
Просмотреть файл

@ -14,18 +14,24 @@
#include "opal/util/if.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/communicator/communicator.h"
#include "btl_elan.h"
#include "btl_elan_frag.h"
#include "btl_elan_proc.h"
#include "btl_elan_endpoint.h"
#include "ompi/datatype/convertor.h"
#include "ompi/mca/mpool/base/base.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "orte/class/orte_proc_table.h"
#include "opal/class/opal_hash_table.h"
#include "stdio.h"
#include "elan/elan.h"
#include "opal/util/os_path.h"
#include "opal/util/opal_environ.h"
#include "orte/util/proc_info.h"
mca_btl_elan_module_t mca_btl_elan_module = {
{
@ -50,7 +56,7 @@ mca_btl_elan_module_t mca_btl_elan_module = {
mca_btl_elan_prepare_dst,
mca_btl_elan_send,
mca_btl_elan_put,
NULL, /*mca_btl_elan_get,*/
mca_btl_elan_get,
NULL, /*mca_btl_elan_dump,*/
NULL, /* mpool */
NULL, /* register error cb */
@ -71,11 +77,58 @@ int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl,
{
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*)btl;
int i, rc;
FILE* file;
char* filename;
ELAN_BASE * base;
ELAN_STATE * state;
ELAN_QUEUE * q= NULL;
ELAN_TPORT * p= NULL;
filename = opal_os_path( false, orte_process_info.proc_session_dir, "ELAN_ID", NULL );
file = fopen( filename, "w" );
for( i = 0; i < (int)nprocs; i++ ) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
fprintf( file, "%s %d\n", ompi_proc->proc_hostname, i );
}
fclose( file );
opal_setenv( "LIBELAN_MACHINES_FILE", filename, true, &environ );
/* opal_setenv( "LIBELAN_MACHINES_FILE", "/home/tma/machinefile", false, &environ );*/
opal_setenv( "MPIRUN_ELANIDMAP_FILE", "/etc/elanidmap", false, &environ );
base = elan_baseInit(0);
if (base == NULL)
return NULL;
state = base->state;
if( NULL == state ) {
mca_btl_base_error_no_nics( "ELAN", "Quadrics" );
return NULL;
}
elan_gsync(base->allGroup);
if ((q = elan_allocQueue(base->state)) == NULL) {
return NULL;
}
if (!(p = elan_tportInit(base->state,
(ELAN_QUEUE *)q,
base->tport_nslots,
base->tport_smallmsg,
base->tport_bigmsg,
base->tport_stripemsg,
ELAN_POLL_EVENT,
base->retryCount,
&base->shm_key,
base->shm_fifodepth,
base->shm_fragsize,
0))) {
return NULL;
}
elan_btl->base = base;
elan_btl->state = state;
elan_btl->queue = q;
elan_btl->tport = p;
elan_btl->elan_vp = state->vp;
elan_btl->elan_nvp = state->nvp;
for(i = 0; i < (int) nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_btl_elan_proc_t* elan_proc;
mca_btl_base_endpoint_t* elan_endpoint;
if(NULL == (elan_proc = mca_btl_elan_proc_create(ompi_proc))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
@ -85,7 +138,6 @@ int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl,
OPAL_THREAD_UNLOCK(&elan_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
elan_endpoint->endpoint_btl = elan_btl;
rc = mca_btl_elan_proc_insert(elan_proc, elan_endpoint);
if(rc != OMPI_SUCCESS) {
@ -94,13 +146,10 @@ int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl,
OPAL_THREAD_UNLOCK(&elan_proc->proc_lock);
continue;
}
ompi_bitmap_set_bit(reachable, i);
OPAL_THREAD_UNLOCK(&elan_proc->proc_lock);
peers[i] = elan_endpoint;
}
return OMPI_SUCCESS;
}
@ -124,10 +173,9 @@ int mca_btl_elan_register( struct mca_btl_base_module_t* btl,
{
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl;
void * tbuf = NULL;
int rc;
int send_len, rc;
bufdesc_t *desc;
mca_btl_elan_frag_t* frag;
elan_btl->elan_reg[tag].cbfunc = cbfunc;
elan_btl->elan_reg[tag].cbdata = cbdata;
if (NULL != cbfunc) {
@ -143,13 +191,12 @@ int mca_btl_elan_register( struct mca_btl_base_module_t* btl,
frag->tag = tag;
frag->type = MCA_BTL_ELAN_HDR_TYPE_RECV;
tbuf = (void*)(frag+1);
send_len = elan_btl->super.btl_eager_limit;
desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
desc->eve = elan_tportRxStart( elan_btl->tport, 0, 0, 0, BTL_ELAN_RECV_MASK,
frag->tag, tbuf, elan_btl->super.btl_eager_limit );
desc->eve = elan_tportRxStart (elan_btl->tport,0 , 0, 0, 0xffffffff, frag->tag, tbuf,send_len) ;
desc->frag = frag;
desc->next = NULL;
BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
}
return OMPI_SUCCESS;
}
@ -166,24 +213,19 @@ mca_btl_base_descriptor_t* mca_btl_elan_alloc(struct mca_btl_base_module_t* btl,
uint8_t order,
size_t size )
{
mca_btl_elan_frag_t* frag;
int rc;
if(size <= btl->btl_eager_limit){
MCA_BTL_TEMPLATE_FRAG_ALLOC_EAGER(frag, rc);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->segment.seg_len = size;
} else if (size <= btl->btl_max_send_size){
MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(frag, rc);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->segment.seg_len = size;
} else {
return NULL;
@ -197,7 +239,6 @@ mca_btl_base_descriptor_t* mca_btl_elan_alloc(struct mca_btl_base_module_t* btl,
frag->btl = (mca_btl_elan_module_t*)btl;
frag->base.order = MCA_BTL_NO_ORDER;
return (mca_btl_base_descriptor_t*)frag;
}
@ -251,11 +292,10 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_src( struct mca_btl_base_module_
MCA_BTL_TEMPLATE_FRAG_RETURN(frag);
return NULL;
}
frag->segment.seg_addr.pval = frag+1;
frag->segment.seg_len = max_data + reserve;
}
else {
else if(max_data+reserve <= btl->btl_max_send_size){
MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(frag, rc);
if(NULL == frag) {
@ -273,9 +313,22 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_src( struct mca_btl_base_module_
MCA_BTL_TEMPLATE_FRAG_RETURN(frag);
return NULL;
}
frag->segment.seg_addr.pval = frag+1;
*size = max_data;
}
frag->segment.seg_len = max_data + reserve;
}else{
MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(frag, rc);
if(NULL == frag) {
return NULL;
}
frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT;
iov.iov_len = max_data;
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data);
*size = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
frag->segment.seg_len = max_data;
}
frag->base.des_src = &(frag->segment);
frag->base.des_src_cnt = 1;
frag->base.order = MCA_BTL_NO_ORDER;
@ -283,6 +336,7 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_src( struct mca_btl_base_module_
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
return &frag->base;
}
/**
@ -309,30 +363,27 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_dst( struct mca_btl_base_module_
{
mca_btl_elan_frag_t* frag;
mca_mpool_base_module_t* mpool = btl->btl_mpool;
int rc;
if( OPAL_UNLIKELY((*size) > UINT32_MAX) ) {
*size = (size_t)UINT32_MAX;
}
MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(frag, rc);
if(NULL == frag) {
return NULL;
}
frag->segment.seg_len = *size;
ompi_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
frag->segment.seg_len = *size;
frag->segment.seg_key.key64 = (uint64_t)(intptr_t)convertor;
/*frag->segment.seg_addr.pval = convertor->pBaseBuf + convertor->bConverted;*/
frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_flags = 0;
frag->base.des_dst = &(frag->segment);
frag->base.des_dst_cnt = 1;
frag->base.order = MCA_BTL_NO_ORDER;
return &frag->base;
}
@ -358,24 +409,20 @@ int mca_btl_elan_send( struct mca_btl_base_module_t* btl,
bufdesc_t * desc;
/* TODO */
/*opal_output(0, "I am send,tag is %d\n", tag);*/
frag->btl = elan_btl;
frag->endpoint = endpoint;
frag->tag = tag;
frag->type = MCA_BTL_ELAN_HDR_TYPE_SEND;
peer = endpoint->elan_vp;
proc = elan_btl->elan_vp;
sbuf = (void *)frag->base.des_src->seg_addr.pval;
send_len = frag->base.des_src->seg_len;
desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
desc->eve = elan_tportTxStart (elan_btl->tport, 0, peer, proc,frag->tag, sbuf, send_len) ;
/*opal_output( 0, "send message startoing from %p with ;length %d\n", sbuf, send_len );*/
/*opal_output( 0, "send message startoing from %d to %d\n", proc, peer );*/
desc->frag = frag;
desc->next = NULL;
BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
return OMPI_SUCCESS;
}
@ -400,17 +447,15 @@ int mca_btl_elan_put( mca_btl_base_module_t* btl,
unsigned char* src_addr = (unsigned char*)src->seg_addr.pval;
size_t src_len = src->seg_len;
unsigned char* dst_addr = (unsigned char*)ompi_ptr_ltop(dst->seg_addr.lval);
/*size_t dst_len = dst->seg_len;*/
bufdesc_t * desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
frag->endpoint = endpoint;
frag->btl = elan_btl;
frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT;
/* opal_output(0, "put from %p to %d peer , %d\n", src_addr, peer, src_len); */
desc->eve = elan_put(elan_btl->state, src_addr, dst_addr, src_len, peer);
desc->frag = frag;
desc->next = NULL;
BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
return OMPI_SUCCESS;
}
@ -426,13 +471,27 @@ int mca_btl_elan_put( mca_btl_base_module_t* btl,
int mca_btl_elan_get( mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* descriptor )
mca_btl_base_descriptor_t* des )
{
/* mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl; */
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) descriptor;
mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl;
mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) des;
int peer = endpoint->elan_vp;
mca_btl_base_segment_t* src = des->des_src;
mca_btl_base_segment_t* dst = des->des_dst;
unsigned char* src_addr = (unsigned char*)src->seg_addr.pval;
size_t src_len = src->seg_len;
unsigned char* dst_addr = (unsigned char*)ompi_ptr_ltop(dst->seg_addr.lval);
/*size_t dst_len = dst->seg_len;*/
bufdesc_t * desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
frag->endpoint = endpoint;
/* TODO */
return OMPI_ERR_NOT_IMPLEMENTED;
frag->btl = elan_btl;
frag->type = MCA_BTL_ELAN_HDR_TYPE_GET;
/*opal_output(0, "get from %p to %d peer , %d\n", src_addr, peer, src_len); */
desc->eve = elan_get(elan_btl->state, src_addr, dst_addr, src_len, peer);
desc->frag = frag;
desc->next = NULL;
BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
return OMPI_SUCCESS;
}
@ -443,20 +502,17 @@ int mca_btl_elan_get( mca_btl_base_module_t* btl,
void cancel_elanRx(mca_btl_elan_module_t* elan_btl)
{
bufdesc_t * index = elan_btl->tportFIFOHead;
int i=0;
while(index!= NULL)
{
if(index->frag->type == MCA_BTL_ELAN_HDR_TYPE_RECV)
{
if(elan_tportRxCancel(index->eve))
{
i++;
MCA_BTL_TEMPLATE_FRAG_RETURN(index->frag);
}
}
index = index->next;
}
opal_output(0, "cancel no. is %d\n", i);
}
int mca_btl_elan_finalize( struct mca_btl_base_module_t* btl )
@ -492,20 +548,20 @@ int mca_btl_elan_ft_event(int state) {
bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl)
{
bufdesc_t * desc = elan_btl->tportFIFOHead;
if( NULL == desc )
return NULL;
if( MCA_BTL_ELAN_HDR_TYPE_RECV == desc->frag->type ) {
if(MCA_BTL_ELAN_HDR_TYPE_RECV == desc->frag->type) {
if( !elan_tportRxDone(desc->eve) )
return NULL;
} else if( MCA_BTL_ELAN_HDR_TYPE_SEND == desc->frag->type ) {
}else if(MCA_BTL_ELAN_HDR_TYPE_SEND == desc->frag->type) {
if( !elan_tportTxDone(desc->eve) )
return NULL;
} else {
if( !elan_done(desc->eve,0) )
}else if(MCA_BTL_ELAN_HDR_TYPE_PUT == desc->frag->type || MCA_BTL_ELAN_HDR_TYPE_GET == desc->frag->type) {
if( !elan_done(desc->eve,0))
return NULL;
}else {
return NULL;
}
elan_btl->tportFIFOHead = elan_btl->tportFIFOHead->next;
if( NULL == elan_btl->tportFIFOHead )
elan_btl->tportFIFOTail = NULL;

6
ompi/mca/btl/elan/btl_elan.h
Просмотреть файл

@ -116,9 +116,7 @@ struct mca_btl_elan_module_t {
opal_mutex_t elan_lock;
struct bufdesc_t * tportFIFOHead;
struct bufdesc_t * tportFIFOTail;
#if defined MCA_BTL_HAS_MPOOL
struct mca_mpool_base_module_t* elan_mpool;
#endif
};
typedef struct mca_btl_elan_module_t mca_btl_elan_module_t;
extern mca_btl_elan_module_t mca_btl_elan_module;
@ -349,7 +347,7 @@ extern bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl);
#define BTL_ELAN_ADD_TO_FIFO(BTL, DESC) \
do { \
OPAL_THREAD_LOCK(&elan_btl->elan_lock); \
OPAL_THREAD_LOCK(&((BTL)->elan_lock)); \
if( (BTL)->tportFIFOTail ) { \
(BTL)->tportFIFOTail->next = (DESC); \
(BTL)->tportFIFOTail = (DESC); \
@ -357,7 +355,7 @@ extern bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl);
(BTL)->tportFIFOHead = (DESC); \
(BTL)->tportFIFOTail = (DESC); \
} \
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); \
OPAL_THREAD_UNLOCK(&((BTL)->elan_lock)); \
} while(0)
#endif

79
ompi/mca/btl/elan/btl_elan_component.c
Просмотреть файл

@ -21,6 +21,7 @@
#include "opal/mca/base/mca_base_param.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "orte/mca/errmgr/errmgr.h"
#include "ompi/mca/mpool/base/base.h"
#include "btl_elan.h"
@ -35,7 +36,7 @@
#include "opal/util/os_path.h"
#include "opal/util/opal_environ.h"
#include "ompi/communicator/communicator.h"
mca_btl_elan_component_t mca_btl_elan_component = {
{
/* First, the mca_base_component_t struct containing meta information
@ -59,7 +60,6 @@ mca_btl_elan_component_t mca_btl_elan_component = {
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_btl_elan_component_init,
mca_btl_elan_component_progress,
}
@ -113,11 +113,12 @@ int mca_btl_elan_component_open(void)
mca_btl_elan_module.super.btl_exclusivity = 0;
mca_btl_elan_module.super.btl_eager_limit = 32*1024;
mca_btl_elan_module.super.btl_min_send_size = 32*1024;
mca_btl_elan_module.super.btl_max_send_size = 32*1024; /*64*1024;*/
mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 *1024;
mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 *1024;
mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128* 1024;
mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | /*MCA_BTL_FLAGS_PUT | */MCA_BTL_FLAGS_SEND;
mca_btl_elan_module.super.btl_max_send_size = 64*1024; /*64*1024;*/
mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 * 1024;
mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128 * 1024;
mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE /*| MCA_BTL_FLAGS_GET */| MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND ;
/* mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE|MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND ;*/
mca_btl_elan_module.super.btl_bandwidth = 2000;
mca_btl_elan_module.super.btl_latency = 5;
mca_btl_base_param_register(&mca_btl_elan_component.super.btl_version,
@ -132,7 +133,7 @@ int mca_btl_elan_component_open(void)
int mca_btl_elan_component_close(void)
{
if( NULL != mca_btl_elan_component.elan_btls )
if( NULL != mca_btl_elan_component.elan_btls ) {
free( mca_btl_elan_component.elan_btls );
/* release resources */
@ -141,7 +142,7 @@ int mca_btl_elan_component_close(void)
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_user);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_max);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_lock);
}
return OMPI_SUCCESS;
}
@ -158,7 +159,7 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
{
mca_btl_base_module_t** btls;
size_t rails, i , count;
size_t rails, i , count, vpid;
ELAN_BASE * base;
ELAN_STATE * state;
ELAN_QUEUE * q= NULL;
@ -198,37 +199,12 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
mca_btl_elan_component.elan_free_list_inc,
NULL ); /* use default allocator */
opal_setenv( "LIBELAN_MACHINES_FILE", "/home/tma/machinefile", false, &environ );
opal_setenv( "MPIRUN_ELANIDMAP_FILE", "/etc/elanidmap", false, &environ );
base = elan_baseInit(0);
if (base == NULL)
return NULL;
state = base->state;
if( NULL == state ) {
mca_btl_base_error_no_nics( "ELAN", "Quadrics" );
return NULL;
}
elan_gsync(base->allGroup);
ompi_modex_send( &mca_btl_elan_component.super.btl_version, &state->vp, sizeof(state->vp));
rails = elan_nRails(state);
mca_btl_elan_component.elan_num_btls = rails;
if ((q = elan_allocQueue(base->state)) == NULL) {
return NULL;
}
if (!(p = elan_tportInit(base->state,
(ELAN_QUEUE *)q,
base->tport_nslots,
base->tport_smallmsg,
base->tport_bigmsg,
base->tport_stripemsg,
ELAN_POLL_EVENT,
base->retryCount,
&base->shm_key,
base->shm_fifodepth,
base->shm_fragsize,
0))) {
return NULL;
}
vpid = orte_process_info.my_name->vpid;
ompi_modex_send( &mca_btl_elan_component.super.btl_version, &vpid, sizeof(vpid));
mca_btl_elan_component.elan_num_btls = 1;
mca_btl_elan_component.elan_btls = malloc( (mca_btl_elan_component.elan_num_btls) * sizeof(mca_btl_base_module_t*) );
for( i = count = 0; i < mca_btl_elan_component.elan_num_btls; i++ ) {
mca_btl_elan_module_t* btl = malloc (sizeof (mca_btl_elan_module_t));
@ -238,12 +214,6 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
OBJ_CONSTRUCT (&btl->elan_lock, opal_mutex_t);
btl->tportFIFOHead=NULL;
btl->tportFIFOTail=NULL;
btl->base = base;
btl->state = state;
btl->queue = q;
btl->tport = p;
btl->elan_vp = state->vp;
btl->elan_nvp = state->nvp;
mca_btl_elan_component.elan_btls[count++] = btl;
}
mca_btl_elan_component.elan_num_btls = count ;
@ -294,8 +264,11 @@ int mca_btl_elan_component_progress()
{
/* it's a put*/
/* call the completion callback */
//opal_output(0, "I am a RDMA\n");
elan_wait(desc->eve,ELAN_WAIT_EVENT);
//opal_output(0, "I am a RDMA_done\n");
frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint, &(frag->base), OMPI_SUCCESS );
//opal_output(0, "I am a RDMA_cb_done\n");
free(desc);
}
else{
@ -315,7 +288,17 @@ int mca_btl_elan_component_progress()
desc->frag = frag;
desc->next = NULL;
BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
OPAL_THREAD_LOCK(&elan_btl->elan_lock);
if(elan_btl->tportFIFOTail)
{
elan_btl->tportFIFOTail->next = desc;
elan_btl->tportFIFOTail=desc;
}
else{
elan_btl->tportFIFOHead = desc;
elan_btl->tportFIFOTail = desc;
}
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
}
} else {
opal_output( 0, "Something bad happened the frag == NULL\n" );
@ -326,5 +309,3 @@ int mca_btl_elan_component_progress()
return num_progressed;
}

2
ompi/mca/btl/elan/btl_elan_endpoint.c
Просмотреть файл

@ -1,5 +1,5 @@
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$

2
ompi/mca/btl/elan/btl_elan_frag.c
Просмотреть файл

@ -1,5 +1,5 @@
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$

2
ompi/mca/btl/elan/btl_elan_frag.h
Просмотреть файл

@ -45,9 +45,7 @@ struct mca_btl_elan_frag_t {
ompi_free_list_t* my_list;
mca_btl_base_tag_t tag;
size_t size;
#if defined MCA_BTL_HAS_MPOOL
struct mca_mpool_base_registration_t* registration;
#endif
};
typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_frag_t);

35
ompi/mca/btl/elan/btl_elan_proc.c
Просмотреть файл

@ -1,5 +1,6 @@
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
@ -13,15 +14,16 @@
#include "orte/class/orte_proc_table.h"
#include "opal/class/opal_hash_table.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "btl_elan.h"
#include "btl_elan_proc.h"
#include "orte/util/proc_info.h"
#include "ompi/communicator/communicator.h"
static void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc);
static void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_btl_elan_proc_t,
opal_list_item_t, mca_btl_elan_proc_construct,
opal_list_item_t,mca_btl_elan_proc_construct,
mca_btl_elan_proc_destruct);
void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc)
@ -47,7 +49,6 @@ void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc)
OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
opal_list_remove_item(&mca_btl_elan_component.elan_procs, &proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
/* release resources */
if(NULL != proc->proc_endpoints) {
free(proc->proc_endpoints);
@ -62,24 +63,18 @@ void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc)
static mca_btl_elan_proc_t* mca_btl_elan_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_btl_elan_proc_t* elan_proc;
OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
for(elan_proc = (mca_btl_elan_proc_t*)
opal_list_get_first(&mca_btl_elan_component.elan_procs);
elan_proc != (mca_btl_elan_proc_t*)
opal_list_get_end(&mca_btl_elan_component.elan_procs);
elan_proc = (mca_btl_elan_proc_t*)opal_list_get_next(elan_proc)) {
if(elan_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
return elan_proc;
}
}
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
return NULL;
}
@ -96,19 +91,15 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
int rc;
size_t size;
mca_btl_elan_proc_t* module_proc = NULL;
/* Check if we have already created a Elan proc
* structure for this ompi process */
module_proc = mca_btl_elan_proc_lookup_ompi(ompi_proc);
if(module_proc != NULL) {
/* Gotcha! */
return module_proc;
}
/* Oops! First time, gotta create a new Elan proc
* out of the ompi_proc ... */
module_proc = OBJ_NEW(mca_btl_elan_proc_t);
if(NULL == module_proc)
return NULL;
@ -119,15 +110,6 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
/* build a unique identifier (of arbitrary
* size) to represent the proc */
module_proc->proc_guid = ompi_proc->proc_name;
/* Elan module doesn't have addresses exported at
* initialization, so the addr_count is set to one. */
/**
orte_hash_table_set_proc( &mca_btl_elan_component.elan_procs,
&module_proc->proc_guid,
module_proc );*/
rc = ompi_modex_recv( &mca_btl_elan_component.super.btl_version,
ompi_proc,
(void**)&module_proc->elan_vp_array,
@ -141,11 +123,11 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
/* XXX: Right now, there can be only 1 peer associated
* with a proc. Needs a little bit change in
* mca_btl_elan_proc_t to allow on demand increasing of
* number of endpoints for this proc */
* number of endpoints for this proc
*/
module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc((1+module_proc->proc_addr_count )* sizeof(mca_btl_base_endpoint_t*));
if(NULL == module_proc->proc_endpoints) {
OBJ_RELEASE(module_proc);
return NULL;
@ -164,12 +146,11 @@ int mca_btl_elan_proc_insert( mca_btl_elan_proc_t* module_proc,
{
/* insert into endpoint array */
size_t i;
module_endpoint->endpoint_proc = module_proc;
module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint;
for( i = 0; i < module_proc->proc_addr_count; i++ ) {
module_endpoint->elan_vp = module_proc->elan_vp_array[i];
/* module_endpoint->elan_vp = ompi_comm_rank(&ompi_mpi_comm_world);*/
return OMPI_SUCCESS;
}
return OMPI_SUCCESS;