ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Update the elan BTL. Now we support the following protocols: send, put

Просмотр исходного кода 
and partially get.

This commit was SVN r15564.
Этот коммит содержится в:
George Bosilca 2007-07-23 19:07:13 +00:00
родитель f219cc1e6e
Коммит 21a7670390
7 изменённых файлов: 637 добавлений и 623 удалений

1082
ompi/mca/btl/elan/btl_elan.c

Разница между файлами не показана из-за своего большого размера Загрузить разницу

6
ompi/mca/btl/elan/btl_elan.h

@ -116,9 +116,7 @@ struct mca_btl_elan_module_t {
opal_mutex_t elan_lock;
struct bufdesc_t * tportFIFOHead;
struct bufdesc_t * tportFIFOTail;
#if defined MCA_BTL_HAS_MPOOL
struct mca_mpool_base_module_t* elan_mpool;
#endif
};
typedef struct mca_btl_elan_module_t mca_btl_elan_module_t;
extern mca_btl_elan_module_t mca_btl_elan_module;
@ -349,7 +347,7 @@ extern bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl);
#define BTL_ELAN_ADD_TO_FIFO(BTL, DESC) \
do { \
OPAL_THREAD_LOCK(&elan_btl->elan_lock); \
OPAL_THREAD_LOCK(&((BTL)->elan_lock)); \
if( (BTL)->tportFIFOTail ) { \
(BTL)->tportFIFOTail->next = (DESC); \
(BTL)->tportFIFOTail = (DESC); \
@ -357,7 +355,7 @@ extern bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl);
(BTL)->tportFIFOHead = (DESC); \
(BTL)->tportFIFOTail = (DESC); \
} \
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); \
OPAL_THREAD_UNLOCK(&((BTL)->elan_lock)); \
} while(0)
#endif

111
ompi/mca/btl/elan/btl_elan_component.c

@ -21,6 +21,7 @@
#include "opal/mca/base/mca_base_param.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "orte/mca/errmgr/errmgr.h"
#include "ompi/mca/mpool/base/base.h"
#include "btl_elan.h"
@ -35,7 +36,7 @@
#include "opal/util/os_path.h"
#include "opal/util/opal_environ.h"
#include "ompi/communicator/communicator.h"
mca_btl_elan_component_t mca_btl_elan_component = {
{
/* First, the mca_base_component_t struct containing meta information
@ -59,7 +60,6 @@ mca_btl_elan_component_t mca_btl_elan_component = {
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_btl_elan_component_init,
mca_btl_elan_component_progress,
}
@ -113,11 +113,12 @@ int mca_btl_elan_component_open(void)
mca_btl_elan_module.super.btl_exclusivity = 0;
mca_btl_elan_module.super.btl_eager_limit = 32*1024;
mca_btl_elan_module.super.btl_min_send_size = 32*1024;
mca_btl_elan_module.super.btl_max_send_size = 32*1024; /*64*1024;*/
mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 *1024;
mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 *1024;
mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128* 1024;
mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | /*MCA_BTL_FLAGS_PUT | */MCA_BTL_FLAGS_SEND;
mca_btl_elan_module.super.btl_max_send_size = 64*1024; /*64*1024;*/
mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 * 1024;
mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128 * 1024;
mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE /*| MCA_BTL_FLAGS_GET */| MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND ;
/* mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE|MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND ;*/
mca_btl_elan_module.super.btl_bandwidth = 2000;
mca_btl_elan_module.super.btl_latency = 5;
mca_btl_base_param_register(&mca_btl_elan_component.super.btl_version,
@ -132,16 +133,16 @@ int mca_btl_elan_component_open(void)
int mca_btl_elan_component_close(void)
{
if( NULL != mca_btl_elan_component.elan_btls )
if( NULL != mca_btl_elan_component.elan_btls ) {
free( mca_btl_elan_component.elan_btls );
/* release resources */
/* release resources */
OBJ_DESTRUCT(&mca_btl_elan_component.elan_procs);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_eager);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_user);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_max);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_lock);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_procs);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_eager);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_user);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_max);
OBJ_DESTRUCT(&mca_btl_elan_component.elan_lock);
}
return OMPI_SUCCESS;
}
@ -158,7 +159,7 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
{
mca_btl_base_module_t** btls;
size_t rails, i , count;
size_t rails, i , count, vpid;
ELAN_BASE * base;
ELAN_STATE * state;
ELAN_QUEUE * q= NULL;
@ -197,61 +198,30 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
mca_btl_elan_component.elan_free_list_max,
mca_btl_elan_component.elan_free_list_inc,
NULL ); /* use default allocator */
opal_setenv( "LIBELAN_MACHINES_FILE", "/home/tma/machinefile", false, &environ );
opal_setenv( "MPIRUN_ELANIDMAP_FILE", "/etc/elanidmap", false, &environ );
base = elan_baseInit(0);
if (base == NULL)
return NULL;
state = base->state;
if( NULL == state ) {
mca_btl_base_error_no_nics( "ELAN", "Quadrics" );
return NULL;
}
elan_gsync(base->allGroup);
ompi_modex_send( &mca_btl_elan_component.super.btl_version, &state->vp, sizeof(state->vp));
rails = elan_nRails(state);
mca_btl_elan_component.elan_num_btls = rails;
if ((q = elan_allocQueue(base->state)) == NULL) {
return NULL;
}
if (!(p = elan_tportInit(base->state,
(ELAN_QUEUE *)q,
base->tport_nslots,
base->tport_smallmsg,
base->tport_bigmsg,
base->tport_stripemsg,
ELAN_POLL_EVENT,
base->retryCount,
&base->shm_key,
base->shm_fifodepth,
base->shm_fragsize,
0))) {
return NULL;
}
vpid = orte_process_info.my_name->vpid;
ompi_modex_send( &mca_btl_elan_component.super.btl_version, &vpid, sizeof(vpid));
mca_btl_elan_component.elan_num_btls = 1;
mca_btl_elan_component.elan_btls = malloc( (mca_btl_elan_component.elan_num_btls) * sizeof(mca_btl_base_module_t*) );
for( i = count = 0; i < mca_btl_elan_component.elan_num_btls; i++ ) {
mca_btl_elan_module_t* btl = malloc (sizeof (mca_btl_elan_module_t));
if(NULL == btl)
continue;
memcpy( btl, &mca_btl_elan_module, sizeof(mca_btl_elan_module_t) );
OBJ_CONSTRUCT (&btl->elan_lock, opal_mutex_t);
mca_btl_elan_module_t* btl = malloc (sizeof (mca_btl_elan_module_t));
if(NULL == btl)
continue;
memcpy( btl, &mca_btl_elan_module, sizeof(mca_btl_elan_module_t) );
OBJ_CONSTRUCT (&btl->elan_lock, opal_mutex_t);
btl->tportFIFOHead=NULL;
btl->tportFIFOTail=NULL;
btl->base = base;
btl->state = state;
btl->queue = q;
btl->tport = p;
btl->elan_vp = state->vp;
btl->elan_nvp = state->nvp;
mca_btl_elan_component.elan_btls[count++] = btl;
mca_btl_elan_component.elan_btls[count++] = btl;
}
mca_btl_elan_component.elan_num_btls = count ;
btls = (mca_btl_base_module_t**)malloc( (mca_btl_elan_component.elan_num_btls) * sizeof(mca_btl_base_module_t*) );
if( NULL == btls ) {
free( mca_btl_elan_component.elan_btls );
mca_btl_elan_component.elan_num_btls = 0; /* no active BTL modules */
return NULL;
free( mca_btl_elan_component.elan_btls );
mca_btl_elan_component.elan_num_btls = 0; /* no active BTL modules */
return NULL;
}
memcpy( btls, mca_btl_elan_component.elan_btls, mca_btl_elan_component.elan_num_btls *sizeof(mca_btl_elan_module_t*) );
*num_btl_modules = mca_btl_elan_component.elan_num_btls;
@ -293,9 +263,12 @@ int mca_btl_elan_component_progress()
else if(frag->type== MCA_BTL_ELAN_HDR_TYPE_PUT || frag->type== MCA_BTL_ELAN_HDR_TYPE_GET )
{
/* it's a put*/
/* call the completion callback */
/* call the completion callback */
//opal_output(0, "I am a RDMA\n");
elan_wait(desc->eve,ELAN_WAIT_EVENT);
//opal_output(0, "I am a RDMA_done\n");
frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint, &(frag->base), OMPI_SUCCESS );
//opal_output(0, "I am a RDMA_cb_done\n");
free(desc);
}
else{
@ -315,7 +288,17 @@ int mca_btl_elan_component_progress()
desc->frag = frag;
desc->next = NULL;
BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
OPAL_THREAD_LOCK(&elan_btl->elan_lock);
if(elan_btl->tportFIFOTail)
{
elan_btl->tportFIFOTail->next = desc;
elan_btl->tportFIFOTail=desc;
}
else{
elan_btl->tportFIFOHead = desc;
elan_btl->tportFIFOTail = desc;
}
OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
}
} else {
opal_output( 0, "Something bad happened the frag == NULL\n" );
@ -326,5 +309,3 @@ int mca_btl_elan_component_progress()
return num_progressed;
}

2
ompi/mca/btl/elan/btl_elan_endpoint.c

@ -1,5 +1,5 @@
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$

2
ompi/mca/btl/elan/btl_elan_frag.c

@ -1,5 +1,5 @@
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$

2
ompi/mca/btl/elan/btl_elan_frag.h

@ -45,9 +45,7 @@ struct mca_btl_elan_frag_t {
ompi_free_list_t* my_list;
mca_btl_base_tag_t tag;
size_t size;
#if defined MCA_BTL_HAS_MPOOL
struct mca_mpool_base_registration_t* registration;
#endif
};
typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_elan_frag_t);

55
ompi/mca/btl/elan/btl_elan_proc.c

@ -1,5 +1,6 @@
/*
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
@ -11,17 +12,18 @@
#include "ompi_config.h"
#include "orte/class/orte_proc_table.h"
#include "opal/class/opal_hash_table.h"
#include "opal/class/opal_hash_table.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "btl_elan.h"
#include "btl_elan_proc.h"
#include "orte/util/proc_info.h"
#include "ompi/communicator/communicator.h"
static void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc);
static void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_btl_elan_proc_t,
opal_list_item_t, mca_btl_elan_proc_construct,
opal_list_item_t,mca_btl_elan_proc_construct,
mca_btl_elan_proc_destruct);
void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc)
@ -47,7 +49,6 @@ void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc)
OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
opal_list_remove_item(&mca_btl_elan_component.elan_procs, &proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
/* release resources */
if(NULL != proc->proc_endpoints) {
free(proc->proc_endpoints);
@ -62,24 +63,18 @@ void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc)
static mca_btl_elan_proc_t* mca_btl_elan_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_btl_elan_proc_t* elan_proc;
OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
for(elan_proc = (mca_btl_elan_proc_t*)
opal_list_get_first(&mca_btl_elan_component.elan_procs);
opal_list_get_first(&mca_btl_elan_component.elan_procs);
elan_proc != (mca_btl_elan_proc_t*)
opal_list_get_end(&mca_btl_elan_component.elan_procs);
opal_list_get_end(&mca_btl_elan_component.elan_procs);
elan_proc = (mca_btl_elan_proc_t*)opal_list_get_next(elan_proc)) {
if(elan_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
return elan_proc;
}
}
OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
return NULL;
}
@ -96,22 +91,18 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
int rc;
size_t size;
mca_btl_elan_proc_t* module_proc = NULL;
/* Check if we have already created a Elan proc
* structure for this ompi process */
module_proc = mca_btl_elan_proc_lookup_ompi(ompi_proc);
if(module_proc != NULL) {
/* Gotcha! */
return module_proc;
}
/* Oops! First time, gotta create a new Elan proc
* out of the ompi_proc ... */
module_proc = OBJ_NEW(mca_btl_elan_proc_t);
if(NULL == module_proc)
return NULL;
if(NULL == module_proc)
return NULL;
/* Initialize number of peer */
module_proc->proc_endpoint_count = 0;
module_proc->proc_ompi = ompi_proc;
@ -119,33 +110,24 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
/* build a unique identifier (of arbitrary
* size) to represent the proc */
module_proc->proc_guid = ompi_proc->proc_name;
/* Elan module doesn't have addresses exported at
* initialization, so the addr_count is set to one. */
/**
orte_hash_table_set_proc( &mca_btl_elan_component.elan_procs,
&module_proc->proc_guid,
module_proc );*/
rc = ompi_modex_recv( &mca_btl_elan_component.super.btl_version,
ompi_proc,
(void**)&module_proc->elan_vp_array,
&size );
if(rc != OMPI_SUCCESS) {
BTL_ERROR(("mca_base_modex_recv: failed with return value=%d", rc));
OBJ_RELEASE(module_proc);
return NULL;
BTL_ERROR(("mca_base_modex_recv: failed with return value=%d", rc));
OBJ_RELEASE(module_proc);
return NULL;
}
module_proc->proc_addr_count = size / sizeof(unsigned int);;
/* XXX: Right now, there can be only 1 peer associated
* with a proc. Needs a little bit change in
* mca_btl_elan_proc_t to allow on demand increasing of
* number of endpoints for this proc */
* number of endpoints for this proc
*/
module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc((1+module_proc->proc_addr_count )* sizeof(mca_btl_base_endpoint_t*));
malloc((1+module_proc->proc_addr_count )* sizeof(mca_btl_base_endpoint_t*));
if(NULL == module_proc->proc_endpoints) {
OBJ_RELEASE(module_proc);
return NULL;
@ -160,16 +142,15 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
* it an address.
*/
int mca_btl_elan_proc_insert( mca_btl_elan_proc_t* module_proc,
mca_btl_base_endpoint_t* module_endpoint )
mca_btl_base_endpoint_t* module_endpoint )
{
/* insert into endpoint array */
size_t i;
module_endpoint->endpoint_proc = module_proc;
module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint;
for( i = 0; i < module_proc->proc_addr_count; i++ ) {
module_endpoint->elan_vp = module_proc->elan_vp_array[i];
/* module_endpoint->elan_vp = ompi_comm_rank(&ompi_mpi_comm_world);*/
return OMPI_SUCCESS;
}
return OMPI_SUCCESS;