ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

A lot of MX fixes.

Просмотр исходного кода 
1. Allow MX bonding via btl_mx_bonding MCA parameter. With this on, Open MPI
  suppose that lib MX will do the bonding, and we will only return one BTL.
  Otherwise, we return as many as devices.
2. Decrease the memory footprint, by cleaning up what we store about the
  peers and how we store it.
3. Allow multiple MX routes that share the same mapper. In this particular
  case we will link by their nic_id.
4. Allow multiple MX routes with multiple mappers. In this case we match
  the NICs based on the last 6 digits of the mapper MAC.
5. Increase the size of the eager and rendez-vous eager limits in the
  case where we are unable to register an unexpected callback with MX.
6. Increase the default max number of MX fragments.
7. Increase the max number of MX BTLs.
8. Only allow mx_if_include and mx_if_exclude if we have acess to the
  mapper.

This commit was SVN r19788.
Этот коммит содержится в:
George Bosilca 2008-10-22 20:12:30 +00:00
родитель a39d8d62e7
Коммит 7dfdf3e907
6 изменённых файлов: 167 добавлений и 153 удалений

1
ompi/mca/btl/mx/btl_mx.c

@ -327,6 +327,7 @@ mca_btl_base_descriptor_t* mca_btl_mx_prepare_dst( struct mca_btl_base_module_t*
mx_segment.segment_ptr = frag->segment[0].seg_addr.pval;
mx_segment.segment_length = frag->segment[0].seg_len;
mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1,
frag->segment[0].seg_key.key64,
BTL_MX_PUT_MASK, NULL, &(frag->mx_request) );

7
ompi/mca/btl/mx/btl_mx.h

@ -1,9 +1,8 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@ -80,6 +79,8 @@ struct mca_btl_mx_component_t {
/**< true if we want to activate the MX support for shared memory */
int32_t mx_support_self;
/**< true if we want to activate the MX support for self communications */
int32_t mx_bonding;
/**< true if MX is in charge of doing the device bonding */
int32_t mx_use_unexpected;
/**< true if Open MPI is allowed to register an unexpected handler with the MX library */
@ -94,12 +95,14 @@ struct mca_btl_mx_component_t {
opal_mutex_t mx_lock; /**< lock for accessing module state */
#if MX_HAVE_MAPPER_STATE
char* mx_if_include; /**< include the following NICs */
char* mx_if_exclude; /**< Exclude the following NICs. These
* values are based on the last 6
* digits in hexadecimal of the MAC
* address of the mapper.
*/
#endif /* MX_HAVE_MAPPER_STATE */
};
typedef struct mca_btl_mx_component_t mca_btl_mx_component_t;

105
ompi/mca/btl/mx/btl_mx_component.c

@ -83,7 +83,7 @@ int mca_btl_mx_component_open(void)
OBJ_CONSTRUCT(&mca_btl_mx_component.mx_procs, opal_list_t);
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "max_btls",
"Maximum number of accepted Myrinet cards",
false, false, 1, &mca_btl_mx_component.mx_max_btls );
false, false, 10, &mca_btl_mx_component.mx_max_btls );
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "timeout",
"Timeout for connections",
false, false, MX_INFINITE, &mca_btl_mx_component.mx_timeout );
@ -99,6 +99,9 @@ int mca_btl_mx_component_open(void)
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "shared_mem",
"Enable the MX support for shared memory",
false, false, 0, &mca_btl_mx_component.mx_support_sharedmem );
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "bonding",
"True if the lib MX is in charge of doing the device bonding, false if Open MPI will take care of it.",
false, false, 0, &mca_btl_mx_component.mx_bonding );
#ifdef HAVE_MX_REGISTER_UNEXP_HANDLER
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "register_unexp",
"Enable the MX support for the unexpected request handler (Open MPI matching)",
@ -112,7 +115,7 @@ int mca_btl_mx_component_open(void)
false, false, 32, &mca_btl_mx_component.mx_free_list_inc );
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "free_list_max",
"Maximum number of request this device is allowed to allocate",
false, false, 128, &mca_btl_mx_component.mx_free_list_max );
false, false, 1024, &mca_btl_mx_component.mx_free_list_max );
mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_mx_component, "max_posted_recv",
"Number of received posted in advance. Increasing this number for"
@ -120,17 +123,24 @@ int mca_btl_mx_component_open(void)
" in performances",
false, false, 16, &mca_btl_mx_component.mx_max_posted_recv );
#if MX_HAVE_MAPPER_STATE
mca_base_param_reg_string( (mca_base_component_t*)&mca_btl_mx_component, "if_include",
"Myrinet card to use (last 6 digits from the mapper MAC)",
false, false, NULL, &mca_btl_mx_component.mx_if_include );
mca_base_param_reg_string( (mca_base_component_t*)&mca_btl_mx_component, "if_exclude",
"Myrinet card to avoid (last 6 digits from the mapper MAC)",
false, false, NULL, &mca_btl_mx_component.mx_if_exclude );
#endif /* MX_HAVE_MAPPER_STATE */
mca_btl_mx_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
mca_btl_mx_module.super.btl_eager_limit = 1024;
mca_btl_mx_module.super.btl_rndv_eager_limit = 1024;
mca_btl_mx_module.super.btl_max_send_size = 8*1024;
if( mca_btl_mx_component.mx_use_unexpected ) {
mca_btl_mx_module.super.btl_eager_limit = 1024;
mca_btl_mx_module.super.btl_rndv_eager_limit = 1024;
} else {
mca_btl_mx_module.super.btl_eager_limit = 4*1024;
mca_btl_mx_module.super.btl_rndv_eager_limit = 4*1024;
}
mca_btl_mx_module.super.btl_max_send_size = 32*1024;
mca_btl_mx_module.super.btl_rdma_pipeline_send_length = 256*1024;
mca_btl_mx_module.super.btl_rdma_pipeline_frag_size = 8*1024*1024;
mca_btl_mx_module.super.btl_min_rdma_pipeline_size = 0;
@ -165,6 +175,7 @@ int mca_btl_mx_component_close(void)
OBJ_DESTRUCT(&mca_btl_mx_component.mx_procs);
OBJ_DESTRUCT(&mca_btl_mx_component.mx_lock);
#if MX_HAVE_MAPPER_STATE
if( NULL != mca_btl_mx_component.mx_if_include ) {
free( mca_btl_mx_component.mx_if_include );
mca_btl_mx_component.mx_if_include = NULL;
@ -173,6 +184,7 @@ int mca_btl_mx_component_close(void)
free( mca_btl_mx_component.mx_if_exclude );
mca_btl_mx_component.mx_if_exclude = NULL;
}
#endif /* MX_HAVE_MAPPER_STATE */
return OMPI_SUCCESS;
}
@ -264,7 +276,6 @@ static mca_btl_mx_module_t* mca_btl_mx_create(uint64_t addr)
(ms.mapper_mac[5]));
}
#endif /* MX_HAVE_MAPPER_STATE */
/* Try to figure out if we are allowed to use this network */
snprintf( mapper_mac, 7, "%6x", mx_unique_network_id );
@ -281,6 +292,7 @@ static mca_btl_mx_module_t* mca_btl_mx_create(uint64_t addr)
nic_id, mapper_mac );*/
return NULL;
}
#endif /* MX_HAVE_MAPPER_STATE */
mx_btl = malloc(sizeof(mca_btl_mx_module_t));
if( NULL == mx_btl ) return NULL;
@ -420,13 +432,6 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
*num_btl_modules = 0;
if (enable_progress_threads) {
opal_output( 0, "mca_btl_mx_component_init: progress threads requested but not supported");
ompi_modex_send(&mca_btl_mx_component.super.btl_version,
NULL, 0);
return NULL;
}
/**
* As the MX MTL get initialized before the MX BTL it will call the
* mx_init and the environment variables set by the BTL will be useless.
@ -450,8 +455,7 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
/* First check if MX is available ... */
if( OMPI_SUCCESS != ompi_common_mx_initialize() ) {
ompi_modex_send(&mca_btl_mx_component.super.btl_version,
NULL, 0);
ompi_modex_send(&mca_btl_mx_component.super.btl_version, NULL, 0);
return NULL;
}
@ -484,30 +488,45 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
/* intialize process hash table */
OBJ_CONSTRUCT( &mca_btl_mx_component.mx_procs, opal_list_t );
/* get the number of card available on the system */
if( (status = mx_get_info( NULL, MX_NIC_COUNT, NULL, 0,
&mca_btl_mx_component.mx_num_btls, sizeof(uint32_t))) != MX_SUCCESS ) {
opal_output( 0, "mca_btl_mx_component_init: mx_get_info(MX_NIC_COUNT) failed with status %d(%s)\n",
status, mx_strerror(status) );
ompi_modex_send(&mca_btl_mx_component.super.btl_version,
NULL, 0);
return NULL;
if( mca_btl_mx_component.mx_bonding ) {
mca_btl_mx_component.mx_num_btls = 1; /* there is only one! */
} else {
/* get the number of card available on the system */
if( (status = mx_get_info( NULL, MX_NIC_COUNT, NULL, 0,
&mca_btl_mx_component.mx_num_btls, sizeof(uint32_t))) != MX_SUCCESS ) {
opal_output( 0, "mca_btl_mx_component_init: mx_get_info(MX_NIC_COUNT) failed with status %d(%s)\n",
status, mx_strerror(status) );
ompi_modex_send(&mca_btl_mx_component.super.btl_version, NULL, 0);
return NULL;
}
if (0 == mca_btl_mx_component.mx_num_btls) {
mca_btl_base_error_no_nics("Myrinet/MX", "NIC");
ompi_modex_send(&mca_btl_mx_component.super.btl_version, NULL, 0);
return NULL;
}
/* determine the NIC ids */
size = sizeof(uint64_t) * (mca_btl_mx_component.mx_num_btls + 1);
if( NULL == (nic_addrs = (uint64_t*)malloc(size)) )
return NULL;
if( (status = mx_get_info( NULL, MX_NIC_IDS, NULL, 0,
nic_addrs, size)) != MX_SUCCESS) {
opal_output(0, "mca_btl_mx_component_init: mx_get_info(MX_NICS_IDS) failed size = %ld [%s] #cards %d\n",
(unsigned long)size, mx_strerror(status), mca_btl_mx_component.mx_num_btls );
free(nic_addrs);
return NULL;
}
/*
* Limit ourselves to the number of devices requested by the users.
*/
if( mca_btl_mx_component.mx_num_btls > mca_btl_mx_component.mx_max_btls ) {
mca_btl_mx_component.mx_num_btls = mca_btl_mx_component.mx_max_btls;
}
}
if (0 == mca_btl_mx_component.mx_num_btls) {
mca_btl_base_error_no_nics("Myrinet/MX", "NIC");
ompi_modex_send(&mca_btl_mx_component.super.btl_version,
NULL, 0);
return NULL;
}
#if 0
/* check for limit on number of btls */
if(mca_btl_mx_component.mx_num_btls > mca_btl_mx_component.mx_max_btls)
mca_btl_mx_component.mx_num_btls = mca_btl_mx_component.mx_max_btls;
#endif
/* Now we know how many NIC are available on the system. We will create a BTL for each one
* and then give a pointer to the BTL to the upper level.
/* Now we know how many NIC are available on the system. We will create a BTL
* for each one and then give a pointer to the BTL to the upper level.
*/
mca_btl_mx_component.mx_btls = malloc( mca_btl_mx_component.mx_num_btls * sizeof(mca_btl_base_module_t*) );
if( NULL == mca_btl_mx_component.mx_btls ) {
@ -515,18 +534,6 @@ mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
return NULL;
}
/* determine the NIC ids */
size = sizeof(uint64_t) * (mca_btl_mx_component.mx_num_btls + 1);
if( NULL == (nic_addrs = (uint64_t*)malloc(size)) )
return NULL;
if( (status = mx_get_info( NULL, MX_NIC_IDS, NULL, 0,
nic_addrs, size)) != MX_SUCCESS) {
opal_output(0, "MX BTL error (mx_get_info failed) size = %ld [%s] #cards %d\n",
(unsigned long)size, mx_strerror(status), mca_btl_mx_component.mx_num_btls );
free(nic_addrs);
return NULL;
}
mx_addrs = (mca_btl_mx_addr_t*)calloc( mca_btl_mx_component.mx_num_btls, sizeof(mca_btl_mx_addr_t) );
if( NULL == mx_addrs ) {
free( nic_addrs );

4
ompi/mca/btl/mx/btl_mx_endpoint.h

@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@ -78,7 +78,7 @@ struct mca_btl_base_endpoint_t {
struct mca_btl_mx_proc_t* endpoint_proc;
/**< proc structure corresponding to endpoint */
struct mca_btl_mx_addr_t mx_peer;
struct mca_btl_mx_addr_t* mx_peer;
/** the address as reported by the peer */
mx_endpoint_addr_t mx_peer_addr;

138
ompi/mca/btl/mx/btl_mx_proc.c

@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@ -35,10 +35,9 @@ OBJ_CLASS_INSTANCE(mca_btl_mx_proc_t,
void mca_btl_mx_proc_construct(mca_btl_mx_proc_t* proc)
{
proc->proc_ompi = 0;
proc->proc_endpoints = NULL;
proc->proc_endpoint_count = 0;
proc->mx_peers_count = 0;
proc->mx_peers = NULL;
proc->mx_routing = NULL;
OBJ_CONSTRUCT(&proc->proc_lock, opal_mutex_t);
/* add to list of all proc instance */
OPAL_THREAD_LOCK(&mca_btl_mx_component.mx_lock);
@ -58,14 +57,14 @@ void mca_btl_mx_proc_destruct(mca_btl_mx_proc_t* proc)
OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock);
/* release resources */
if( NULL != proc->proc_endpoints ) {
free(proc->proc_endpoints);
proc->proc_endpoints = NULL;
}
if( NULL != proc->mx_peers ) {
free(proc->mx_peers);
proc->mx_peers = NULL;
}
if( NULL != proc->mx_routing ) {
free(proc->mx_routing);
proc->mx_routing = NULL;
}
}
@ -95,27 +94,26 @@ static mca_btl_mx_proc_t* mca_btl_mx_proc_lookup_ompi(ompi_proc_t* ompi_proc)
return NULL;
}
/*
/**
* Create a MX process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_btl_mx_proc_t instance. We cache
* additional data (specifically the list of mca_btl_mx_endpoint_t instances,
* and published addresses) associated w/ a given destination on this
* datastructure.
*/
mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc)
{
mca_btl_mx_proc_t* module_proc = NULL;
mca_btl_mx_addr_t *mx_peers;
int rc;
int i, j, rc, mx_peers_count, *mx_routing;
bool at_least_one_route = false;
size_t size;
/* Check if we have already created a MX proc
* structure for this ompi process */
module_proc = mca_btl_mx_proc_lookup_ompi(ompi_proc);
if( module_proc != NULL ) {
/* Gotcha! */
return module_proc;
return module_proc; /* Gotcha! */
}
/* query for the peer address info */
@ -135,47 +133,70 @@ mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc)
ORTE_NAME_PRINT(&ompi_proc->proc_name) );
return NULL;
}
/* Let's see if we have a way to connect to the remote proc using MX.
* Without the routing information from the mapper, it is pretty
* to do this. Right now, we base this connection detection on the last
* 6 digits of the mapper MAC.
*/
mx_peers_count = size / sizeof(mca_btl_mx_addr_t);
mx_routing = (int*)malloc( mx_peers_count * sizeof(int) );
for( i = 0; i < mx_peers_count; mx_routing[i++] = -1 );
for( i = 0; i < mx_peers_count; i++ ) {
mca_btl_mx_module_t* mx_btl;
#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
BTL_MX_ADDR_NTOH(mx_peers[rc]);
#endif
for( j = 0; j < mca_btl_mx_component.mx_num_btls; j++ ) {
mx_btl = mca_btl_mx_component.mx_btls[j];
if( mx_btl->mx_unique_network_id == mx_peers[j].unique_network_id ) {
/* There is at least one connection between these two nodes */
if( -1 == mx_routing[j] ) {
/* First connection */
mx_routing[j] = i;
at_least_one_route = true;
break;
}
/* If multiple remote endpoints match mine, we keep going. As a
* result we will match them in order, i.e. remote endpoint 0
* will be connected to local endpoint 0.
*/
}
}
}
if( false == at_least_one_route ) {
free(mx_routing);
return NULL;
}
module_proc = OBJ_NEW(mca_btl_mx_proc_t);
module_proc->proc_ompi = ompi_proc;
module_proc->mx_peers_count = size / sizeof(mca_btl_mx_addr_t);
#if OMPI_ENABLE_HETEROGENEOUS_SUPPORT
for (rc = 0 ; rc < module_proc->mx_peers_count ; ++rc) {
BTL_MX_ADDR_NTOH(mx_peers[rc]);
}
#endif
module_proc->mx_peers = mx_peers;
module_proc->proc_ompi = ompi_proc;
module_proc->mx_peers_count = mx_peers_count;
module_proc->mx_peers = mx_peers;
module_proc->mx_routing = mx_routing;
return module_proc;
}
/*
/**
* Note that this routine must be called with the lock on the process
* already held. Insert a btl instance into the proc array and assign
* it an address.
* it an address.
*/
int mca_btl_mx_proc_insert( mca_btl_mx_proc_t* module_proc,
mca_btl_mx_endpoint_t* module_endpoint )
{
mca_btl_mx_module_t* mx_btl;
int i, j;
int btl_index, peer_endpoint_index;
/**
* Check if there is any Myrinet network between myself and the peer
*/
for( i = 0; i < mca_btl_mx_component.mx_num_btls; i++ ) {
mx_btl = mca_btl_mx_component.mx_btls[i];
for( j = 0; j < module_proc->mx_peers_count; j++ ) {
if( mx_btl->mx_unique_network_id == module_proc->mx_peers[j].unique_network_id ) {
/* There is at least one connection between these two nodes */
goto create_peer_endpoint;
}
}
for( btl_index = 0; btl_index < mca_btl_mx_component.mx_num_btls; btl_index++ ) {
mx_btl = mca_btl_mx_component.mx_btls[btl_index];
peer_endpoint_index = module_proc->mx_routing[btl_index];
if( (-1 != peer_endpoint_index) && (mx_btl == module_endpoint->endpoint_btl) ) {
module_endpoint->mx_peer = module_proc->mx_peers + peer_endpoint_index;
module_endpoint->endpoint_proc = module_proc;
return OMPI_SUCCESS;
}
}
module_proc->mx_peers_count = 0;
/**
@ -183,26 +204,6 @@ int mca_btl_mx_proc_insert( mca_btl_mx_proc_t* module_proc,
* way to communicate with the peer.
*/
return OMPI_ERROR;
create_peer_endpoint:
mx_btl = module_endpoint->endpoint_btl;
for( j = 0; j < module_proc->mx_peers_count; j++ ) {
if( mx_btl->mx_unique_network_id == module_proc->mx_peers[j].unique_network_id ) {
module_endpoint->mx_peer.nic_id = module_proc->mx_peers[j].nic_id;
module_endpoint->mx_peer.endpoint_id = module_proc->mx_peers[j].endpoint_id;
break;
}
}
if( NULL == module_proc->proc_endpoints ) {
module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc(module_proc->mx_peers_count * sizeof(mca_btl_base_endpoint_t*));
if( NULL == module_proc->proc_endpoints ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
}
module_endpoint->endpoint_proc = module_proc;
return OMPI_SUCCESS;
}
int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint )
@ -215,8 +216,20 @@ int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint )
module_endpoint->status = MCA_BTL_MX_CONNECTION_PENDING;
retry_connect:
#if 0
{
uint64_t nic_id;
uint32_t endpoint_id;
mx_decompose_endpoint_addr( module_endpoint->endpoint_btl->mx_endpoint_addr,
&nic_id, &endpoint_id );
opal_output(0, "Connect local (nic_id %llx, endpoint_id %x) to \n"
" remote (nic_id %llx, endpoint_id %x)\n",
nic_id, endpoint_id,
module_endpoint->mx_peer->nic_id, module_endpoint->mx_peer->endpoint_id );
}
#endif
mx_status = mx_connect( module_endpoint->endpoint_btl->mx_endpoint,
module_endpoint->mx_peer.nic_id, module_endpoint->mx_peer.endpoint_id,
module_endpoint->mx_peer->nic_id, module_endpoint->mx_peer->endpoint_id,
mca_btl_mx_component.mx_filter, mca_btl_mx_component.mx_timeout, &mx_remote_addr );
if( MX_SUCCESS != mx_status ) {
if( MX_TIMEOUT == mx_status )
@ -225,13 +238,13 @@ int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint )
{
char peer_name[MX_MAX_HOSTNAME_LEN];
if( MX_SUCCESS != mx_nic_id_to_hostname( module_endpoint->mx_peer.nic_id, peer_name ) )
sprintf( peer_name, "unknown %lx nic_id", (long)module_endpoint->mx_peer.nic_id );
if( MX_SUCCESS != mx_nic_id_to_hostname( module_endpoint->mx_peer->nic_id, peer_name ) )
sprintf( peer_name, "unknown %lx nic_id", (long)module_endpoint->mx_peer->nic_id );
opal_output( 0, "mx_connect fail for %s with key %x (error %s)\n\tUnique ID (local %x remote %x)\n",
peer_name, mca_btl_mx_component.mx_filter, mx_strerror(mx_status),
module_endpoint->endpoint_btl->mx_unique_network_id,
module_endpoint->mx_peer.unique_network_id );
module_endpoint->mx_peer->unique_network_id );
}
module_endpoint->status = MCA_BTL_MX_NOT_REACHEABLE;
return OMPI_ERROR;
@ -239,6 +252,5 @@ int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint )
module_endpoint->mx_peer_addr = mx_remote_addr;
module_endpoint->status = MCA_BTL_MX_CONNECTED;
module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint;
return OMPI_SUCCESS;
}

65
ompi/mca/btl/mx/btl_mx_proc.h

@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@ -24,44 +24,35 @@
#include "btl_mx.h"
#include "btl_mx_endpoint.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
BEGIN_C_DECLS
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each
* BTL instance that attempts to open a connection to the process.
*/
struct mca_btl_mx_proc_t {
opal_list_item_t super;
/**< allow proc to be placed on a list */
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each BTL instance that attempts to open a connection to the process.
*/
struct mca_btl_mx_proc_t {
opal_list_item_t super;
/**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
mca_btl_mx_addr_t *mx_peers; /**< peers addresses */
int mx_peers_count;
int* mx_routing; /**< peer routing information */
opal_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_btl_mx_proc_t mca_btl_mx_proc_t;
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
OBJ_CLASS_DECLARATION(mca_btl_mx_proc_t);
mca_btl_mx_addr_t *mx_peers; /**< peers addresses */
int mx_peers_count;
mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc);
int mca_btl_mx_proc_insert(mca_btl_mx_proc_t*, mca_btl_base_endpoint_t*);
int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint );
struct mca_btl_base_endpoint_t **proc_endpoints;
/**< array of endpoints that have been created to access this proc */
END_C_DECLS
size_t proc_endpoint_count;
/**< number of endpoints */
opal_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_btl_mx_proc_t mca_btl_mx_proc_t;
OBJ_CLASS_DECLARATION(mca_btl_mx_proc_t);
mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc);
int mca_btl_mx_proc_insert(mca_btl_mx_proc_t*, mca_btl_base_endpoint_t*);
int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint );
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif
#endif /* MCA_BTL_MX_PROC_H */