From 35de50b8232e498eafcfaf13aa19da7511367fa8 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Tue, 17 Jul 2012 14:51:41 +0000 Subject: [PATCH] remove the elan btl This commit was SVN r26798. --- ompi/mca/btl/elan/Makefile.am | 39 -- ompi/mca/btl/elan/btl_elan.c | 662 ------------------------- ompi/mca/btl/elan/btl_elan.h | 141 ------ ompi/mca/btl/elan/btl_elan_component.c | 370 -------------- ompi/mca/btl/elan/btl_elan_endpoint.c | 48 -- ompi/mca/btl/elan/btl_elan_endpoint.h | 51 -- ompi/mca/btl/elan/btl_elan_frag.c | 65 --- ompi/mca/btl/elan/btl_elan_frag.h | 85 ---- ompi/mca/btl/elan/btl_elan_proc.c | 149 ------ ompi/mca/btl/elan/btl_elan_proc.h | 57 --- ompi/mca/btl/elan/configure.m4 | 76 --- 11 files changed, 1743 deletions(-) delete mode 100644 ompi/mca/btl/elan/Makefile.am delete mode 100644 ompi/mca/btl/elan/btl_elan.c delete mode 100644 ompi/mca/btl/elan/btl_elan.h delete mode 100644 ompi/mca/btl/elan/btl_elan_component.c delete mode 100644 ompi/mca/btl/elan/btl_elan_endpoint.c delete mode 100644 ompi/mca/btl/elan/btl_elan_endpoint.h delete mode 100644 ompi/mca/btl/elan/btl_elan_frag.c delete mode 100644 ompi/mca/btl/elan/btl_elan_frag.h delete mode 100644 ompi/mca/btl/elan/btl_elan_proc.c delete mode 100644 ompi/mca/btl/elan/btl_elan_proc.h delete mode 100644 ompi/mca/btl/elan/configure.m4 diff --git a/ompi/mca/btl/elan/Makefile.am b/ompi/mca/btl/elan/Makefile.am deleted file mode 100644 index 03ac3d4dbe..0000000000 --- a/ompi/mca/btl/elan/Makefile.am +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2004-2007 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -AM_CPPFLAGS = $(btl_elan_CPPFLAGS) - -btl_elan_sources = btl_elan.c btl_elan.h btl_elan_component.c btl_elan_endpoint.c \ - btl_elan_endpoint.h btl_elan_frag.c btl_elan_frag.h btl_elan_proc.c btl_elan_proc.h - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_ompi_btl_elan_DSO -component_noinst = -component_install = mca_btl_elan.la -else -component_noinst = libmca_btl_elan.la -component_install = -endif - -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) -mca_btl_elan_la_SOURCES = $(btl_elan_sources) -mca_btl_elan_la_LIBADD = $(btl_elan_LIBS) -mca_btl_elan_la_LDFLAGS = -module -avoid-version $(btl_elan_LDFLAGS) - -noinst_LTLIBRARIES = $(component_noinst) -libmca_btl_elan_la_SOURCES = $(btl_elan_sources) -libmca_btl_elan_la_LIBADD = $(btl_elan_LIBS) -libmca_btl_elan_la_LDFLAGS = -module -avoid-version $(btl_elan_LDFLAGS) diff --git a/ompi/mca/btl/elan/btl_elan.c b/ompi/mca/btl/elan/btl_elan.c deleted file mode 100644 index 2b882e0425..0000000000 --- a/ompi/mca/btl/elan/btl_elan.c +++ /dev/null @@ -1,662 +0,0 @@ -/* - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "opal/class/opal_bitmap.h" -#include "opal/util/output.h" -#include "ompi/mca/btl/btl.h" -#include "btl_elan.h" -#include "btl_elan_frag.h" -#include "btl_elan_proc.h" -#include "btl_elan_endpoint.h" - -#include "opal/datatype/opal_convertor.h" -#include "ompi/mca/btl/base/base.h" - -#include -#include "elan/elan.h" -#include "opal/util/os_path.h" -#include "opal/util/opal_environ.h" -#include "orte/util/proc_info.h" - -/** - * - */ -extern char** environ; - -/** - * Reduce function that compute the MAX over an array of integers. - */ -static void __reduce_max_fn( void *vin, void* vinout, int* count, void* handle ) -{ - int *in = (int*)vin, *inout = (int*)vinout; - int i; - - for( i = 0; i < (*count); i++ ) { - if( in[i] > inout[i] ) inout[i] = in[i]; - } -} - -/** - * PML->BTL notification of change in the process list. - * - * @param btl (IN) - * @param nprocs (IN) Number of processes - * @param procs (IN) Set of processes - * @param peers (OUT) Set of (optional) peer addressing info. - * @param peers (IN/OUT) Set of processes that are reachable via this BTL. - * @return OMPI_SUCCESS or error status on failure. - * - */ -static int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t **ompi_procs, - struct mca_btl_base_endpoint_t** peers, - opal_bitmap_t* reachable ) -{ - mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*)btl; - int i, j, rc; - char* filename; - FILE* file; - ELAN_BASE* base; - - filename = opal_os_path( false, orte_process_info.proc_session_dir, "ELAN_ID", NULL ); - file = fopen( filename, "w" ); - fprintf( file, "%s %d\n", ompi_proc_local_proc->proc_hostname, elan_btl->elan_position ); - - for(i = 0; i < (int)nprocs; i++) { - struct ompi_proc_t* ompi_proc = ompi_procs[i]; - mca_btl_elan_proc_t* elan_proc; - mca_btl_base_endpoint_t* elan_endpoint; - - /* Don't use Elan for local communications */ - if( ompi_proc_local_proc == ompi_proc ) - continue; - - if(NULL == (elan_proc = mca_btl_elan_proc_create(ompi_proc))) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - elan_endpoint = OBJ_NEW(mca_btl_elan_endpoint_t); - if(NULL == elan_endpoint) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - elan_endpoint->endpoint_btl = elan_btl; - - OPAL_THREAD_LOCK(&elan_proc->proc_lock); - rc = mca_btl_elan_proc_insert(elan_proc, elan_endpoint); - OPAL_THREAD_UNLOCK(&elan_proc->proc_lock); - - if( OMPI_SUCCESS != rc ) { - OBJ_RELEASE(elan_endpoint); - OBJ_RELEASE(elan_proc); - continue; - } - for( j = 0; j < (int)elan_proc->proc_rail_count; j++ ) { - fprintf( file, "%s %d\n", ompi_proc->proc_hostname, - elan_proc->position_id_array[j] ); - } - opal_bitmap_set_bit(reachable, i); - peers[i] = elan_endpoint; - } - fclose(file); - /* Set the environment before firing up the Elan library */ - opal_setenv( "LIBELAN_MACHINES_FILE", filename, true, &environ ); - opal_setenv( "MPIRUN_ELANIDMAP_FILE", mca_btl_elan_component.elanidmap_file, - false, &environ ); - - base = elan_baseInit(0); - if( NULL == base ) - return OMPI_ERR_OUT_OF_RESOURCE; - if( NULL == base->state ) { - mca_btl_base_error_no_nics( "ELAN", "Quadrics" ); - return OMPI_ERR_OUT_OF_RESOURCE; - } - elan_btl->base = base; - elan_btl->elan_vp = base->state->vp; - - { - unsigned int* vp_array = (unsigned int*)calloc( nprocs, sizeof(unsigned int) ); - /* Set my position in the array with the Elan vp */ - vp_array[(int)ompi_proc_local_proc->proc_name.vpid] = elan_btl->elan_vp; - - /* Do a reduce with the previously defined MAX function. The outcome will be - * that at each process vpid index we will have their Elan vp. With this Elan - * vp we can therefore communicate with the process. - */ - elan_reduce( base->allGroup, vp_array, vp_array, sizeof(unsigned int), (int)nprocs, - __reduce_max_fn, NULL, 0, 0, - ELAN_REDUCE_COMMUTE | ELAN_RESULT_ALL | base->group_flags, 0); - - for(i = 0; i < (int)nprocs; i++) { - if(NULL == peers[i]) - continue; - peers[i]->elan_vp = vp_array[(int)ompi_procs[i]->proc_name.vpid]; - } - free(vp_array); - } - - /* Create the tport global queue */ - if( (elan_btl->tport_queue = elan_gallocQueue(base, base->allGroup)) == NULL ) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - /* Create and initialize the tport */ - if( !(elan_btl->tport = elan_tportInit(base->state, - elan_btl->tport_queue, - mca_btl_elan_component.elan_max_posted_recv, - base->tport_smallmsg, - mca_btl_elan_module.super.btl_eager_limit, - base->tport_stripemsg, - ELAN_POLL_EVENT, - base->retryCount, - &base->shm_key, - base->shm_fifodepth, - base->shm_fragsize, - 0))) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - /* Create the receive queue */ - if( (elan_btl->global_queue = elan_gallocQueue(base, base->allGroup)) == NULL ) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - mca_btl_elan_component.queue_max_size = elan_queueMaxSlotSize( base->state ) - - sizeof(mca_btl_elan_hdr_t); - - elan_btl->rx_queue = - elan_queueRxInit( base->state, /* ELAN_STATE *state */ - elan_btl->global_queue, /* ELAN_QUEUE *queue */ - mca_btl_elan_component.elan_max_posted_recv, /* int nSlots */ - (int)mca_btl_elan_component.queue_max_size, /* int slotSize */ - ELAN_RAIL_ALL, /* int rail */ - 0 ); /* ELAN_FLAGS flags */ - - elan_btl->tx_queue = - elan_queueTxInit( base->state, /* ELAN_STATE *state */ - elan_btl->global_queue, /* ELAN_QUEUE *q */ - ELAN_RAIL_ALL, /* int rail */ - 0 ); /* ELAN_FLAGS flags */ - - for( i = 0; i < mca_btl_elan_component.elan_max_posted_recv; i++ ) { - mca_btl_elan_frag_t* frag; - - MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc ); - if( OPAL_UNLIKELY(NULL == frag) ) { - return OMPI_ERROR; - } - frag->segment.seg_addr.pval = (void*)(frag + 1); - frag->base.des_dst = &(frag->segment); - frag->base.des_dst_cnt = 1; - frag->base.des_src = NULL; - frag->base.des_src_cnt = 0; - frag->type = MCA_BTL_ELAN_HDR_TYPE_RECV; - frag->elan_event = elan_tportRxStart( elan_btl->tport, - ELAN_TPORT_RXBUF | ELAN_TPORT_RXANY, - 0, 0, 0, 0, - frag->segment.seg_addr.pval, - mca_btl_elan_module.super.btl_eager_limit ); - opal_list_append( &(elan_btl->recv_list), (opal_list_item_t*)frag ); - } - - /* enable the network */ - elan_enable_network( elan_btl->base->state ); - - return OMPI_SUCCESS; -} - -/** - * PML->BTL notification of change in the process list. - * - * @param btl (IN) BTL instance - * @param nproc (IN) Number of processes. - * @param procs (IN) Set of processes. - * @param peers (IN) Set of peer data structures. - * @return Status indicating if cleanup was successful - * - */ -static int mca_btl_elan_del_procs( struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t **procs, - struct mca_btl_base_endpoint_t ** endpoints ) -{ - return OMPI_SUCCESS; -} - -/** - * Allocate a descriptor with a segment of the requested size. - * Note that the BTL layer may choose to return a smaller size - * if it cannot support the request. - * - * @param btl (IN) BTL module - * @param size (IN) Request segment size. - */ -static mca_btl_base_descriptor_t* -mca_btl_elan_alloc( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* peer, - uint8_t order, - size_t size, - uint32_t flags ) -{ - mca_btl_elan_frag_t* frag = NULL; - ptrdiff_t hdr_skip = 0; - int rc; - - if( size <= btl->btl_eager_limit ) { - MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc); - if( size <= mca_btl_elan_component.queue_max_size ) { /* This will be going over the queue */ - hdr_skip = sizeof(mca_btl_elan_hdr_t); - } - } else if( size <= btl->btl_max_send_size ) { - MCA_BTL_ELAN_FRAG_ALLOC_MAX(frag, rc); - } - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - - frag->segment.seg_addr.pval = (void*)((char*)(frag + 1) + hdr_skip); - frag->segment.seg_len = size; - frag->base.des_src = &(frag->segment); - frag->base.des_src_cnt = 1; - frag->base.des_dst = NULL; - frag->base.des_dst_cnt = 0; - frag->base.des_flags = flags; - frag->btl = (mca_btl_elan_module_t*)btl; - frag->endpoint = peer; - frag->base.order = MCA_BTL_NO_ORDER; - return (mca_btl_base_descriptor_t*)frag; -} - - -/** - * Return a segment allocated by this BTL. - * - * @param btl (IN) BTL module - * @param descriptor (IN) Allocated descriptor. - */ -static int mca_btl_elan_free( struct mca_btl_base_module_t* btl, - mca_btl_base_descriptor_t* des ) -{ - mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)des; - MCA_BTL_ELAN_FRAG_RETURN(frag); - return OMPI_SUCCESS; -} - -/** - * Prepare a descriptor for send/rdma using the supplied - * convertor. If the convertor references data that is contigous, - * the descriptor may simply point to the user buffer. Otherwise, - * this routine is responsible for allocating buffer space and - * packing if required. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL peer addressing - * @param convertor (IN) Data type convertor - * @param reserve (IN) Additional bytes requested by upper layer to precede user data - * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) - */ -static mca_btl_base_descriptor_t* -mca_btl_elan_prepare_src( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags) -{ - mca_btl_elan_frag_t* frag; - struct iovec iov; - uint32_t iov_count = 1; - size_t max_data = *size, skip = 0; - int rc; - - if( OPAL_UNLIKELY(max_data > UINT32_MAX) ) { - max_data = (size_t)UINT32_MAX; - } - if( 0 != reserve ) { - if( max_data + reserve <= btl->btl_eager_limit ) { - MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc); - - if( (max_data + reserve) <= mca_btl_elan_component.queue_max_size ) { - skip = sizeof(mca_btl_elan_hdr_t); - } - } else { - MCA_BTL_ELAN_FRAG_ALLOC_MAX(frag, rc); - - if( (max_data + reserve) > btl->btl_max_send_size ) { - max_data = btl->btl_max_send_size - reserve; - } - } - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - frag->segment.seg_addr.pval = (void*)((unsigned char*)(frag + 1) + skip); - iov.iov_len = max_data; - iov.iov_base = (unsigned char*)frag->segment.seg_addr.pval + reserve; - - rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data ); - *size = max_data; - if( rc < 0 ) { - MCA_BTL_ELAN_FRAG_RETURN(frag); - return NULL; - } - frag->segment.seg_len = max_data + reserve; - } else { /* this is a real RDMA operation */ - MCA_BTL_ELAN_FRAG_ALLOC_USER(frag, rc); - if(OPAL_UNLIKELY(NULL == frag)) { - return NULL; - } - frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT; - iov.iov_len = max_data; - iov.iov_base = NULL; - opal_convertor_pack(convertor, &iov, &iov_count, &max_data); - *size = max_data; - frag->segment.seg_addr.pval = iov.iov_base; - frag->segment.seg_len = max_data; - } - - frag->base.des_src = &(frag->segment); - frag->base.des_src_cnt = 1; - frag->base.order = MCA_BTL_NO_ORDER; - frag->base.des_dst = NULL; - frag->base.des_dst_cnt = 0; - frag->base.des_flags = flags; - return &frag->base; - -} - -/** - * Prepare a descriptor for send/rdma using the supplied - * convertor. If the convertor references data that is contigous, - * the descriptor may simply point to the user buffer. Otherwise, - * this routine is responsible for allocating buffer space and - * packing if required. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL peer addressing - * @param convertor (IN) Data type convertor - * @param reserve (IN) Additional bytes requested by upper layer to precede user data - * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) - */ - -static mca_btl_base_descriptor_t* -mca_btl_elan_prepare_dst( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags ) -{ - mca_btl_elan_frag_t* frag; - size_t origin, position = *size; - int rc; - - if( OPAL_UNLIKELY((*size) > UINT32_MAX) ) { - *size = (size_t)UINT32_MAX; - } - MCA_BTL_ELAN_FRAG_ALLOC_USER(frag, rc); - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - - opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) ); - origin = convertor->bConverted; - position += origin; - opal_convertor_set_position( convertor, &position ); - *size = position - origin; - - frag->segment.seg_len = *size; - frag->base.des_src = NULL; - frag->base.des_src_cnt = 0; - frag->base.des_flags = flags; - frag->base.des_dst = &(frag->segment); - frag->base.des_dst_cnt = 1; - frag->base.order = MCA_BTL_NO_ORDER; - return &frag->base; -} - - -/** - * Initiate an asynchronous send. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transfered - * @param tag (IN) The tag value used to notify the peer. - */ - -static int mca_btl_elan_send( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_btl_base_descriptor_t* descriptor, - mca_btl_base_tag_t tag ) - -{ - mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl; - mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)descriptor; - mca_btl_elan_hdr_t* elan_hdr = (mca_btl_elan_hdr_t*)(frag+1); - int send_len; - - frag->btl = elan_btl; - frag->endpoint = endpoint; - frag->tag = tag; - frag->type = MCA_BTL_ELAN_HDR_TYPE_SEND; - - if( frag->segment.seg_len <= mca_btl_elan_component.queue_max_size ) { - elan_hdr->tag = (int)tag; - elan_hdr->length = (int)frag->segment.seg_len; - send_len = frag->segment.seg_len + sizeof(mca_btl_elan_hdr_t); - frag->elan_event = elan_queueTx( elan_btl->tx_queue, - endpoint->elan_vp, - (void*)elan_hdr, - send_len, ELAN_RAIL_ALL ); - if( OPAL_UNLIKELY(NULL == frag->elan_event) ) { - opal_output( 0, "elan_queueTx failed for destination %d\n", endpoint->elan_vp ); - return OMPI_ERROR; - } - } else { - frag->elan_event = elan_tportTxStart( elan_btl->tport, 0, endpoint->elan_vp, - elan_btl->elan_vp, frag->tag, - (void*)elan_hdr, frag->segment.seg_len ); - if( OPAL_UNLIKELY(NULL == frag->elan_event) ) { - opal_output( 0, "elan_tportTxStart failed for destination %d\n", endpoint->elan_vp ); - return OMPI_ERROR; - } - } - if( elan_poll( frag->elan_event, 0 ) ) { - int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP ); - - frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint, - &(frag->base), OMPI_SUCCESS ); - if( btl_ownership ) { - MCA_BTL_ELAN_FRAG_RETURN(frag); - } - return OMPI_SUCCESS; - } - /* Add the fragment to the pending send list */ - OPAL_THREAD_LOCK( &elan_btl->elan_lock ); - opal_list_append( &(elan_btl->send_list), (opal_list_item_t*)frag ); - OPAL_THREAD_UNLOCK( &elan_btl->elan_lock ); - return OMPI_ERR_RESOURCE_BUSY; -} - - -/** - * Initiate an asynchronous put. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transferred - */ -static int mca_btl_elan_put( mca_btl_base_module_t* btl, - mca_btl_base_endpoint_t* endpoint, - mca_btl_base_descriptor_t* des ) -{ - mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl; - mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) des; - int peer = endpoint->elan_vp; - mca_btl_base_segment_t* src = des->des_src; - mca_btl_base_segment_t* dst = des->des_dst; - unsigned char* src_addr = (unsigned char*)src->seg_addr.pval; - size_t src_len = src->seg_len; - unsigned char* dst_addr = (unsigned char*)dst->seg_addr.pval; - - frag->endpoint = endpoint; - frag->btl = elan_btl; - frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT; - frag->elan_event = elan_put(elan_btl->base->state, src_addr, dst_addr, src_len, peer); - /* Add the fragment to the pending RDMA list */ - OPAL_THREAD_LOCK( &elan_btl->elan_lock ); - opal_list_append( &(elan_btl->rdma_list), (opal_list_item_t*)frag ); - OPAL_THREAD_UNLOCK( &elan_btl->elan_lock ); - return OMPI_SUCCESS; -} - - -/** - * Initiate an asynchronous get. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transferred - * - */ - -static int mca_btl_elan_get( mca_btl_base_module_t* btl, - mca_btl_base_endpoint_t* endpoint, - mca_btl_base_descriptor_t* des ) -{ - mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl; - mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) des; - int peer = endpoint->elan_vp; - mca_btl_base_segment_t* src = des->des_src; - mca_btl_base_segment_t* dst = des->des_dst; - unsigned char* src_addr = (unsigned char*)src->seg_addr.pval; - size_t src_len = src->seg_len; - unsigned char* dst_addr = (unsigned char*)dst->seg_addr.lval; - - frag->endpoint = endpoint; - frag->btl = elan_btl; - frag->type = MCA_BTL_ELAN_HDR_TYPE_GET; - opal_output( 0, "elan_get( remote %p, local %p, length %d, peer %d )\n", - (void*)src_addr, (void*)dst_addr, (int)src_len, peer ); - frag->elan_event = elan_get(elan_btl->base->state, src_addr, dst_addr, src_len, peer); - /* Add the fragment to the pending RDMA list */ - OPAL_THREAD_LOCK( &elan_btl->elan_lock ); - opal_list_append( &(elan_btl->rdma_list), (opal_list_item_t*)frag ); - OPAL_THREAD_UNLOCK( &elan_btl->elan_lock ); - return OMPI_SUCCESS; -} - -int mca_btl_elan_finalize( struct mca_btl_base_module_t* btl ) -{ - mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl; - int i, num_btls; - - /* First find the correct BTL in the list attached to the component */ - num_btls = mca_btl_elan_component.elan_num_btls; - for( i = 0; i < num_btls; i++ ) { - if( elan_btl == mca_btl_elan_component.elan_btls[i] ) { - /* disable the network */ - elan_disable_network( elan_btl->base->state ); - - /* Remove it from the list attached to the component */ - if( i == (num_btls-1) ) { - mca_btl_elan_component.elan_btls[i] = NULL; - } else { - mca_btl_elan_component.elan_btls[i] = mca_btl_elan_component.elan_btls[num_btls-1]; - } - mca_btl_elan_component.elan_num_btls--; - - /* Cancel all posted receives */ - - /* Release the internal structures */ - OBJ_DESTRUCT(&elan_btl->recv_list); - OBJ_DESTRUCT(&elan_btl->send_list); - OBJ_DESTRUCT(&elan_btl->rdma_list); - OBJ_DESTRUCT(&elan_btl->elan_lock); - /* The BTL is clean, remove it */ - free(elan_btl); - - return OMPI_SUCCESS; - } - } - /* This BTL is not present in the list attached to the communicator */ - return OMPI_ERROR; -} - -int mca_btl_elan_ft_event(int state) -{ - if(OPAL_CRS_CHECKPOINT == state) { - ; - } - else if(OPAL_CRS_CONTINUE == state) { - ; - } - else if(OPAL_CRS_RESTART == state) { - ; - } - else if(OPAL_CRS_TERM == state ) { - ; - } - else { - ; - } - - return OMPI_SUCCESS; -} - -static void mca_btl_elan_dump( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - int verbose ) -{ -} - -static int -mca_btl_elan_register_error( struct mca_btl_base_module_t* btl, - mca_btl_base_module_error_cb_fn_t cbfunc ) -{ - return OMPI_SUCCESS; -} - -mca_btl_elan_module_t mca_btl_elan_module = { - { - &mca_btl_elan_component.super, - 0, /* max size of first fragment */ - 0, /* min send fragment size */ - 0, /* max send fragment size */ - 0, /* btl_rdma_pipeline_offset */ - 0, /* btl_rdma_pipeline_frag_size */ - 0, /* btl_min_rdma_pipeline_size */ - 0, /* exclusivity */ - 0, /* latency */ - 0, /* bandwidth */ - 0, /* flags */ - 0, /* segment size */ - mca_btl_elan_add_procs, - mca_btl_elan_del_procs, - NULL, /* btl_register */ - mca_btl_elan_finalize, - mca_btl_elan_alloc, - mca_btl_elan_free, - mca_btl_elan_prepare_src, - mca_btl_elan_prepare_dst, - mca_btl_elan_send, - NULL, /* send immediate */ - mca_btl_elan_put, - mca_btl_elan_get, - mca_btl_elan_dump, - NULL, /* mpool */ - mca_btl_elan_register_error, /* register error cb */ - mca_btl_elan_ft_event /* mca_btl_elan_ft_event*/ - } -}; diff --git a/ompi/mca/btl/elan/btl_elan.h b/ompi/mca/btl/elan/btl_elan.h deleted file mode 100644 index 58f252bb8c..0000000000 --- a/ompi/mca/btl/elan/btl_elan.h +++ /dev/null @@ -1,141 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; -*- */ -/* - * Copyright (c) 2004-2007 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ -#ifndef MCA_BTL_ELAN_H -#define MCA_BTL_ELAN_H - -#include "ompi_config.h" - -#include "opal/mca/event/event.h" -#include "ompi/class/ompi_free_list.h" -#include "ompi/mca/btl/btl.h" -#include "ompi/mca/btl/base/btl_base_error.h" - -#include "ompi/mca/btl/btl.h" -#include "ompi/mca/btl/base/base.h" -#include "btl_elan_endpoint.h" -#include "btl_elan_frag.h" - -#include "elan/elan.h" - -BEGIN_C_DECLS - -/** - * ELAN BTL component. - */ - -struct mca_btl_elan_component_t { - mca_btl_base_component_2_0_0_t super; /**< base BTL component */ - - size_t queue_max_size; - /**< maximum amount of data transfered using the queues */ - - uint32_t elan_num_btls; - /**< number of hcas available to the ELAN component */ - - struct mca_btl_elan_module_t **elan_btls; - /**< array of available BTL modules */ - - int elan_free_list_num; - /**< initial size of free lists */ - - int elan_free_list_max; - /**< maximum size of free lists */ - - int elan_free_list_inc; - /**< number of elements to alloc when growing free lists */ - - int elan_max_posted_recv; - /**< number of pre-posted receives */ - - /* free list of fragment descriptors */ - ompi_free_list_t elan_frag_eager; - ompi_free_list_t elan_frag_max; - ompi_free_list_t elan_frag_user; - - opal_list_t elan_procs; - /**< list of elan proc structures */ - - opal_mutex_t elan_lock; - /**< lock for accessing module state */ - - char* elanidmap_file; /**< name of the ELANIDMAP file */ -}; -typedef struct mca_btl_elan_component_t mca_btl_elan_component_t; - -OMPI_MODULE_DECLSPEC extern mca_btl_elan_component_t mca_btl_elan_component; - -/** - * BTL Module Interface - */ - -struct mca_btl_elan_module_t { - mca_btl_base_module_t super; /**< base BTL interface */ - int expect_tport_recv; - int elan_vp; - int elan_position; /**< position of this elan interface */ - ELAN_BASE* base; - ELAN_TPORT* tport; - ELAN_QUEUE* global_queue; /**< The global queue */ - ELAN_QUEUE* tport_queue; - ELAN_QUEUE_RX* rx_queue; /**< The local receive queue */ - ELAN_QUEUE_TX* tx_queue; /**< The global send queue */ - opal_mutex_t elan_lock; - opal_list_t send_list; /**< list of posted sends */ - opal_list_t rdma_list; /**< list of posted receives */ - opal_list_t recv_list; -}; -typedef struct mca_btl_elan_module_t mca_btl_elan_module_t; -extern mca_btl_elan_module_t mca_btl_elan_module; - -typedef struct mca_btl_elan_hdr_t { - int tag; - int length; -} mca_btl_elan_hdr_t; - -/** - * Register ELAN component parameters with the MCA framework - */ -extern int mca_btl_elan_component_open(void); - -/** - * Any final cleanup before being unloaded. - */ -extern int mca_btl_elan_component_close(void); - -/** - * ELAN component initialization. - * - * @param num_btl_modules (OUT) Number of BTLs returned in BTL array. - * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE) - * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE) - */ -extern mca_btl_base_module_t** -mca_btl_elan_component_init( int* num_btl_modules, - bool allow_multi_user_threads, - bool have_hidden_threads ); - -/** - * ELAN component progress. - */ -extern int mca_btl_elan_component_progress(void); - -extern int mca_btl_elan_finalize( struct mca_btl_base_module_t* btl ); - -extern int mca_btl_elan_ft_event(int state); - -END_C_DECLS - -#endif /* MCA_BTL_ELAN_H */ - diff --git a/ompi/mca/btl/elan/btl_elan_component.c b/ompi/mca/btl/elan/btl_elan_component.c deleted file mode 100644 index fc06329c82..0000000000 --- a/ompi/mca/btl/elan/btl_elan_component.c +++ /dev/null @@ -1,370 +0,0 @@ -/* - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include "ompi/constants.h" -#include "opal/mca/event/event.h" -#include "opal/util/output.h" -#include "ompi/mca/btl/btl.h" - -#include "opal/mca/base/mca_base_param.h" -#include "ompi/runtime/ompi_module_exchange.h" - -#include "ompi/mca/mpool/base/base.h" - -#include "btl_elan.h" -#include "btl_elan_frag.h" -#include "btl_elan_endpoint.h" - -#include "ompi/mca/btl/base/base.h" -#include "ompi/mca/btl/base/btl_base_error.h" - -#include "elan/elan.h" - -#include "opal/util/opal_environ.h" - -#define ELAN_MAX_BTL 10 - -mca_btl_elan_component_t mca_btl_elan_component = { - { - /* First, the mca_base_component_t struct containing meta information - about the component itself */ - - { - MCA_BTL_BASE_VERSION_2_0_0, - - "elan", /* MCA component name */ - OMPI_MAJOR_VERSION, /* MCA component major version */ - OMPI_MINOR_VERSION, /* MCA component minor version */ - OMPI_RELEASE_VERSION, /* MCA component release version */ - mca_btl_elan_component_open, /* component open */ - mca_btl_elan_component_close /* component close */ - }, - { - /* The component is not checkpoint ready */ - MCA_BASE_METADATA_PARAM_NONE - }, - mca_btl_elan_component_init, - mca_btl_elan_component_progress, - } -}; - - -/* - * utility routines for parameter registration - */ - -static inline int -mca_btl_elan_param_register_int( const char* param_name, - int default_value ) -{ - int id = mca_base_param_register_int("btl","elan",param_name,NULL,default_value); - int param_value = default_value; - mca_base_param_lookup_int(id,¶m_value); - return param_value; -} - -/* - * Called by MCA framework to open the component, registers - * component parameters. - */ - -int mca_btl_elan_component_open(void) -{ - /* initialize state */ - mca_btl_elan_component.elan_num_btls = 0; - mca_btl_elan_component.elan_btls = NULL; - - mca_btl_elan_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT; - mca_btl_elan_module.super.btl_eager_limit = 32*1024; - mca_btl_elan_module.super.btl_rndv_eager_limit = mca_btl_elan_module.super.btl_eager_limit; - mca_btl_elan_module.super.btl_max_send_size = 64*1024; /*64*1024;*/ - mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 * 1024; - mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 * 1024; - mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128 * 1024; - mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND; - mca_btl_elan_module.super.btl_seg_size = sizeof (mca_btl_base_segment_t); - mca_btl_elan_module.super.btl_bandwidth = 1959; - mca_btl_elan_module.super.btl_latency = 4; - mca_btl_base_param_register(&mca_btl_elan_component.super.btl_version, - &mca_btl_elan_module.super); - - mca_base_param_reg_string( (mca_base_component_t*)&mca_btl_elan_component, "elanidmap", - "System-wide configuration file for the Quadrics network (elanidmap)", - false, false, "/etc/elanidmap", &mca_btl_elan_component.elanidmap_file ); - mca_base_param_reg_int( (mca_base_component_t*)&mca_btl_elan_component, "max_posted_recv", - "Number of received posted in advance. Increasing this number for" - " communication bound application can lead to visible improvement" - " in performances", - false, false, 128, &mca_btl_elan_component.elan_max_posted_recv ); - - /* register Elan4 component parameters */ - mca_btl_elan_component.elan_free_list_num = - mca_btl_elan_param_register_int( "free_list_num", 8 ); - mca_btl_elan_component.elan_free_list_max = - mca_btl_elan_param_register_int( "free_list_max", - (mca_btl_elan_component.elan_free_list_num + - mca_btl_elan_component.elan_max_posted_recv) ); - mca_btl_elan_component.elan_free_list_inc = - mca_btl_elan_param_register_int( "free_list_inc", 32 ); - - return OMPI_SUCCESS; -} - -/* - * component cleanup - sanity checking of queue lengths - */ - -int mca_btl_elan_component_close(void) -{ - if( NULL != mca_btl_elan_component.elan_btls ) { - free( mca_btl_elan_component.elan_btls ); - mca_btl_elan_component.elan_btls = NULL; - mca_btl_elan_component.elan_num_btls = 0; - - /* release resources */ - OBJ_DESTRUCT(&mca_btl_elan_component.elan_procs); - OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_eager); - OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_user); - OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_max); - OBJ_DESTRUCT(&mca_btl_elan_component.elan_lock); - } - return OMPI_SUCCESS; -} - -/* - * Elan4 component initialization: - * (1) read interface list from kernel and compare against component parameters - * then create a BTL instance for selected interfaces - * (2) setup Elan4 listen socket for incoming connection attempts - * (3) register BTL parameters with the MCA - */ -mca_btl_base_module_t** -mca_btl_elan_component_init( int *num_btl_modules, - bool enable_progress_threads, - bool enable_mpi_threads ) -{ - - mca_btl_base_module_t** btls = NULL; - - *num_btl_modules = 0; - - /* There is no support for a progress thread yet. */ - if (enable_progress_threads) { - ompi_modex_send(&mca_btl_elan_component.super.btl_version, NULL, 0); - return NULL; - } - - OBJ_CONSTRUCT (&mca_btl_elan_component.elan_lock, opal_mutex_t); - OBJ_CONSTRUCT (&mca_btl_elan_component.elan_frag_eager, ompi_free_list_t); - OBJ_CONSTRUCT (&mca_btl_elan_component.elan_frag_max, ompi_free_list_t); - OBJ_CONSTRUCT (&mca_btl_elan_component.elan_frag_user, ompi_free_list_t); - OBJ_CONSTRUCT(&mca_btl_elan_component.elan_procs, opal_list_t); - ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_eager, - sizeof(mca_btl_elan_frag_t) + mca_btl_elan_module.super.btl_eager_limit, - opal_cache_line_size, - OBJ_CLASS(mca_btl_elan_frag_t), - 0,opal_cache_line_size, - mca_btl_elan_component.elan_free_list_num, - mca_btl_elan_component.elan_free_list_max, - mca_btl_elan_component.elan_free_list_inc, - NULL ); /* use default allocator */ - - ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_user, - sizeof(mca_btl_elan_frag_t), - opal_cache_line_size, - OBJ_CLASS(mca_btl_elan_frag_t), - 0,opal_cache_line_size, - mca_btl_elan_component.elan_free_list_num, - mca_btl_elan_component.elan_free_list_max, - mca_btl_elan_component.elan_free_list_inc, - NULL ); /* use default allocator */ - - ompi_free_list_init_new( &mca_btl_elan_component.elan_frag_max, - sizeof(mca_btl_elan_frag_t)+mca_btl_elan_module.super.btl_max_send_size, - opal_cache_line_size, - OBJ_CLASS(mca_btl_elan_frag_t), - 0,opal_cache_line_size, - mca_btl_elan_component.elan_free_list_num, - mca_btl_elan_component.elan_free_list_max, - mca_btl_elan_component.elan_free_list_inc, - NULL ); /* use default allocator */ - - mca_btl_elan_component.elan_num_btls = ELAN_MAX_BTL; /* no more than that */ - mca_btl_elan_component.elan_btls = calloc( mca_btl_elan_component.elan_num_btls, - sizeof(mca_btl_base_module_t*) ); - /* Retrieve the positions of the node in the elan network */ - { - FILE* position; - char filename[255], file_line[255]; - int index, count = 0, positions[ELAN_MAX_BTL]; - mca_btl_elan_module_t* btl; - - for( index = 0; index < ELAN_MAX_BTL; index++ ) { - snprintf( filename, 255, "/proc/qsnet/elan4/device%d/position", index ); - position = fopen( filename, "r" ); - if( NULL == position ) { - continue; - } - if( 0 == fscanf( position, "%s%i", file_line, &positions[count] ) ) { - opal_output( 0, "Unable to read the network position" ); - continue; - } - fclose(position); - btl = (mca_btl_elan_module_t*)malloc (sizeof (mca_btl_elan_module_t)); - if(NULL == btl) { - opal_output( 0, "No enough memory to allocate the Elan internal structures" ); - return NULL; - } - memcpy( btl, &mca_btl_elan_module, sizeof(mca_btl_elan_module_t) ); - OBJ_CONSTRUCT( &btl->elan_lock, opal_mutex_t ); - OBJ_CONSTRUCT( &btl->send_list, opal_list_t ); - OBJ_CONSTRUCT( &btl->rdma_list, opal_list_t ); - OBJ_CONSTRUCT( &btl->recv_list, opal_list_t ); - - btl->expect_tport_recv = 1; - btl->elan_position = positions[count]; - - mca_btl_elan_component.elan_btls[count++] = btl; - } - mca_btl_elan_component.elan_num_btls = count; - /* Publish the network positions for the current node */ - ompi_modex_send( &mca_btl_elan_component.super.btl_version, positions, - count * sizeof(int)); - } - - if(mca_btl_elan_component.elan_num_btls) { - btls = (mca_btl_base_module_t**)malloc( mca_btl_elan_component.elan_num_btls * - sizeof(mca_btl_base_module_t*) ); - if( NULL == btls ) { - free( mca_btl_elan_component.elan_btls ); - mca_btl_elan_component.elan_num_btls = 0; /* no active BTL modules */ - return NULL; - } - memcpy( btls, mca_btl_elan_component.elan_btls, - mca_btl_elan_component.elan_num_btls * sizeof(mca_btl_elan_module_t*) ); - } - *num_btl_modules = mca_btl_elan_component.elan_num_btls; - return btls; -} - -/* - * Elan4 component progress. - */ -int mca_btl_elan_component_progress( void ) -{ - int num_progressed = 0, i; - - for( i = 0; i < (int)mca_btl_elan_component.elan_num_btls; i++ ) { - mca_btl_elan_module_t* elan_btl = mca_btl_elan_component.elan_btls[i]; - - /* This is a fast receive over the queue */ - if( elan_queueRxPoll( elan_btl->rx_queue, 0 ) ) { - mca_btl_active_message_callback_t* reg; - mca_btl_elan_hdr_t* elan_hdr = NULL; - mca_btl_elan_frag_t frag; - - elan_hdr = (mca_btl_elan_hdr_t*)elan_queueRxWait( elan_btl->rx_queue, NULL, 0 ); - frag.base.des_dst = &frag.segment; - frag.base.des_dst->seg_addr.pval = (void*)(elan_hdr+1); - frag.base.des_dst->seg_len = (size_t)elan_hdr->length; - frag.base.des_dst_cnt = 1; - frag.tag = (mca_btl_base_tag_t)elan_hdr->tag; - frag.size = elan_hdr->length; - - reg = mca_btl_base_active_message_trigger + frag.tag; - reg->cbfunc( &(elan_btl->super), frag.tag, &(frag.base), reg->cbdata ); - elan_queueRxComplete( elan_btl->rx_queue ); - num_progressed++; - } - /* This is the slower receive over the tport */ - if(elan_btl->expect_tport_recv && !OPAL_THREAD_TRYLOCK(&elan_btl->elan_lock)) { - mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)opal_list_get_first( &(elan_btl->recv_list) ); - if( elan_done(frag->elan_event, 0) ) { - int tag; - size_t length; - mca_btl_active_message_callback_t* reg; - void* recv_buf; - recv_buf = (mca_btl_elan_hdr_t*)elan_tportRxWait( frag->elan_event, - NULL, &tag, &length ); - num_progressed++; - /*elan_btl->expect_tport_recv--;*/ - - opal_list_remove_first( &(elan_btl->recv_list) ); - OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); - - frag->base.des_dst->seg_addr.pval = (void*)recv_buf; - frag->base.des_dst->seg_len = length; - frag->tag = (mca_btl_base_tag_t)tag; - reg = mca_btl_base_active_message_trigger + frag->tag; - reg->cbfunc( &(elan_btl->super), frag->tag, &(frag->base), reg->cbdata ); - if( recv_buf != (void*)(frag+1) ) { - elan_tportBufFree( elan_btl->tport, recv_buf ); - frag->base.des_dst->seg_addr.pval = (void*)(frag+1); - } - - frag->elan_event = elan_tportRxStart( elan_btl->tport, - ELAN_TPORT_RXBUF | ELAN_TPORT_RXANY, - 0, 0, 0, 0, - frag->base.des_dst->seg_addr.pval, - mca_btl_elan_module.super.btl_eager_limit ); - OPAL_THREAD_LOCK(&elan_btl->elan_lock); - opal_list_append( &(elan_btl->recv_list), (opal_list_item_t*)frag ); - } - OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); - } - /* If there are any pending sends check their completion */ - recheck_send_list: - if( !opal_list_is_empty( &(elan_btl->send_list) ) && !OPAL_THREAD_TRYLOCK(&elan_btl->elan_lock) ) { - mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)opal_list_get_first( &(elan_btl->send_list) ); - if( (NULL != frag) && elan_poll(frag->elan_event, 0) ) { - int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP ); - - opal_list_remove_first( &(elan_btl->send_list) ); - OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); - num_progressed++; - - frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint, - &(frag->base), OMPI_SUCCESS ); - if( btl_ownership ) { - MCA_BTL_ELAN_FRAG_RETURN(frag); - } - goto recheck_send_list; - } else { - OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); - } - } - recheck_rdma_list: - /* If any RDMA have been posted, check their status */ - if( !opal_list_is_empty( &(elan_btl->rdma_list) ) && !OPAL_THREAD_TRYLOCK(&elan_btl->elan_lock) ) { - mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*)opal_list_get_first( &(elan_btl->rdma_list) ); - if( (NULL != frag) && elan_poll(frag->elan_event, 0) ) { - int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP ); - - opal_list_remove_first( &(elan_btl->rdma_list) ); - OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); - num_progressed++; - - frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint, - &(frag->base), OMPI_SUCCESS ); - if( btl_ownership ) { - MCA_BTL_ELAN_FRAG_RETURN(frag); - } - goto recheck_rdma_list; - } else { - OPAL_THREAD_UNLOCK(&elan_btl->elan_lock); - } - } - } - - return num_progressed; -} diff --git a/ompi/mca/btl/elan/btl_elan_endpoint.c b/ompi/mca/btl/elan/btl_elan_endpoint.c deleted file mode 100644 index bcea940be8..0000000000 --- a/ompi/mca/btl/elan/btl_elan_endpoint.c +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include -#include -#include "ompi/types.h" -#include "btl_elan.h" -#include "btl_elan_endpoint.h" -#include "btl_elan_proc.h" -#include "btl_elan_frag.h" - - -/* - * Initialize state of the endpoint instance. - * - */ - -static void mca_btl_elan_endpoint_construct(mca_btl_base_endpoint_t* endpoint) -{ - endpoint->endpoint_btl = NULL; - endpoint->endpoint_proc = NULL; -} - -/* - * Destroy a endpoint - * - */ - -static void mca_btl_elan_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) -{ -} - - -OBJ_CLASS_INSTANCE(mca_btl_elan_endpoint_t, - opal_list_item_t, - mca_btl_elan_endpoint_construct, - mca_btl_elan_endpoint_destruct); - diff --git a/ompi/mca/btl/elan/btl_elan_endpoint.h b/ompi/mca/btl/elan/btl_elan_endpoint.h deleted file mode 100644 index 4619e2c1a7..0000000000 --- a/ompi/mca/btl/elan/btl_elan_endpoint.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2004-2007 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_ELAN_ENDPOINT_H -#define MCA_BTL_ELAN_ENDPOINT_H - -#include "opal/class/opal_list.h" -#include "opal/mca/event/event.h" -#include "btl_elan_frag.h" -#include "btl_elan.h" - -BEGIN_C_DECLS - -/** - * State of ELAN endpoint connection. - */ - -/** - * An abstraction that represents a connection to a endpoint process. - * An instance of mca_btl_base_endpoint_t is associated w/ each process - * and BTL pair at startup. However, connections to the endpoint - * are established dynamically on an as-needed basis: - */ - -struct mca_btl_base_endpoint_t { - opal_list_item_t super; - - struct mca_btl_elan_module_t* endpoint_btl; - /**< BTL instance that created this connection */ - - struct mca_btl_elan_proc_t* endpoint_proc; - /**< proc structure corresponding to endpoint */ - - unsigned int elan_vp; -}; - -typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t; -typedef mca_btl_base_endpoint_t mca_btl_elan_endpoint_t; -OBJ_CLASS_DECLARATION(mca_btl_elan_endpoint_t); - -END_C_DECLS - -#endif /* MCA_BTL_ELAN_ENDPOINT_H */ diff --git a/ompi/mca/btl/elan/btl_elan_frag.c b/ompi/mca/btl/elan/btl_elan_frag.c deleted file mode 100644 index 745069d15a..0000000000 --- a/ompi/mca/btl/elan/btl_elan_frag.c +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "btl_elan.h" - -static void mca_btl_elan_frag_common_constructor(mca_btl_elan_frag_t* frag) -{ - frag->base.des_src = NULL; - frag->base.des_src_cnt = 0; - frag->base.des_dst = NULL; - frag->base.des_dst_cnt = 0; - frag->elan_event = NULL; - frag->base.des_dst = &(frag->segment); - frag->base.des_dst_cnt = 1; -} - -static void mca_btl_elan_frag_eager_constructor(mca_btl_elan_frag_t* frag) -{ - frag->size = mca_btl_elan_module.super.btl_eager_limit; - frag->my_list = &mca_btl_elan_component.elan_frag_eager; - mca_btl_elan_frag_common_constructor(frag); -} - -static void mca_btl_elan_frag_max_constructor(mca_btl_elan_frag_t* frag) -{ - frag->size = mca_btl_elan_module.super.btl_max_send_size; - frag->my_list = &mca_btl_elan_component.elan_frag_max; - mca_btl_elan_frag_common_constructor(frag); -} - -static void mca_btl_elan_frag_user_constructor(mca_btl_elan_frag_t* frag) -{ - frag->size = 0; - frag->my_list = &mca_btl_elan_component.elan_frag_user; - mca_btl_elan_frag_common_constructor(frag); -} - - -OBJ_CLASS_INSTANCE( mca_btl_elan_frag_t, - mca_btl_base_descriptor_t, - NULL, - NULL ); - -OBJ_CLASS_INSTANCE( mca_btl_elan_frag_eager_t, - mca_btl_base_descriptor_t, - mca_btl_elan_frag_eager_constructor, - NULL ); - -OBJ_CLASS_INSTANCE( mca_btl_elan_frag_max_t, - mca_btl_base_descriptor_t, - mca_btl_elan_frag_max_constructor, - NULL ); - -OBJ_CLASS_INSTANCE( mca_btl_elan_frag_user_t, - mca_btl_base_descriptor_t, - mca_btl_elan_frag_user_constructor, - NULL ); diff --git a/ompi/mca/btl/elan/btl_elan_frag.h b/ompi/mca/btl/elan/btl_elan_frag.h deleted file mode 100644 index 61f6f7149c..0000000000 --- a/ompi/mca/btl/elan/btl_elan_frag.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_ELAN_FRAG_H -#define MCA_BTL_ELAN_FRAG_H - -#include "ompi_config.h" -#include "btl_elan.h" - -BEGIN_C_DECLS - -#define MCA_BTL_ELAN_HDR_TYPE_SEND 1 -#define MCA_BTL_ELAN_HDR_TYPE_PUT 2 -#define MCA_BTL_ELAN_HDR_TYPE_GET 3 -#define MCA_BTL_ELAN_HDR_TYPE_RECV 4 - - -/** - * Elan send fragment derived type. - */ -struct mca_btl_elan_frag_t { - mca_btl_base_descriptor_t base; - mca_btl_base_segment_t segment; - struct mca_btl_base_endpoint_t *endpoint; - struct mca_btl_elan_module_t* btl; - int type; - ompi_free_list_t* my_list; - mca_btl_base_tag_t tag; - struct ELAN_EVENT* elan_event; - size_t size; -}; -typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_t; -OBJ_CLASS_DECLARATION(mca_btl_elan_frag_t); - -typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_eager_t; - -OBJ_CLASS_DECLARATION(mca_btl_elan_frag_eager_t); - -typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_max_t; - -OBJ_CLASS_DECLARATION(mca_btl_elan_frag_max_t); - -typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_user_t; - -OBJ_CLASS_DECLARATION(mca_btl_elan_frag_user_t); - - -/* - * Macros to allocate/return descriptors from module specific - * free list(s). - */ -#define MCA_BTL_ELAN_FRAG_ALLOC_LIST( list, frag, rc ) \ -{ \ - ompi_free_list_item_t *item; \ - OMPI_FREE_LIST_GET(&(list), item, rc); \ - frag = (mca_btl_elan_frag_t*) item; \ -} - -#define MCA_BTL_ELAN_FRAG_ALLOC_EAGER(frag, rc) \ - MCA_BTL_ELAN_FRAG_ALLOC_LIST(mca_btl_elan_component.elan_frag_eager, frag, rc) - -#define MCA_BTL_ELAN_FRAG_ALLOC_MAX(frag, rc) \ - MCA_BTL_ELAN_FRAG_ALLOC_LIST(mca_btl_elan_component.elan_frag_max, frag, rc) - -#define MCA_BTL_ELAN_FRAG_ALLOC_USER(frag, rc) \ - MCA_BTL_ELAN_FRAG_ALLOC_LIST(mca_btl_elan_component.elan_frag_user, frag, rc) - -#define MCA_BTL_ELAN_FRAG_RETURN(frag) \ - { \ - OMPI_FREE_LIST_RETURN(frag->my_list, \ - (ompi_free_list_item_t*)(frag)); \ - } - - -END_C_DECLS - -#endif /* MCA_BTL_ELAN_FRAG_H */ diff --git a/ompi/mca/btl/elan/btl_elan_proc.c b/ompi/mca/btl/elan/btl_elan_proc.c deleted file mode 100644 index 4f065e31a1..0000000000 --- a/ompi/mca/btl/elan/btl_elan_proc.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2004-2011 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "ompi/runtime/ompi_module_exchange.h" -#include "btl_elan.h" -#include "btl_elan_proc.h" - -static void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc); -static void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc); - -OBJ_CLASS_INSTANCE(mca_btl_elan_proc_t, - opal_list_item_t,mca_btl_elan_proc_construct, - mca_btl_elan_proc_destruct); - -void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* elan_proc) -{ - elan_proc->proc_ompi = 0; - elan_proc->proc_rail_count = 0; - elan_proc->proc_endpoints = 0; - elan_proc->proc_endpoint_count = 0; - OBJ_CONSTRUCT(&elan_proc->proc_lock, opal_mutex_t); - /* add to list of all proc instance */ - OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock); - opal_list_append(&mca_btl_elan_component.elan_procs, &elan_proc->super); - OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock); -} - -/* - * Cleanup elan proc instance - */ - -void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* elan_proc) -{ - /* remove from list of all proc instances */ - OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock); - opal_list_remove_item(&mca_btl_elan_component.elan_procs, &elan_proc->super); - OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock); - /* release resources */ - if(NULL != elan_proc->proc_endpoints) { - free(elan_proc->proc_endpoints); - } - OBJ_DESTRUCT(&elan_proc->proc_lock); -} - -/* - * Look for an existing Elan process instances based on the associated - * ompi_proc_t instance. - */ -static mca_btl_elan_proc_t* mca_btl_elan_proc_lookup_ompi(ompi_proc_t* ompi_proc) -{ - mca_btl_elan_proc_t* elan_proc; - OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock); - for(elan_proc = (mca_btl_elan_proc_t*) - opal_list_get_first(&mca_btl_elan_component.elan_procs); - elan_proc != (mca_btl_elan_proc_t*) - opal_list_get_end(&mca_btl_elan_component.elan_procs); - elan_proc = (mca_btl_elan_proc_t*)opal_list_get_next(elan_proc)) { - if(elan_proc->proc_ompi == ompi_proc) { - OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock); - return elan_proc; - } - } - OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock); - return NULL; -} - -/* - * Create a ELAN process structure. There is a one-to-one correspondence - * between a ompi_proc_t and a mca_btl_elan_proc_t instance. We cache - * additional data (specifically the list of mca_btl_elan_endpoint_t instances, - * and published addresses) associated w/ a given destination on this - * datastructure. - */ - -mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc) -{ - int rc; - size_t size; - mca_btl_elan_proc_t* module_proc = NULL; - /* Check if we have already created a Elan proc - * structure for this ompi process */ - module_proc = mca_btl_elan_proc_lookup_ompi(ompi_proc); - if(module_proc != NULL) { - /* Gotcha! */ - return module_proc; - } - /* Oops! First time, gotta create a new Elan proc - * out of the ompi_proc ... */ - module_proc = OBJ_NEW(mca_btl_elan_proc_t); - if(NULL == module_proc) - return NULL; - /* Initialize number of peer */ - module_proc->proc_endpoint_count = 0; - module_proc->proc_ompi = ompi_proc; - - /* build a unique identifier (of arbitrary - * size) to represent the proc */ - rc = ompi_modex_recv( &mca_btl_elan_component.super.btl_version, - ompi_proc, - (void**)&module_proc->position_id_array, - &size ); - if(rc != OMPI_SUCCESS) { - BTL_ERROR(("mca_base_modex_recv: failed with return value=%d", rc)); - OBJ_RELEASE(module_proc); - return NULL; - } - module_proc->proc_rail_count = size / sizeof(unsigned int);; - /* XXX: Right now, there can be only 1 peer associated - * with a proc. Needs a little bit change in - * mca_btl_elan_proc_t to allow on demand increasing of - * number of endpoints for this proc - */ - - module_proc->proc_endpoints = (mca_btl_base_endpoint_t**) - malloc((1+module_proc->proc_rail_count )* sizeof(mca_btl_base_endpoint_t*)); - if(NULL == module_proc->proc_endpoints) { - OBJ_RELEASE(module_proc); - return NULL; - } - return module_proc; -} - - -/* - * Note that this routine must be called with the lock on the process - * already held. Insert a btl instance into the proc array and assign - * it an address. - */ -int mca_btl_elan_proc_insert( mca_btl_elan_proc_t* module_proc, - mca_btl_base_endpoint_t* module_endpoint ) -{ - /* insert into endpoint array */ - module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint; - - module_endpoint->endpoint_proc = module_proc; - module_endpoint->elan_vp = module_proc->proc_ompi->proc_name.vpid; - - return OMPI_SUCCESS; -} - diff --git a/ompi/mca/btl/elan/btl_elan_proc.h b/ompi/mca/btl/elan/btl_elan_proc.h deleted file mode 100644 index 1a698dc887..0000000000 --- a/ompi/mca/btl/elan/btl_elan_proc.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2004-2011 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_ELAN_PROC_H -#define MCA_BTL_ELAN_PROC_H - -#include "opal/class/opal_object.h" -#include "ompi/proc/proc.h" -#include "btl_elan.h" -#include "btl_elan_endpoint.h" - -BEGIN_C_DECLS - -/** - * Represents the state of a remote process and the set of addresses - * that it exports. Also cache an instance of mca_btl_base_endpoint_t for - * each - * BTL instance that attempts to open a connection to the process. - */ -struct mca_btl_elan_proc_t { - opal_list_item_t super; - /**< allow proc to be placed on a list */ - - ompi_proc_t *proc_ompi; - /**< pointer to corresponding ompi_proc_t */ - - unsigned int *position_id_array; - - size_t proc_rail_count; - /**< number of addresses published by endpoint */ - - struct mca_btl_base_endpoint_t **proc_endpoints; - /**< array of endpoints that have been created to access this proc */ - - size_t proc_endpoint_count; - /**< number of endpoints */ - - opal_mutex_t proc_lock; - /**< lock to protect against concurrent access to proc state */ -}; -typedef struct mca_btl_elan_proc_t mca_btl_elan_proc_t; -OBJ_CLASS_DECLARATION(mca_btl_elan_proc_t); - -mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc); -int mca_btl_elan_proc_insert(mca_btl_elan_proc_t*, mca_btl_base_endpoint_t*); - -END_C_DECLS - -#endif /* MCA_BTL_ELAN_PROC_H */ diff --git a/ompi/mca/btl/elan/configure.m4 b/ompi/mca/btl/elan/configure.m4 deleted file mode 100644 index 024c0ce3c7..0000000000 --- a/ompi/mca/btl/elan/configure.m4 +++ /dev/null @@ -1,76 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2007 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# OMPI_CHECK_ELAN(prefix, [action-if-found], [action-if-not-found]) -# -------------------------------------------------------- -# check if Elan support can be found. sets prefix_{CPPFLAGS, -# LDFLAGS, LIBS} as needed and runs action-if-found if there is -# support, otherwise executes action-if-not-found -AC_DEFUN([OMPI_CHECK_ELAN],[ - AC_ARG_WITH([elan], - [AC_HELP_STRING([--with-elan(=DIR)], - [Build Elan (QsNet2) support, searching for libraries in DIR])]) - OMPI_CHECK_WITHDIR([elan], [$with_elan], [include/elan/elan.h]) - AC_ARG_WITH([elan-libdir], - [AC_HELP_STRING([--with-elan-libdir=DIR], - [Search for Elan (QsNet2) libraries in DIR])]) - OMPI_CHECK_WITHDIR([elan-libdir], [$with_elan_libdir], [libelan.*]) - - AS_IF([test "$with_elan" != "no"], - [AS_IF([test ! -z "$with_elan" -a "$with_elan" != "yes"], - [ompi_check_elan_dir="$with_elan"]) - AS_IF([test ! -z "$with_elan_libdir" -a "$with_elan_libdir" != "yes"], - [ompi_check_elan_libdir="$with_elan_libdir"]) - - OMPI_CHECK_PACKAGE([$1], - [elan/elan.h], - [elan], - [elan_init], - [], - [$ompi_check_elan_dir], - [$ompi_check_elan_libdir], - [ompi_check_elan_happy="yes"], - [ompi_check_elan_happy="no"]) - ], - [ompi_check_elan_happy="no"]) - - AS_IF([test "$ompi_check_elan_happy" = "yes"], - [$2], - [AS_IF([test ! -z "$with_elan" -a "$with_elan" != "no"], - [AC_MSG_ERROR([Elan (QsNet2) support requested but not found. Aborting])]) - $3]) - ]) - -# MCA_btl_elan_CONFIG([action-if-can-compile], -# [action-if-cant-compile]) -# ------------------------------------------------ -AC_DEFUN([MCA_ompi_btl_elan_CONFIG],[ - AC_CONFIG_FILES([ompi/mca/btl/elan/Makefile]) - - OMPI_CHECK_ELAN([btl_elan], - [btl_elan_happy="yes"], - [btl_elan_happy="no"]) - - AS_IF([test "$btl_elan_happy" = "yes"], - [btl_elan_WRAPPER_EXTRA_LDFLAGS="$btl_elan_LDFLAGS" - btl_elan_WRAPPER_EXTRA_LIBS="$btl_elan_LIBS" - $1], - [$2]) - - # substitute in the things needed to build elan - AC_SUBST([btl_elan_CFLAGS]) - AC_SUBST([btl_elan_CPPFLAGS]) - AC_SUBST([btl_elan_LDFLAGS]) - AC_SUBST([btl_elan_LIBS]) -])dnl -