diff --git a/ompi/mca/btl/sicortex/.ompi_ignore b/ompi/mca/btl/sicortex/.ompi_ignore deleted file mode 100755 index e69de29bb2..0000000000 diff --git a/ompi/mca/btl/sicortex/.ompi_unignore b/ompi/mca/btl/sicortex/.ompi_unignore deleted file mode 100755 index 0ba70841a4..0000000000 --- a/ompi/mca/btl/sicortex/.ompi_unignore +++ /dev/null @@ -1,2 +0,0 @@ -tma -bosilca diff --git a/ompi/mca/btl/sicortex/Makefile.am b/ompi/mca/btl/sicortex/Makefile.am deleted file mode 100755 index 5c452e818d..0000000000 --- a/ompi/mca/btl/sicortex/Makefile.am +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2008-2009 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -AM_CPPFLAGS = $(btl_sicortex_CPPFLAGS) - -btl_sicortex_sources = btl_sicortex.c btl_sicortex.h btl_sicortex_component.c btl_sicortex_endpoint.c \ - btl_sicortex_endpoint.h btl_sicortex_frag.c btl_sicortex_frag.h btl_sicortex_proc.c btl_sicortex_proc.h - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if OMPI_BUILD_btl_sicortex_DSO -component_noinst = -component_install = mca_btl_sicortex.la -else -component_noinst = libmca_btl_sicortex.la -component_install = -endif - -mcacomponentdir = $(libdir)/openmpi -mcacomponent_LTLIBRARIES = $(component_install) -mca_btl_sicortex_la_SOURCES = $(btl_sicortex_sources) -mca_btl_sicortex_la_CFLAGS = -DBCAST -mca_btl_sicortex_la_LIBADD = -lscmpi_debug $(btl_sicortex_LIBS) -mca_btl_sicortex_la_LDFLAGS = -module -avoid-version $(btl_sicortex_LDFLAGS) - -noinst_LTLIBRARIES = $(component_noinst) -libmca_btl_sicortex_la_SOURCES = $(btl_sicortex_sources) -libmca_btl_sicortex_la_CFLAGS = -DBCAST -libmca_btl_sicortex_la_LIBADD = -lscmpi_debug $(btl_sicortex_LIBS) -libmca_btl_sicortex_la_LDFLAGS = -module -avoid-version $(btl_sicortex_LDFLAGS) diff --git a/ompi/mca/btl/sicortex/btl_sicortex.c b/ompi/mca/btl/sicortex/btl_sicortex.c deleted file mode 100755 index 9783ecf6fd..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex.c +++ /dev/null @@ -1,843 +0,0 @@ -/* - * Copyright (c) 2004-2009 The University of Tennessee and The Univer - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include -#include "orte/util/show_help.h" -#include "opal/util/if.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/btl/btl.h" - -#include "btl_sicortex.h" -#include "btl_sicortex_frag.h" -#include "btl_sicortex_proc.h" -#include "btl_sicortex_endpoint.h" -#include "ompi/datatype/convertor.h" -#include "ompi/mca/mpool/base/base.h" -#include "ompi/mca/mpool/mpool.h" - -#include -#include -#include -#include - - -static int mca_btl_sicortex_add_procs( struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t **ompi_procs, - struct mca_btl_base_endpoint_t** peers, - opal_bitmap_t* reachable ) -{ - mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*)btl; - int i,rc,index = 0; - - sicortex_btl->peers = (mca_btl_base_endpoint_t**)malloc(sizeof(mca_btl_base_endpoint_t*)*(nprocs-1)); - for(i = 0; i < (int) nprocs; i++) { - - struct ompi_proc_t* ompi_proc = ompi_procs[i]; - mca_btl_sicortex_proc_t* sicortex_proc; - mca_btl_base_endpoint_t* sicortex_endpoint; - - if(ompi_proc_local_proc == ompi_proc) - continue; - - if(NULL == (sicortex_proc = mca_btl_sicortex_proc_create(ompi_proc))) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - - /* - * Check to make sure that the peer has at least as many interface - * addresses exported as we are trying to use. If not, then - * don't bind this PTL instance to the proc. - */ - - OPAL_THREAD_LOCK(&sicortex_proc->proc_lock); - - /* The btl_proc datastructure is shared by all SICORTEX PTL - * instances that are trying to reach this destination. - * Cache the peer instance on the btl_proc. - */ - sicortex_endpoint = OBJ_NEW(mca_btl_sicortex_endpoint_t); - if(NULL == sicortex_endpoint) { - OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock); - return OMPI_ERR_OUT_OF_RESOURCE; - } - sicortex_endpoint->route = (peer_struct_t*)malloc(sizeof(peer_struct_t)); - sicortex_endpoint->endpoint_btl = sicortex_btl; - rc = mca_btl_sicortex_proc_insert(sicortex_proc, sicortex_endpoint); - if(rc != OMPI_SUCCESS) { - OBJ_RELEASE(sicortex_endpoint); - OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock); - continue; - } - rc = scdma_set_route(sicortex_btl->ctx, - sicortex_endpoint->route->route_handles[0], - sicortex_endpoint->route->route_handles[1], - sicortex_endpoint->route->route_handles[2], - sicortex_endpoint->route->context_id, - &(sicortex_endpoint->route->ports[0]), - &(sicortex_endpoint->route->ports[1]), - &(sicortex_endpoint->route->ports[2])); - if(rc != 0) { - OBJ_RELEASE(sicortex_endpoint); - OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock); - continue; - } - opal_bitmap_set_bit(reachable, i); - OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock); - sicortex_btl->peers[index++] = peers[i] = sicortex_endpoint; - } - - sicortex_btl->bds_size = sicortex_btl->ctx->bdt_size; - sicortex_btl->bds_limit = sicortex_btl->ctx->num_bdt_entries; - sicortex_btl->bds_head = 0; - - return OMPI_SUCCESS; -} - -static int -mca_btl_sicortex_del_procs(struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t **procs, - struct mca_btl_base_endpoint_t ** peers) -{ - /* TODO */ - return OMPI_SUCCESS; -} - -/** - * Allocate a segment. - * - * @param btl (IN) BTL module - * @param size (IN) Request segment size. - */ - -static mca_btl_base_descriptor_t* -mca_btl_sicortex_alloc( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - uint8_t order, - size_t size, - uint32_t flags ) -{ - mca_btl_sicortex_frag_t* frag = NULL; - int rc; - - if(size <= btl->btl_eager_limit) { - MCA_BTL_SICORTEX_FRAG_ALLOC_EAGER(sicortex_btl, frag, rc); - } else if( size <= btl->btl_max_send_size ) { - MCA_BTL_SICORTEX_FRAG_ALLOC_MAX(sicortex_btl, frag, rc); - } - if(OPAL_UNLIKELY(NULL == frag)) { - return NULL; - } - frag->segment.seg_len = size; - frag->segment.seg_addr.pval = (void*)((char*)(frag + 1)); - frag->segment.seg_len = size; - frag->base.des_src = &frag->segment; - frag->base.des_src_cnt = 1; - frag->base.des_flags = flags; - frag->base.order = MCA_BTL_NO_ORDER; - return (mca_btl_base_descriptor_t*)frag; -} - - -/** - * Return a segment - */ - -static int -mca_btl_sicortex_free( struct mca_btl_base_module_t* btl, - mca_btl_base_descriptor_t* des ) -{ - mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*)des; - MCA_BTL_SICORTEX_FRAG_RETURN(btl, frag); - return OMPI_SUCCESS; -} - -/** - * Initiate an immediate blocking send. - * Completion Semantics: the BTL will make a best effort - * to send the header and "size" bytes from the datatype using the convertor. - * The header is guaranteed to be delivered entirely in the first segment. - * Should the BTL be unable to deliver the data due to resource constraints - * the BTL will return a descriptor (via the OUT param) - * of size "payload_size + header_size". - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param convertor (IN) Data type convertor - * @param header (IN) Pointer to header. - * @param header_size (IN) Size of header. * @param payload_size (IN) Size of payload (from convertor). - * @param order (IN) The ordering tag (may be MCA_BTL_NO_ORDER) - * @param flags (IN) Flags. - * @param tag (IN) The tag value used to notify the peer. - * @param descriptor (OUT) The descriptor to be returned unable to be sent immediately - */ -static int mca_btl_sicortex_sendi( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct ompi_convertor_t* convertor, - void* header, - size_t header_size, - size_t payload_size, - uint8_t order, - uint32_t flags, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t** descriptor ) -{ - mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl; - size_t max_data, length = header_size + payload_size; - struct iovec iov; - uint32_t iov_count = 1, length_on_wire; - scdma_hw_cmd_t* cmd; - char* payload; - - if( length > SCDMA_HW_CMD_PAYLOAD_MAX_SIZE ) { - *descriptor = mca_btl_sicortex_alloc( btl, endpoint, order, length, flags ); - return OMPI_ERR_RESOURCE_BUSY; - } - - /* Adjust the length on the wire to allow the DMA engine to - * know exactly how many cache lines have to be transfered. - */ - length_on_wire = scdma_send_event_len(length + 2 * sizeof(uint64_t)); - cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx); - cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT, - length_on_wire, - endpoint->route->route_handles[0/*sicortex_btl->handle_count*/], - endpoint->route->ports[0/*sicortex_btl->handle_count*/]); - /*sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM;*/ - cmd->payload[0] = (((uint64_t)tag << 16) | (EVENT_SEND_IM << 8) - | (length + 2 * sizeof(uint64_t)) ); - payload = (char*)&cmd->payload[1]; - memcpy( payload, header, header_size ); - - iov.iov_base = payload + header_size; - iov.iov_len = payload_size; - max_data = payload_size; - - if( 0 != payload_size ) { - ompi_convertor_pack( convertor, &iov, &iov_count, &max_data ); - assert( max_data == payload_size ); - } - - if( length > (SCDMA_HW_CMD_PAYLOAD_MAX_SIZE-64) ) - cmd->payload[15] = 0; - scdma_cq_post_fastpath(sicortex_btl->ctx,cmd->header); - - return OMPI_SUCCESS; -} - -/** - * Initiate an asynchronous send. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transfered - * @param tag (IN) The tag value used to notify the peer. - */ - -static int -mca_btl_sicortex_send( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_btl_base_descriptor_t* descriptor, - mca_btl_base_tag_t tag ) -{ - mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl; - mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*)descriptor; - int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP ); - size_t length = frag->segment.seg_len; - int offset, len, length_on_wire; - scdma_hw_cmd_t *cmd; - scdma_hw_put_im_heap_cmd_t *pb; - - length_on_wire = length + 2 * sizeof(uint64_t); - frag->endpoint = endpoint; - - if( SCDMA_HW_CMD_PAYLOAD_MAX_SIZE < length_on_wire ) { - offset = 0; - do { - if( length_on_wire > 16*8 ) - len = 8*8;/* SCDMA_HW_CMD_PAYLOAD_MAX_SIZE;*/ - else - len = length_on_wire - 2 * 8; - cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx); - pb = (scdma_hw_put_im_heap_cmd_t *)cmd; - pb->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_PUT_IM_HP, - (len + 0x17) & ~0x7, /* 2 * 64 bits + len */ - endpoint->route->route_handles[0/*sicortex_btl->handle_count*/], - endpoint->route->ports[0/*sicortex_btl->handle_count*/]); - pb->count_offset = (SCDMA_USABLE_HEAP_OFFSET + offset); - memcpy( scdma_cmd_payload(cmd), (char*)frag->base.des_src->seg_addr.pval + offset, len); - scdma_cq_post_fastpath(sicortex_btl->ctx, pb->header); - offset += len; - length_on_wire -= len; - } while(length_on_wire > 2*8); - cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx); - cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT, - 0x10, - endpoint->route->route_handles[0/*sicortex_btl->handle_count*/], - endpoint->route->ports[0/*sicortex_btl->handle_count*/]); - cmd->payload[0] = (((uint64_t)length << 32) | ((uint64_t)tag << 16) | - (EVENT_SEND_IM_PUT_POST << 8) | 0x10); - cmd->payload[1] = 0; - /* Next operation will take another path. */ - /*sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM;*/ - } else { - - /* Adjust the length on the wire to allow the DMA engine to - * know exactly how many cache lines have to be transfered. - */ - length_on_wire = scdma_send_event_len(length + 2 * sizeof(uint64_t)); - cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx); - cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT, - length_on_wire, - endpoint->route->route_handles[0/*sicortex_btl->handle_count*/], - endpoint->route->ports[0/*sicortex_btl->handle_count*/]); - cmd->payload[0] = (((uint64_t)tag << 16) | (EVENT_SEND_IM << 8) - | (length + 2 * sizeof(uint64_t)) ); - memcpy( &cmd->payload[1], frag->base.des_src->seg_addr.pval, length ); - - cmd->payload[15] = 0; - } - - scdma_cq_post_fastpath(sicortex_btl->ctx,cmd->header); - - if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) { - frag->base.des_cbfunc( &(sicortex_btl->super), frag->endpoint, - &(frag->base), OMPI_SUCCESS); - } - if( btl_ownership ){ - MCA_BTL_SICORTEX_FRAG_RETURN(sicortex_btl, frag); - } - return 1; -} - -#define MCA_BTL_SICORTEX_PACK_BD(v64bits, offset, overhead, bd_index, num_entries) \ - do { \ - v64bits = (((uint64_t)(offset) << 48) | ((uint64_t)(overhead) << 32) | \ - (((uint64_t)(bd_index) << 16) | (uint64_t)(num_entries))); \ - } while (0) - -#define MCA_BTL_SICORTEX_UNPACK_BD(v64bits, offset, overhead, bd_index, num_entries) \ - do { \ - (num_entries) = ((v64bits) & 0xFFFF); \ - (bd_index) = (((v64bits) >> 16) & 0xFFFF); \ - (overhead) = (((v64bits) >> 32) & 0xFFFF); \ - (offset) = (((v64bits) >> 48) & 0xFFFF); \ - } while (0) - -static inline void -mca_btl_sicortex_map_bds( mca_btl_sicortex_module_t* sicortex_btl, - void* base_addr, - uint16_t offset, - size_t* size, - uint16_t* bd_start, - uint16_t* num_entries ) -{ - OPAL_THREAD_LOCK( &sicortex_btl->sicortex_lock); - *bd_start = sicortex_btl->bds_head; - *num_entries = 1 + (offset + *size) / sicortex_btl->bds_size; - sicortex_btl->bds_head += *num_entries; - if( sicortex_btl->bds_head >= sicortex_btl->bds_limit ) { - sicortex_btl->bds_head = 0; /* rollback at the begining */ - *num_entries = sicortex_btl->bds_limit - *bd_start; - *size = (*size % sicortex_btl->bds_size) + (*num_entries - 1) * sicortex_btl->bds_size; - } - OPAL_THREAD_UNLOCK( &sicortex_btl->sicortex_lock); - /*opal_output(0, "map base_addr %p size %ld [bd_start %d: num_entries %d]", - base_addr, (unsigned long)*size, *bd_start, *num_entries );*/ -} - -/** - * Pack data and return a descriptor that can be - * used for send/put. - * - * @param btl (IN) BTL module - * @param peer (IN) BTL peer addressing - */ -static mca_btl_base_descriptor_t* -mca_btl_sicortex_prepare_src( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, - struct ompi_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags ) -{ - mca_btl_sicortex_module_t *sicortex_btl = (mca_btl_sicortex_module_t *)btl; - mca_btl_sicortex_frag_t* frag; - struct iovec iov; - uint32_t iov_count = 1; - size_t max_data = *size; - int rc; - - if( 0 != reserve ) { - if( max_data + reserve <= btl->btl_eager_limit ) { - MCA_BTL_SICORTEX_FRAG_ALLOC_EAGER(sicortex_btl,frag, rc); - } else { - MCA_BTL_SICORTEX_FRAG_ALLOC_MAX(sicortex_btl,frag, rc); - if( (max_data + reserve) > btl->btl_max_send_size ) { - max_data = btl->btl_max_send_size - reserve; - } - } - if(OPAL_UNLIKELY(NULL == frag)) { - return NULL; - } - frag->segment.seg_addr.pval = (void*)((unsigned char*)(frag + 1)); - iov.iov_len = max_data; - iov.iov_base = (unsigned char*)frag->segment.seg_addr.pval + reserve; - rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data ); - *size = max_data; - if( rc < 0 ) { - MCA_BTL_SICORTEX_FRAG_RETURN(sicortex_btl, frag); - return NULL; - } - frag->segment.seg_len = max_data + reserve; - } else { /* this is a real RDMA operation */ - uint16_t bd_index, num_entries, offset; - void* base_addr; - - MCA_BTL_SICORTEX_FRAG_ALLOC_USER(sicortex_btl,frag, rc); - if(NULL == frag) { - return NULL; - } - /** - * The best we can do is to decrease the number of bd we will have to map and - * to decrease the number of over 64K segments we have to transfer. Therefore, - * we will limit the amount of data to be sent to whatever is on the first 64K - * fragment plus a multiple of 64K. - * We know that the offset and the overhead will be 16 bits each. Similarly, - * the bd_index and num_entries are smaller than 16 bits each, so we can - * pack all required information in 64 bits. - */ - ompi_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) ); - base_addr = (void*)(frag->segment.seg_addr.lval & ~((uintptr_t)0xFFFF)); - offset = frag->segment.seg_addr.lval & (uintptr_t)0xffff; - - mca_btl_sicortex_map_bds( sicortex_btl, base_addr, offset, - size, &bd_index, &num_entries); - assert( *size != 0 ); - iov.iov_base = NULL; - iov.iov_len = *size; - ompi_convertor_pack(convertor, &iov, &iov_count, &max_data); - *size = max_data; - assert( 0 != iov_count ); - frag->segment.seg_addr.pval = iov.iov_base; - frag->segment.seg_len = max_data; - assert( frag->segment.seg_len != 0 ); - assert( frag->segment.seg_addr.pval != NULL ); - - rc = scdma_map_bds(sicortex_btl->ctx, bd_index, - base_addr, num_entries); - - frag->base.des_src = &(frag->segment); - MCA_BTL_SICORTEX_PACK_BD( frag->base.des_src->seg_key.key64, - offset, 0, bd_index, num_entries ); - /*opal_output(0, "prepare_src index %d offset %d (num_entries %d) length %d [base %p addr %p]", - bd_index, offset, num_entries, (int)max_data, base_addr, (void*)iov.iov_base );*/ - } - - frag->base.des_src = &(frag->segment); - frag->base.des_src_cnt = 1; - frag->base.order = MCA_BTL_NO_ORDER; - frag->base.des_dst = NULL; - frag->base.des_dst_cnt = 0; - frag->base.des_flags = flags; - return &frag->base; -} - -/** - * Prepare a descriptor for send/rdma using the supplied - * convertor. If the convertor references data that is contigous, - * the descriptor may simply point to the user buffer. Otherwise, - * this routine is responsible for allocating buffer space and - * packing if required. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL peer addressing - * @param convertor (IN) Data type convertor - * @param reserve (IN) Additional bytes requested by upper layer to precede user data - * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) - */ - -static mca_btl_base_descriptor_t* -mca_btl_sicortex_prepare_dst( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, - struct ompi_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags) -{ - mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl; - mca_btl_sicortex_frag_t* frag; - uint16_t bd_index, offset, num_entries; - int rc, length; - void* base_addr; - - MCA_BTL_SICORTEX_FRAG_ALLOC_USER(btl, frag, rc); - if(OPAL_UNLIKELY(NULL == frag)) { - return NULL; - } - - /* Ignore all about alignments. We will cope with these on the _put function */ - ompi_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) ); - offset = (frag->segment.seg_addr.lval & (uintptr_t)0xffff); - base_addr = (void*)(frag->segment.seg_addr.lval & ~((uintptr_t)0xffff)); - - mca_btl_sicortex_map_bds( sicortex_btl, base_addr, offset, - size, &bd_index, &num_entries ); - - frag->segment.seg_len = *size; - length = *size; - - rc = scdma_map_bds(sicortex_btl->ctx, bd_index, - base_addr, num_entries); - - frag->base.des_src = NULL; - frag->base.des_src_cnt = 0; - frag->base.des_dst = &frag->segment; - - MCA_BTL_SICORTEX_PACK_BD(frag->base.des_dst->seg_key.key64, - offset, 0, bd_index, num_entries ); - /*opal_output(0, "prepare_dst index %d offset %d (num_entries %d) length %d [base %p addr %p]", - bd_index, offset, num_entries, length, base_addr, (void*)frag->segment.seg_addr.pval);*/ - frag->base.des_dst_cnt = 1; - frag->base.des_flags = flags; - return &frag->base; -} - -#define MCA_BTL_SICORTEX_NEXT_BDS(val) \ - do { \ - if( ++(val) >= sicortex_btl->bds_limit ) \ - (val) = 0; \ - } while(0) - -/** - * Initiate an asynchronous put. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transferred - */ -static int -mca_btl_sicortex_put( mca_btl_base_module_t* btl, - mca_btl_base_endpoint_t* endpoint, - mca_btl_base_descriptor_t* des ) -{ - mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl; - mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*) des; - scdma_hw_put_bf_bf_cmd_t *pb; - uint32_t dst_offset, dst_overhead, src_offset, src_overhead; - uint32_t dst_bd_index, dst_num_entries, src_bd_index, src_num_entries; - uint32_t total_length = des->des_src->seg_len, seg_length, align; - uintptr_t dst_ptr, src_ptr; - - frag->endpoint = endpoint; - frag->count = 0; - - MCA_BTL_SICORTEX_UNPACK_BD( des->des_src->seg_key.key64, src_offset, src_overhead, - src_bd_index, src_num_entries ); - MCA_BTL_SICORTEX_UNPACK_BD( des->des_dst->seg_key.key64, dst_offset, dst_overhead, - dst_bd_index, dst_num_entries ); - src_ptr = des->des_src->seg_addr.lval; - dst_ptr = des->des_dst->seg_addr.lval; - /*opal_output(0, "src key %llx (src_ptr %llx) dst key %llx (dst_ptr %llx)", - (unsigned long long)des->des_src->seg_key.key64, (unsigned long long)src_ptr, - (unsigned long long)des->des_dst->seg_key.key64, (unsigned long long)dst_ptr );*/ - - /* Let's check for the conditions imposed by the hardware: - * - If the destination buffer does not start and end at cache block (32-byte) - * boundaries, software must use another mechanism (probably built upon - * Send Event) to deliver the data which belongs in partial blocks. - * - If the destination buffer does not start at a 64-byte boundary, the - * transfer will make most efficient use of the memory bandwidth if the - * library uses a short segment to achieve 64-byte alignment for the - * bulk of the transfer. - * - If the source and destination buffers do not have the same alignment, - * the starting offset in the source buffer should be specified to align the - * first packet to a 64-byte boundary at the destination. - * - If the source and destination buffer alignments differ by an amount which - * is not a multiple of 4, the alignment must be adjusted by a software copy - * before or after the transfer. - */ - if( (align = (src_ptr ^ dst_ptr) & 0x3) ) { - opal_output(0, "Impossible to do a direct PUT of %d bytes. Data will have to be moved around by %d bytes (src_ptr %p dst_ptr %p)", - total_length, (int)((src_ptr | dst_ptr) & 0x3), (void*)src_ptr, (void*)dst_ptr); - align = dst_ptr & 0x3; - src_ptr = ((src_ptr + 4) & ~0x3) | align; - align = des->des_src->seg_addr.lval - src_ptr; - total_length -= align; - /*return OMPI_ERR_OUT_OF_RESOURCE;*/ - } - /* Send a small amount of data in such a way that the destination PUT will - * be aligned on the cache boundaries (64 bytes). - */ - if( (align = (0x20 - (dst_ptr & 0x1F))) ) { - uint32_t length_on_wire = scdma_send_event_len(align + 3 * sizeof(uint64_t)); - scdma_hw_cmd_t* cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx); - - cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT, - length_on_wire, - endpoint->route->route_handles[sicortex_btl->handle_count], - endpoint->route->ports[sicortex_btl->handle_count]); - cmd->payload[0] = (((uint64_t)0 << 16) | (EVENT_SEND_PUT_ALIGN << 8) - | (align + 3 * sizeof(uint64_t)) ); - cmd->payload[1] = dst_ptr; - memcpy( &(cmd->payload[2]), (void*)src_ptr, align ); - cmd->payload[15] = 0; - scdma_cq_post_fastpath(sicortex_btl->ctx, cmd->header); - src_ptr += align; - dst_ptr += align; - total_length -= align; - dst_offset += align; - /*opal_output(0, "Align the begining of the destination buffer by %d (dst_ptr %llx dst_offset %d)\n", - align, (unsigned long long)dst_ptr, dst_offset );*/ - if( dst_offset >= (64*1024) ) { - assert( 0 == (dst_offset % (64*1024)) ); - dst_offset = 0; - MCA_BTL_SICORTEX_NEXT_BDS(dst_bd_index); - dst_num_entries--; - } - src_offset += align; - if( src_offset >= (64*1024) ) { - src_offset %= (64*1024); - MCA_BTL_SICORTEX_NEXT_BDS(src_bd_index); - src_num_entries--; - } - } - /* Send a small amount of data to make sure that the last PUT will - * stop on a cache boundary alignment (64 bytes). - */ - if( (align = (total_length & 0x1F)) ) { - uint32_t length_on_wire = scdma_send_event_len(align + 3 * sizeof(uint64_t)); - scdma_hw_cmd_t* cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx); - - cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT, - length_on_wire, - endpoint->route->route_handles[sicortex_btl->handle_count], - endpoint->route->ports[sicortex_btl->handle_count]); - cmd->payload[0] = (((uint64_t)0 << 16) | (EVENT_SEND_PUT_ALIGN << 8) - | (align + 3 * sizeof(uint64_t)) ); - cmd->payload[1] = dst_ptr + total_length - align; - memcpy( &(cmd->payload[2]), (void*)(src_ptr + total_length - align), align ); - cmd->payload[15] = 0; - scdma_cq_post_fastpath(sicortex_btl->ctx, cmd->header); - total_length -= align; - /*opal_output(0, "Align the end of the destination buffer by %d (dst_ptr %llx)\n", - align, (unsigned long long)(dst_ptr + total_length) );*/ - } - - /* We can use overlapping on the remote side, but not locally. Therefore, we will - * compute the size of the first chunck, which will allow us to align locally on - * the 64K boundary. Then we will put 64K by 64K, except for the last put, - * where we will have to recompute the overhead. - */ - seg_length = (64 * 1024) - dst_offset; - do { - if( seg_length > total_length ) { - seg_length = total_length; - } - - frag->count++; /* count how many puts per fragment */ - /* Prepare the PUT order */ - pb = (scdma_hw_put_bf_bf_cmd_t*)scdma_cq_head_spinwait(sicortex_btl->ctx); - scdma_build_put_bf_bf_cmd( (void*)pb, - endpoint->route->route_handles[sicortex_btl->handle_count], - endpoint->route->ports[sicortex_btl->handle_count], - src_bd_index, src_offset, - dst_bd_index, dst_offset, - seg_length, 0, (uintptr_t)frag ); - scdma_cq_post(sicortex_btl->ctx); - - /*opal_output(0, "put src(index %d:offset %d) dst(index %d:offset %d) length %d route %d", - src_bd_index, src_offset, dst_bd_index, dst_offset, seg_length, - sicortex_btl->handle_count);*/ - - /* Update local informations: the route and the count */ - sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM; - - /* Now let's update the values */ - src_offset = src_offset + seg_length; - if( src_offset >= (64 * 1024) ) { - src_offset = src_offset % (64 * 1024); - MCA_BTL_SICORTEX_NEXT_BDS(src_bd_index); - } - dst_offset = 0; /* We're now indexed by destination pages */ - MCA_BTL_SICORTEX_NEXT_BDS(dst_bd_index); - total_length -= seg_length; /* Update the remaining length */ - seg_length = 64 * 1024; /* We're dealing now with full pages */ - } while (total_length != 0); - - return OMPI_SUCCESS; -} - - -/** - * Initiate an asynchronous get. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transferre - * - */ -#if 0 -static int -mca_btl_sicortex_get( mca_btl_base_module_t* btl, - mca_btl_base_endpoint_t* endpoint, - mca_btl_base_descriptor_t* des) -{ - mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl; - mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*) des; - void * cmd; - mca_btl_base_segment_t *src = des->des_src; - mca_btl_base_segment_t *dst = des->des_dst; - int seg_length = src->seg_len; - uint32_t tx_bd_index =(uint32_t)((src->seg_key.key64 >> 48) &0xffff);/* src->seg_key.key32[0];*/ - uint32_t tx_bd_offset =(uint32_t)((src->seg_key.key64 >> 32) &0xffff); /*src->seg_key.key32[1];*/ - uint32_t remote_route_handle = (uint32_t)(src->seg_key.key64 &0xffffffff); - uint32_t rx_begin_bd_index =(uint32_t)((dst->seg_key.key64 >> 48) &0xffff); - uint32_t rx_begin_bd_offset=(uint32_t) ((dst->seg_key.key64 >> 32) & 0xffff); - uint32_t rx_end_bd_index = (uint32_t)((dst->seg_key.key64 >>16) &0xffff); - uint32_t rx_end_bd_offset = (uint32_t)(dst->seg_key.key64 & 0xffff); - uint64_t sw_bucket = (uintptr_t)frag; - int length = rx_begin_bd_offset + seg_length; - uint32_t counter = rx_begin_bd_index; - int rx_bd_offset; - int tx_bd_index_now; - int tx_bd_index_next = tx_bd_index; - int tx_bd_offset_next = tx_bd_offset; - int tx_bd_offset_now; - sw_bucket = sw_bucket | 0x1; - /* opal_output(0,"sender %llx tx_bd_index %d\t tx_bd_offset %d\trx_begin_bd_index %d\t rx_begin_bd_offset %d\trx_end_bd_index%d\trx_end_bd_offset%d\t ",sw_bucket, tx_bd_index,tx_bd_offset,rx_begin_bd_index,rx_begin_bd_offset,rx_end_bd_index,rx_end_bd_offset);*/ - frag->endpoint = endpoint; - frag->count = 0; - frag->bds_index = tx_bd_index; - frag->remote_bds_start= rx_begin_bd_offset; - do { - frag->count++; - rx_bd_offset = 0; - tx_bd_offset_now = tx_bd_offset_next; - tx_bd_index_now = tx_bd_index_next; - if(counter == rx_begin_bd_index) { - rx_bd_offset = rx_begin_bd_offset; - if(length > BD_SIZE) { - seg_length = BD_SIZE - rx_begin_bd_offset; - length -= BD_SIZE; - } else { - seg_length = seg_length; - length =0; - } - } else if(counter == rx_end_bd_index) { - seg_length = rx_end_bd_offset; - length = 0; - } else if(counter> rx_begin_bd_index && counter< rx_end_bd_index) { - seg_length = BD_SIZE; - length -= BD_SIZE; - } else { - opal_output(0,"hahaha, u r wrong!!!"); - } - if( tx_bd_offset_now + seg_length >= BD_SIZE ) - tx_bd_index_next = tx_bd_index_now+1; - tx_bd_offset_next = (tx_bd_offset_now + seg_length) %(BD_SIZE); - - cmd = (void*)scdma_cq_head_spinwait(sicortex_btl->ctx); - scdma_build_send_put_bf_bf_cmd(cmd, - endpoint->route->route_handles[sicortex_btl->handle_count], - endpoint->route->ports[sicortex_btl->handle_count], - remote_route_handle, - tx_bd_index_now, - tx_bd_offset_now, - counter, - rx_bd_offset, - seg_length, - 0, - sw_bucket); - sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM; - scdma_cq_post(sicortex_btl->ctx); - } while( counter++ != rx_end_bd_index ); - - return OMPI_SUCCESS; -} -#endif -/* - * Cleanup/release module resources. - */ - -static int mca_btl_sicortex_finalize(struct mca_btl_base_module_t* btl) -{ - mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl; - - OBJ_DESTRUCT(&sicortex_btl->sicortex_lock); - free(sicortex_btl); - - return OMPI_SUCCESS; -} - -static int mca_btl_sicortex_ft_event(int state) -{ - if(OPAL_CRS_CHECKPOINT == state) { - ; - } - else if(OPAL_CRS_CONTINUE == state) { - ; - } - else if(OPAL_CRS_RESTART == state) { - ; - } - else if(OPAL_CRS_TERM == state ) { - ; - } - else { - ; - } - - return OMPI_SUCCESS; -} - -mca_btl_sicortex_module_t mca_btl_sicortex_module = { - { - &mca_btl_sicortex_component.super, - 0, /* max size of first fragment */ - 0, /* min send fragment size*/ - 0, /* max send fragment size */ - 0, /* rdma pipeline offset */ - 0, /* rdma pipeline frag size */ - 0, /* min rdma pipeline size */ - 0, /* exclusivity */ - 0, /* latency */ - 0, /* bandwidth */ - 0, /* flags */ - mca_btl_sicortex_add_procs, - mca_btl_sicortex_del_procs, - NULL, - mca_btl_sicortex_finalize, - mca_btl_sicortex_alloc, - mca_btl_sicortex_free, - mca_btl_sicortex_prepare_src, - mca_btl_sicortex_prepare_dst, - mca_btl_sicortex_send, - mca_btl_sicortex_sendi,/* send immediate */ - mca_btl_sicortex_put, /* put */ -#if 0 - mca_btl_sicortex_get, /* get */ -#else - NULL, /* get */ -#endif - NULL, /*dump */ - NULL, /* mpool */ - NULL, /* register error cb */ - mca_btl_sicortex_ft_event - } -}; - diff --git a/ompi/mca/btl/sicortex/btl_sicortex.h b/ompi/mca/btl/sicortex/btl_sicortex.h deleted file mode 100755 index 5da1469de0..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex.h +++ /dev/null @@ -1,166 +0,0 @@ - -/* - * Copyright (c) 2004-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ -#ifndef MCA_PTL_SICORTEX_H -#define MCA_PTL_SICORTEX_H - -/* Standard system includes */ -#include -#include - -/* Open MPI includes */ -#include "opal/class/opal_bitmap.h" -#include "opal/event/event.h" -#include "orte/util/show_help.h" -#include "ompi/class/ompi_free_list.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/btl/btl.h" -#include "ompi/mca/btl/base/base.h" -#include "ompi/mca/mpool/mpool.h" -#include "ompi/mca/btl/btl.h" - - -#include -#include -#include -#include - - -#if defined(c_plusplus) || defined(__cplusplus) -extern "C" { -#endif - -#define MCA_BTL_HAS_MPOOL 0 -#define SICORTEX_PORTS_NUM 3 -#define BD_SIZE 64*1024 -/* - * Infiniband (SICORTEX) BTL component. - */ - -enum eventcodes { EVENT_SEND = (SCDMA_HW_EVENT_TYPE_LIMIT + 1), - EVENT_SEND_IM, - EVENT_SEND_PUT_ALIGN, - EVENT_SEND_IM_PUT_POST, - EVENT_PUT }; - -struct peer_struct_t { - uint64_t context_id; - uint32_t route_handles[3]; - uint32_t ports[3]; -}; -typedef struct peer_struct_t peer_struct_t; - - -struct mca_btl_sicortex_component_t { - mca_btl_base_component_2_0_0_t super; /**< base BTL component */ - - size_t sicortex_num_btls; - /**< number of hcas available to the SICORTEX component */ - - size_t sicortex_max_btls; - - struct mca_btl_sicortex_module_t **sicortex_btls; - /**< array of available BTL modules */ - - int sicortex_free_list_num; - /**< initial size of free lists */ - - int sicortex_free_list_max; - /**< maximum size of free lists */ - - int sicortex_free_list_inc; - /**< number of elements to alloc when growing free lists */ - - opal_list_t sicortex_procs; - /**< list of sicortex proc structures */ - - opal_mutex_t sicortex_lock; - /**< lock for accessing module state */ - - ompi_free_list_t sicortex_frag_eager; - ompi_free_list_t sicortex_frag_max; - ompi_free_list_t sicortex_frag_user; - - char* sicortex_mpool_name; - /**< name of memory pool */ - - bool leave_pinned; - /**< pin memory on first use and leave pinned */ - -}; -typedef struct mca_btl_sicortex_component_t mca_btl_sicortex_component_t; - -OMPI_MODULE_DECLSPEC extern mca_btl_sicortex_component_t mca_btl_sicortex_component; - -/** - * BTL Module Interface - */ -struct mca_btl_sicortex_module_t { - mca_btl_base_module_t super; - - scdma_context_t *ctx; - uint64_t localContext; - struct mca_btl_base_endpoint_t** peers; - uint32_t bds_size; - uint32_t bds_limit; - uint32_t bds_head; - int handle_count; - opal_mutex_t sicortex_lock; - -#if MCA_BTL_HAS_MPOOL - struct mca_mpool_base_module_t* sicortex_mpool; -#endif -}; -typedef struct mca_btl_sicortex_module_t mca_btl_sicortex_module_t; -extern mca_btl_sicortex_module_t mca_btl_sicortex_module; - - -/** - * Register SICORTEX component parameters with the MCA framework - */ -extern int mca_btl_sicortex_component_open(void); - -/** - * Any final cleanup before being unloaded. - */ -extern int mca_btl_sicortex_component_close(void); - -/** - * SICORTEX component initialization. - * - * @param num_btl_modules (OUT) Number of BTLs returned in BTL array. - * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE) - * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE) - */ -extern mca_btl_base_module_t** mca_btl_sicortex_component_init( - int *num_btl_modules, - bool allow_multi_user_threads, - bool have_hidden_threads -); - - -/** - * SICORTEX component progress. - */ -extern int mca_btl_sicortex_component_progress(void); - - - - - - -#if defined(c_plusplus) || defined(__cplusplus) -} -#endif -#endif diff --git a/ompi/mca/btl/sicortex/btl_sicortex_component.c b/ompi/mca/btl/sicortex/btl_sicortex_component.c deleted file mode 100755 index a58fdd3e5d..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex_component.c +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Copyright (c) 2004-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include "ompi/constants.h" -#include "opal/event/event.h" -#include "opal/util/if.h" -#include "opal/util/argv.h" -#include "orte/util/show_help.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/btl/btl.h" - -#include "opal/mca/base/mca_base_param.h" -#include "ompi/runtime/ompi_module_exchange.h" -#include "orte/mca/errmgr/errmgr.h" -#include "ompi/mca/mpool/base/base.h" -#include "btl_sicortex.h" -#include "btl_sicortex_frag.h" -#include "btl_sicortex_endpoint.h" -#include "ompi/mca/btl/base/base.h" -#include "ompi/datatype/convertor.h" - -mca_btl_sicortex_component_t mca_btl_sicortex_component = { - { - /* First, the mca_base_component_t struct containing meta information - about the component itself */ - - { - /* Indicate that we are a pml v1.0.0 component (which also implies a - specific MCA version) */ - - MCA_BTL_BASE_VERSION_2_0_0, - - "sicortex", /* MCA component name */ - OMPI_MAJOR_VERSION, /* MCA component major version */ - OMPI_MINOR_VERSION, /* MCA component minor version */ - OMPI_RELEASE_VERSION, /* MCA component release version */ - mca_btl_sicortex_component_open, /* component open */ - mca_btl_sicortex_component_close /* component close */ - }, - - /* Next the MCA v1.0.0 component meta data */ - - { - /* The component is not checkpoint ready */ - MCA_BASE_METADATA_PARAM_NONE - }, - - mca_btl_sicortex_component_init, - mca_btl_sicortex_component_progress, - } -}; - - -/* - * utility routines for parameter registration - */ - -static inline char* -mca_btl_sicortex_param_register_string( const char* param_name, - const char* default_value ) -{ - char *param_value; - int id = mca_base_param_register_string( "btl", "sicortex", param_name, - NULL, default_value ); - mca_base_param_lookup_string(id, ¶m_value); - return param_value; -} - -static inline int -mca_btl_sicortex_param_register_int( const char* param_name, - int default_value ) -{ - int id = mca_base_param_register_int( "btl", "sicortex", param_name, - NULL, default_value ); - int param_value = default_value; - mca_base_param_lookup_int(id,¶m_value); - return param_value; -} - -/* - * Called by MCA framework to open the component, registers - * component parameters. - */ - -int mca_btl_sicortex_component_open(void) -{ - /* initialize state */ - mca_btl_sicortex_component.sicortex_num_btls = 0; - mca_btl_sicortex_component.sicortex_btls = NULL; - - /* initialize objects */ - OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_procs, opal_list_t); - OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_lock, opal_mutex_t); - - /* register SICORTEX component parameters */ - mca_btl_sicortex_component.sicortex_max_btls = - mca_btl_sicortex_param_register_int("max_modules", 4); - mca_btl_sicortex_component.sicortex_free_list_num = - mca_btl_sicortex_param_register_int ("free_list_num", 8); - mca_btl_sicortex_component.sicortex_free_list_max = - mca_btl_sicortex_param_register_int ("free_list_max", 1024); - mca_btl_sicortex_component.sicortex_free_list_inc = - mca_btl_sicortex_param_register_int ("free_list_inc", 32); - mca_btl_sicortex_component.sicortex_mpool_name = - mca_btl_sicortex_param_register_string("mpool", "sicortex"); - - mca_btl_sicortex_module.super.btl_exclusivity = 0; - mca_btl_sicortex_module.super.btl_eager_limit = 2400;/* 4*1024 - sizeof(mca_btl_base_header_t);*/ - mca_btl_sicortex_module.super.btl_rndv_eager_limit = 4*1024;/*4*1024 - sizeof(mca_btl_base_header_t);*/ - mca_btl_sicortex_module.super.btl_max_send_size = 4*1024;/*4*1024 - sizeof(mca_btl_base_header_t);*/ - mca_btl_sicortex_module.super.btl_min_rdma_pipeline_size = 1024*1024; - mca_btl_sicortex_module.super.btl_rdma_pipeline_frag_size = 1024*1024; - mca_btl_sicortex_module.super.btl_rdma_pipeline_send_length = 1024*1024; - mca_btl_sicortex_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_PUT; - mca_btl_sicortex_module.super.btl_latency = 3; /* Microsec */ - mca_btl_sicortex_module.super.btl_bandwidth = 14000; /* Mbs */ - - mca_btl_base_param_register(&mca_btl_sicortex_component.super.btl_version, - &mca_btl_sicortex_module.super); - - return OMPI_SUCCESS; -} - -/* - * component cleanup - sanity checking of queue lengths - */ - -int mca_btl_sicortex_component_close(void) -{ - if( 0 != mca_btl_sicortex_component.sicortex_num_btls ) { - OBJ_DESTRUCT(&mca_btl_sicortex_component.sicortex_frag_eager); - OBJ_DESTRUCT(&mca_btl_sicortex_component.sicortex_frag_max); - OBJ_DESTRUCT(&mca_btl_sicortex_component.sicortex_frag_user); - mca_btl_sicortex_component.sicortex_num_btls = 0; - free(mca_btl_sicortex_component.sicortex_btls); - mca_btl_sicortex_component.sicortex_btls = NULL; - } - - return OMPI_SUCCESS; -} - -static int mca_btl_sicortex_modex_send(void) -{ - int rc; - size_t i; - size_t size; - uint64_t *addrs = NULL; - - size = mca_btl_sicortex_component.sicortex_num_btls * sizeof( uint64_t); - if(0 != size){ - addrs = (uint64_t*) malloc(size); - for( i = 0; i < mca_btl_sicortex_component.sicortex_num_btls; i++ ) { - mca_btl_sicortex_module_t *btl = mca_btl_sicortex_component.sicortex_btls[i]; - addrs[i]= btl->localContext; - } - } - rc = ompi_modex_send(&mca_btl_sicortex_component.super.btl_version, addrs, size); - if( NULL != addrs ) { - free(addrs); - } - return rc; - -} - -mca_btl_base_module_t** -mca_btl_sicortex_component_init( int *num_btl_modules, - bool enable_progress_threads, - bool enable_mpi_threads ) -{ - int rc; - mca_btl_base_module_t **btls; - uint64_t localContext; - mca_btl_sicortex_module_t *btl; - scdma_context_t *ctx = (scdma_context_t*) malloc(sizeof(scdma_context_t)); - - *num_btl_modules = 0; - - /*init scdma*/ - rc = scdma_open( -1, /* Process (-1 for ANY) */ - ctx, /* the context... */ - 131072, /* event queue size */ - 16384, /* cmd queue size */ - 65536, /* heap size */ - 4096, /* num RDT entries */ - 4096); /* num BDT entries */ - if( rc ) { - opal_output( 0, "[%s:%d] error in initializing the sicortex library\n", __FILE__, __LINE__ ); - return NULL; - } - - /* get context of each card */ - localContext = scdma_get_my_context_id(ctx); - mca_btl_sicortex_component.sicortex_btls = malloc( mca_btl_sicortex_component.sicortex_max_btls * sizeof (mca_btl_sicortex_module_t*)); - if( NULL == mca_btl_sicortex_component.sicortex_btls ) { - return NULL; - } - - mca_btl_sicortex_component.sicortex_num_btls = 0; - btl = (mca_btl_sicortex_module_t *)malloc( sizeof(mca_btl_sicortex_module_t) ); - memcpy (btl, &mca_btl_sicortex_module, sizeof(mca_btl_sicortex_module_t)); - btl->localContext = localContext; - btl->ctx = ctx; - btl->handle_count = 0; - - OBJ_CONSTRUCT(&btl->sicortex_lock, opal_mutex_t); - - mca_btl_sicortex_component.sicortex_btls[mca_btl_sicortex_component.sicortex_num_btls] = btl; - mca_btl_sicortex_component.sicortex_num_btls++; - - if (OMPI_SUCCESS != mca_btl_sicortex_modex_send()) { - return NULL; - } - - /* initialize free lists */ - OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_frag_eager, ompi_free_list_t); - ompi_free_list_init_new( &mca_btl_sicortex_component.sicortex_frag_eager, - sizeof(mca_btl_sicortex_frag_eager_t) + mca_btl_sicortex_module.super.btl_eager_limit, - CACHE_LINE_SIZE, - OBJ_CLASS (mca_btl_sicortex_frag_eager_t), - 0, - CACHE_LINE_SIZE, - mca_btl_sicortex_component.sicortex_free_list_num, - mca_btl_sicortex_component.sicortex_free_list_max, - mca_btl_sicortex_component.sicortex_free_list_inc, - btl->super.btl_mpool ); - - OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_frag_max, ompi_free_list_t); - ompi_free_list_init_new( &mca_btl_sicortex_component.sicortex_frag_max, - sizeof (mca_btl_sicortex_frag_max_t) + mca_btl_sicortex_module.super.btl_max_send_size, - CACHE_LINE_SIZE, - OBJ_CLASS (mca_btl_sicortex_frag_max_t), - 0, - CACHE_LINE_SIZE, - mca_btl_sicortex_component.sicortex_free_list_num, - mca_btl_sicortex_component.sicortex_free_list_max, - mca_btl_sicortex_component.sicortex_free_list_inc, - btl->super.btl_mpool ); - - OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_frag_user, ompi_free_list_t); - ompi_free_list_init_new( &mca_btl_sicortex_component.sicortex_frag_user, - sizeof (mca_btl_sicortex_frag_user_t), - CACHE_LINE_SIZE, - OBJ_CLASS (mca_btl_sicortex_frag_user_t), - 0, - CACHE_LINE_SIZE, - mca_btl_sicortex_component.sicortex_free_list_num, - mca_btl_sicortex_component.sicortex_free_list_max, - mca_btl_sicortex_component.sicortex_free_list_inc, - NULL ); - - /* return array of BTLs */ - btls = (mca_btl_base_module_t**) malloc ( - mca_btl_sicortex_component.sicortex_num_btls * sizeof(mca_btl_base_module_t *)); - if( NULL == btls ){ - return NULL; - } - - memcpy(btls, mca_btl_sicortex_component.sicortex_btls, - mca_btl_sicortex_component.sicortex_num_btls * sizeof(mca_btl_sicortex_module_t *)); - *num_btl_modules = mca_btl_sicortex_component.sicortex_num_btls; - return btls; -} - -int mca_btl_sicortex_component_progress(void) -{ - int count = 0; - size_t i; - uint64_t *evt; - - for( i = 0; i < mca_btl_sicortex_component.sicortex_num_btls; i++ ) { - mca_btl_sicortex_module_t* btl = mca_btl_sicortex_component.sicortex_btls[i]; - int pooling = 0; - - while( pooling++ < 5000 ) { - evt = (uint64_t *) scdma_eq_next_event(btl->ctx); - if( NULL != evt ) break; - } - if( NULL == evt ) continue; - - switch ((evt[0] >> 8) & 0xff) { - - case SCDMA_HW_EVENT_TYPE_RDT_FAULT: { - uint64_t sw_bucket = evt[1]; - - opal_output( 0, "Error RDT_FAULT sw_bucket %llx", (unsigned long long)sw_bucket ); - scdma_eq_processed_event(btl->ctx); - break; - } - case SCDMA_HW_EVENT_TYPE_BDT_FAULT: { - uint64_t sw_bucket = evt[1]; - - opal_output( 0, "Error BDT_FAULT sw_bucket %llx", (unsigned long long)sw_bucket ); - scdma_eq_processed_event(btl->ctx); - break; - } - case SCDMA_HW_EVENT_TYPE_SEG_ABORT: { - uint64_t sw_bucket = evt[1]; - - opal_output( 0, "Error SEG_ABORT sw_bucket %llx", (unsigned long long)sw_bucket ); - scdma_eq_processed_event(btl->ctx); - break; - } - - case SCDMA_HW_EVENT_TYPE_DEFERRED_CMD: { - void *cmd = (void *) scdma_cq_head_spinwait(btl->ctx); - - memcpy(cmd, &evt[1], 0x78); - scdma_cq_post(btl->ctx); - scdma_eq_processed_event(btl->ctx); - break; - } - - case SCDMA_HW_EVENT_TYPE_RX_END_SEG: { - scdma_hw_cmd_t *cmd_2; - uint64_t swbucket = evt[1]; - - scdma_eq_processed_event(btl->ctx); - cmd_2 = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(btl->ctx); - cmd_2->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT, 0x18, - btl->peers[0]->route->route_handles[btl->handle_count], - btl->peers[0]->route->ports[btl->handle_count]); - btl->handle_count = (btl->handle_count +1) % SICORTEX_PORTS_NUM; - cmd_2->payload[0] = (EVENT_PUT << 8) + 0x18; - cmd_2->payload[1] = swbucket; - cmd_2->payload[2] = 0; - scdma_cq_post_fastpath(btl->ctx,cmd_2->header); - break; - } - - /* Receive small message */ - case EVENT_SEND_IM: { - mca_btl_active_message_callback_t* reg; - mca_btl_sicortex_frag_t frag; - - frag.base.des_dst = &frag.segment; - frag.base.des_dst->seg_addr.pval = (void*) (evt+1); - frag.base.des_dst->seg_len = (size_t)((evt[0] & SCDMA_HW_CMD_LEN_MASK) - - 2 * sizeof(uint64_t)); - frag.base.des_dst_cnt = 1; - frag.tag = (mca_btl_base_tag_t)((evt[0] >> 16) & 0xff); - reg = mca_btl_base_active_message_trigger + frag.tag; - reg->cbfunc( &(btl->super), frag.tag, &(frag.base), reg->cbdata ); - scdma_eq_processed_event(btl->ctx); - count++; - break; - } - - case EVENT_SEND_PUT_ALIGN: { - /* There are at most 2 PUT_ALIGN per segment, one of the begining and - * one for the end. As the DMA guarantee that the orders in each route and port - * are completed in order, we don't have to care about these two PUT_ALIGN - * as they will always be followed by a real PUT on the same route. When the - * PUT complete, these two PUT_ALIGN have to be completed. - */ - uint32_t length = ((evt[0] & SCDMA_HW_CMD_LEN_MASK) - 3 * sizeof(uint64_t)); - void* ptr = (void*)evt[1]; - memcpy( ptr, &(evt[2]), length ); - /*opal_output(0, "PUT_ALIGN ptr %p length %d\n", ptr, length );*/ - scdma_eq_processed_event(btl->ctx); - break; - } - - case EVENT_SEND_IM_PUT_POST: { - mca_btl_active_message_callback_t* reg; - mca_btl_sicortex_frag_t frag; - - frag.base.des_dst = &frag.segment; - frag.base.des_dst->seg_addr.pval = (void*)((char*)scdma_heap_addr(btl->ctx) + SCDMA_USABLE_HEAP_OFFSET); - frag.base.des_dst->seg_len = (size_t)((evt[0]>> 32 & 0xffffffff)); - frag.base.des_dst_cnt = 1; - frag.tag = (mca_btl_base_tag_t)((evt[0]>>16)& 0xff); - reg = mca_btl_base_active_message_trigger + frag.tag; - reg->cbfunc( &(btl->super), frag.tag, &(frag.base), reg->cbdata ); - scdma_eq_processed_event(btl->ctx); - count++; - break; - } - - case EVENT_PUT: { - mca_btl_sicortex_frag_t *frag = (mca_btl_sicortex_frag_t *)(uintptr_t) evt[1]; - if( --frag->count == 0 ) { - int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP ); - count++; - frag->base.des_cbfunc( &(btl->super), frag->endpoint,&(frag->base), OMPI_SUCCESS ); - if( btl_ownership ) { - MCA_BTL_SICORTEX_FRAG_RETURN(btl, frag); - } - } - scdma_eq_processed_event(btl->ctx); - break; - } - - default: - opal_output(0, "Got a badley formatted event (evt[0] = 0x%llx)\n", - (long long unsigned int)evt[0]); - scdma_eq_processed_event(btl->ctx); - sleep(5); - break; - } - } - return count; -} - - - diff --git a/ompi/mca/btl/sicortex/btl_sicortex_endpoint.c b/ompi/mca/btl/sicortex/btl_sicortex_endpoint.c deleted file mode 100755 index 895b80ca19..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex_endpoint.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2008-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include -#include -#include "ompi/types.h" -#include "orte/mca/oob/base/base.h" -#include "orte/mca/rml/rml.h" -#include "orte/mca/errmgr/errmgr.h" -#include "opal/dss/dss.h" -#include "btl_sicortex.h" -#include "btl_sicortex_endpoint.h" -#include "btl_sicortex_proc.h" -#include "btl_sicortex_frag.h" - - -/* - * Initialize state of the endpoint instance. - * - */ - -static void mca_btl_sicortex_endpoint_construct(mca_btl_base_endpoint_t* endpoint) -{ - endpoint->endpoint_btl = 0; - endpoint->endpoint_proc = 0; -} - -/* - * Destroy a endpoint - * - */ - -static void mca_btl_sicortex_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) -{ -} - - -OBJ_CLASS_INSTANCE( - mca_btl_sicortex_endpoint_t, - opal_list_item_t, - mca_btl_sicortex_endpoint_construct, - mca_btl_sicortex_endpoint_destruct); - diff --git a/ompi/mca/btl/sicortex/btl_sicortex_endpoint.h b/ompi/mca/btl/sicortex/btl_sicortex_endpoint.h deleted file mode 100755 index 045638ed7a..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex_endpoint.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2008-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_SICORTEX_ENDPOINT_H -#define MCA_BTL_SICORTEX_ENDPOINT_H - -#include "opal/class/opal_list.h" -#include "opal/event/event.h" -#include "ompi/mca/pml/pml.h" -#include "ompi/mca/btl/btl.h" -#include "btl_sicortex_frag.h" -#include "btl_sicortex.h" -#if defined(c_plusplus) || defined(__cplusplus) -extern "C" { -#endif - -/** - * An abstraction that represents a connection to a endpoint process. - * An instance of mca_btl_base_endpoint_t is associated w/ each process - * and BTL pair at startup. However, connections to the endpoint - * are established dynamically on an as-needed basis: - */ - -struct mca_btl_base_endpoint_t { - opal_list_item_t super; - - struct mca_btl_sicortex_module_t* endpoint_btl; - /**< BTL instance that created this connection */ - - struct mca_btl_sicortex_proc_t* endpoint_proc; - /**< proc structure corresponding to endpoint */ - - struct peer_struct_t* route; -}; - -typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t; -typedef mca_btl_base_endpoint_t mca_btl_sicortex_endpoint_t; -OBJ_CLASS_DECLARATION(mca_btl_sicortex_endpoint_t); - -#if defined(c_plusplus) || defined(__cplusplus) -} -#endif -#endif diff --git a/ompi/mca/btl/sicortex/btl_sicortex_frag.c b/ompi/mca/btl/sicortex/btl_sicortex_frag.c deleted file mode 100755 index 7ea28283ad..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex_frag.c +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2008-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "btl_sicortex_frag.h" - - - -static void mca_btl_sicortex_frag_common_constructor(mca_btl_sicortex_frag_t* frag) -{ - frag->base.des_src = NULL; - frag->base.des_src_cnt = 0; - frag->base.des_dst = &(frag->segment); - frag->base.des_dst_cnt = 1; -#if MCA_BTL_HAS_MPOOL - frag->registration = NULL; -#endif -} - -static void mca_btl_sicortex_frag_eager_constructor(mca_btl_sicortex_frag_t* frag) -{ - frag->size = mca_btl_sicortex_module.super.btl_eager_limit; - frag->my_list = &mca_btl_sicortex_component.sicortex_frag_eager; - mca_btl_sicortex_frag_common_constructor(frag); -} - -static void mca_btl_sicortex_frag_max_constructor(mca_btl_sicortex_frag_t* frag) -{ - frag->size = mca_btl_sicortex_module.super.btl_max_send_size; - frag->my_list = &mca_btl_sicortex_component.sicortex_frag_max; - mca_btl_sicortex_frag_common_constructor(frag); -} - -static void mca_btl_sicortex_frag_user_constructor(mca_btl_sicortex_frag_t* frag) -{ - frag->size = 0; - frag->my_list = &mca_btl_sicortex_component.sicortex_frag_user; - mca_btl_sicortex_frag_common_constructor(frag); -} - - -OBJ_CLASS_INSTANCE( - mca_btl_sicortex_frag_t, - mca_btl_base_descriptor_t, - NULL, - NULL); - -OBJ_CLASS_INSTANCE( - mca_btl_sicortex_frag_eager_t, - mca_btl_base_descriptor_t, - mca_btl_sicortex_frag_eager_constructor, - NULL); - -OBJ_CLASS_INSTANCE( - mca_btl_sicortex_frag_max_t, - mca_btl_base_descriptor_t, - mca_btl_sicortex_frag_max_constructor, - NULL); - -OBJ_CLASS_INSTANCE( - mca_btl_sicortex_frag_user_t, - mca_btl_base_descriptor_t, - mca_btl_sicortex_frag_user_constructor, - NULL); - diff --git a/ompi/mca/btl/sicortex/btl_sicortex_frag.h b/ompi/mca/btl/sicortex/btl_sicortex_frag.h deleted file mode 100755 index c3b7b36d7a..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex_frag.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2008-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_SICORTEX_FRAG_H -#define MCA_BTL_SICORTEX_FRAG_H - -#include "ompi_config.h" -#include "ompi/class/ompi_free_list.h" -#include "btl_sicortex.h" - -BEGIN_C_DECLS - -/** - * SICORTEX send frasicortexent derived type. - */ -struct mca_btl_sicortex_frag_t { - mca_btl_base_descriptor_t base; - mca_btl_base_segment_t segment; - ompi_free_list_t* my_list; - struct mca_btl_base_endpoint_t* endpoint; - mca_btl_base_tag_t tag; - int count; - size_t size; -#if MCA_BTL_HAS_MPOOL - struct mca_mpool_base_registration_t* registration; -#endif -}; -typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_t; -OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_t); - -typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_eager_t; - -OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_eager_t); - -typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_max_t; - -OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_max_t); - -typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_user_t; - -OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_user_t); - - -/* - * Macros to allocate/return descriptors from module specific - * free list(s). - */ - -#define MCA_BTL_SICORTEX_FRAG_ALLOC(list, frag, rc) \ - do { \ - ompi_free_list_item_t *_item; \ - OMPI_FREE_LIST_GET((list), _item, rc); \ - frag = (mca_btl_sicortex_frag_t*)_item; \ - } while (0) - -#define MCA_BTL_SICORTEX_FRAG_RETURN(btl, frag) \ - do { \ - OMPI_FREE_LIST_RETURN(frag->my_list, \ - (ompi_free_list_item_t*)(frag)); \ - } while (0) - -#define MCA_BTL_SICORTEX_FRAG_ALLOC_EAGER(btl, frag, rc) \ - MCA_BTL_SICORTEX_FRAG_ALLOC(&mca_btl_sicortex_component.sicortex_frag_eager, frag, rc ) - -#define MCA_BTL_SICORTEX_FRAG_ALLOC_MAX(btl, frag, rc) \ - MCA_BTL_SICORTEX_FRAG_ALLOC(&mca_btl_sicortex_component.sicortex_frag_max, frag, rc ) - -#define MCA_BTL_SICORTEX_FRAG_ALLOC_USER(btl, frag, rc) \ - MCA_BTL_SICORTEX_FRAG_ALLOC(&mca_btl_sicortex_component.sicortex_frag_user, frag, rc ) - -END_C_DECLS - -#endif diff --git a/ompi/mca/btl/sicortex/btl_sicortex_proc.c b/ompi/mca/btl/sicortex/btl_sicortex_proc.c deleted file mode 100755 index f9596e1ce8..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex_proc.c +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2008-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "opal/class/opal_hash_table.h" -#include "ompi/runtime/ompi_module_exchange.h" - -#include "btl_sicortex.h" -#include "btl_sicortex_proc.h" - -static void mca_btl_sicortex_proc_construct(mca_btl_sicortex_proc_t* proc); -static void mca_btl_sicortex_proc_destruct(mca_btl_sicortex_proc_t* proc); - -OBJ_CLASS_INSTANCE(mca_btl_sicortex_proc_t, - opal_list_item_t, mca_btl_sicortex_proc_construct, - mca_btl_sicortex_proc_destruct); - -void mca_btl_sicortex_proc_construct(mca_btl_sicortex_proc_t* sicortex_proc) -{ - sicortex_proc->proc_ompi = 0; - sicortex_proc->proc_addr_count = 0; - sicortex_proc->proc_endpoints = 0; - sicortex_proc->proc_endpoint_count = 0; - OBJ_CONSTRUCT(&sicortex_proc->proc_lock, opal_mutex_t); - /* add to list of all proc instance */ - OPAL_THREAD_LOCK(&mca_btl_sicortex_component.sicortex_lock); - opal_list_append(&mca_btl_sicortex_component.sicortex_procs, &sicortex_proc->super); - OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock); -} - -/* - * Cleanup ib proc instance - */ - -void mca_btl_sicortex_proc_destruct(mca_btl_sicortex_proc_t* sicortex_proc) -{ - /* remove from list of all proc instances */ - OPAL_THREAD_LOCK(&mca_btl_sicortex_component.sicortex_lock); - opal_list_remove_item(&mca_btl_sicortex_component.sicortex_procs, &sicortex_proc->super); - OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock); - - /* release resources */ - if(NULL != sicortex_proc->proc_endpoints) { - free(sicortex_proc->proc_endpoints); - } -} - - -/* - * Look for an existing SICORTEX process instances based on the associated - * ompi_proc_t instance. - */ -static mca_btl_sicortex_proc_t* mca_btl_sicortex_proc_lookup_ompi(ompi_proc_t* ompi_proc) -{ - mca_btl_sicortex_proc_t* sicortex_proc; - - OPAL_THREAD_LOCK(&mca_btl_sicortex_component.sicortex_lock); - - for(sicortex_proc = (mca_btl_sicortex_proc_t*) - opal_list_get_first(&mca_btl_sicortex_component.sicortex_procs); - sicortex_proc != (mca_btl_sicortex_proc_t*) - opal_list_get_end(&mca_btl_sicortex_component.sicortex_procs); - sicortex_proc = (mca_btl_sicortex_proc_t*)opal_list_get_next(sicortex_proc)) { - - if(sicortex_proc->proc_ompi == ompi_proc) { - OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock); - return sicortex_proc; - } - - } - - OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock); - - return NULL; -} - -/* - * Create a SICORTEX process structure. There is a one-to-one correspondence - * between a ompi_proc_t and a mca_btl_sicortex_proc_t instance. We cache - * additional data (specifically the list of mca_btl_sicortex_endpoint_t instances, - * and published addresses) associated w/ a given destination on this - * datastructure. - */ - -mca_btl_sicortex_proc_t* mca_btl_sicortex_proc_create(ompi_proc_t* ompi_proc) -{ - mca_btl_sicortex_proc_t* module_proc = NULL; - int rc; - size_t size; - - /* Check if we have already created a SICORTEX proc - * structure for this ompi process */ - module_proc = mca_btl_sicortex_proc_lookup_ompi(ompi_proc); - - if(module_proc != NULL) { - return module_proc; - } - - /* Oops! First time, gotta create a new SICORTEX proc - * out of the ompi_proc ... */ - - module_proc = OBJ_NEW(mca_btl_sicortex_proc_t); - - /* Initialize number of peer */ - module_proc->proc_endpoint_count = 0; - - module_proc->proc_ompi = ompi_proc; - - /* build a unique identifier (of arbitrary - * size) to represent the proc */ - module_proc->proc_guid = ompi_proc->proc_name; - rc = ompi_modex_recv( - &mca_btl_sicortex_component.super.btl_version, - ompi_proc, - (void*)&module_proc->proc_addrs, - &size); - - if(OMPI_SUCCESS != rc) { - OBJ_RELEASE(module_proc); - return NULL; - } - - if((size % sizeof(uint64_t))!= 0 ){ - OBJ_RELEASE(module_proc); - return NULL; - } - - - module_proc->proc_addr_count = size/sizeof(uint64_t); - if(0 == module_proc->proc_addr_count){ - module_proc->proc_endpoints = NULL; - }else{ - module_proc->proc_endpoints = (mca_btl_base_endpoint_t**) - malloc(module_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*)); - } - - - if(NULL == module_proc->proc_endpoints) { - OBJ_RELEASE(module_proc); - return NULL; - } - return module_proc; -} - - -/* - * Note that this routine must be called with the lock on the process - * already held. Insert a btl instance into the proc array and assign - * it an address. - */ -int mca_btl_sicortex_proc_insert(mca_btl_sicortex_proc_t* module_proc, - mca_btl_base_endpoint_t* module_endpoint) -{ - /* insert into endpoint array */ - module_endpoint->route = (peer_struct_t*)malloc(sizeof(peer_struct_t)); - module_endpoint->route->context_id = module_proc->proc_addrs[module_proc->proc_endpoint_count]; - module_endpoint->route->route_handles[0]=((module_proc->proc_endpoint_count*3)+0)<<3; - module_endpoint->route->route_handles[1]=((module_proc->proc_endpoint_count*3)+1)<<3; - module_endpoint->route->route_handles[2]=((module_proc->proc_endpoint_count*3)+2)<<3; - module_endpoint->endpoint_proc = module_proc; - module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint; - - return OMPI_SUCCESS; -} diff --git a/ompi/mca/btl/sicortex/btl_sicortex_proc.h b/ompi/mca/btl/sicortex/btl_sicortex_proc.h deleted file mode 100755 index f945006ffe..0000000000 --- a/ompi/mca/btl/sicortex/btl_sicortex_proc.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2008-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_SICORTEX_PROC_H -#define MCA_BTL_SICORTEX_PROC_H - -#include "opal/class/opal_object.h" -#include "ompi/proc/proc.h" -#include "btl_sicortex.h" -#include "btl_sicortex_endpoint.h" - -#if defined(c_plusplus) || defined(__cplusplus) -extern "C" { -#endif - -/** - * Represents the state of a remote process and the set of addresses - * that it exports. Also cache an instance of mca_btl_base_endpoint_t for - * each - * BTL instance that attempts to open a connection to the process. - */ -struct mca_btl_sicortex_proc_t { - opal_list_item_t super; - /**< allow proc to be placed on a list */ - - ompi_proc_t *proc_ompi; - /**< pointer to corresponding ompi_proc_t */ - - orte_process_name_t proc_guid; - /**< globally unique identifier for the process */ - - uint64_t* proc_addrs; - - size_t proc_addr_count; - /**< number of addresses published by endpoint */ - - struct mca_btl_base_endpoint_t **proc_endpoints; - /**< array of endpoints that have been created to access this proc */ - - size_t proc_endpoint_count; - /**< number of endpoints */ - - opal_mutex_t proc_lock; - /**< lock to protect against concurrent access to proc state */ -}; -typedef struct mca_btl_sicortex_proc_t mca_btl_sicortex_proc_t; -OBJ_CLASS_DECLARATION(mca_btl_sicortex_proc_t); - -mca_btl_sicortex_proc_t* mca_btl_sicortex_proc_create(ompi_proc_t* ompi_proc); -int mca_btl_sicortex_proc_insert(mca_btl_sicortex_proc_t*, mca_btl_base_endpoint_t*); - -#if defined(c_plusplus) || defined(__cplusplus) -} -#endif -#endif diff --git a/ompi/mca/btl/sicortex/configure.m4 b/ompi/mca/btl/sicortex/configure.m4 deleted file mode 100755 index bc26e439e6..0000000000 --- a/ompi/mca/btl/sicortex/configure.m4 +++ /dev/null @@ -1,71 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2008-2009 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# OMPI_CHECK_SICORTEX(prefix, [action-if-found], [action-if-not-found]) -# -------------------------------------------------------- -# check if Sicortex support can be found. sets prefix_{CPPFLAGS, -# LDFLAGS, LIBS} as needed and runs action-if-found if there is -# support, otherwise executes action-if-not-found -AC_DEFUN([OMPI_CHECK_SICORTEX],[ - AC_ARG_WITH([sicortex], - [AC_HELP_STRING([--with-sicortex(=DIR)], - [Build Sicortex (scdma) support, searching for libraries in DIR])]) - AC_ARG_WITH([sicortex-libdir], - [AC_HELP_STRING([--with-sicortex-libdir=DIR], - [Search for Sicortex (scdma) libraries in DIR])]) - - AS_IF([test "$with_sicortex" != "no"], - [AS_IF([test ! -z "$with_sicortex" -a "$with_sicortex" != "yes"], - [ompi_check_sicortex_dir="$with_sicortex"]) - AS_IF([test ! -z "$with_sicortex_libdir" -a "$with_sicortex_libdir" != "yes"], - [ompi_check_sicortex_libdir="$with_sicortex_libdir"]) - - OMPI_CHECK_PACKAGE([$1], - [sicortex/scdma.h], - [scdma], - [scdma_open], - [], - [$ompi_check_sicortex_dir], - [$ompi_check_sicortex_libdir], - [ompi_check_sicortex_happy="yes"], - [ompi_check_sicortex_happy="no"]) - ], - [ompi_check_sicortex_happy="no"]) - - AS_IF([test "$ompi_check_sicortex_happy" = "yes"], - [$2], - [AS_IF([test ! -z "$with_sicortex" -a "$with_sicortex" != "no"], - [AC_MSG_ERROR([Sicortex (scdma) support requested but not found. Aborting])]) - $3]) - ]) - -# MCA_btl_sicortex_CONFIG([action-if-can-compile], -# [action-if-cant-compile]) -# ------------------------------------------------ -AC_DEFUN([MCA_btl_sicortex_CONFIG],[ - OMPI_CHECK_SICORTEX([btl_sicortex], - [btl_sicortex_happy="yes"], - [btl_sicortex_happy="no"]) - - AS_IF([test "$btl_sicortex_happy" = "yes"], - [btl_sicortex_WRAPPER_EXTRA_LDFLAGS="$btl_sicortex_LDFLAGS" - btl_sicortex_WRAPPER_EXTRA_LIBS="$btl_sicortex_LIBS" - $1], - [$2]) - - # substitute in the things needed to build sicortex - AC_SUBST([btl_sicortex_CFLAGS]) - AC_SUBST([btl_sicortex_CPPFLAGS]) - AC_SUBST([btl_sicortex_LDFLAGS]) - AC_SUBST([btl_sicortex_LIBS]) -])dnl - diff --git a/ompi/mca/btl/sicortex/configure.params b/ompi/mca/btl/sicortex/configure.params deleted file mode 100755 index 2e44f8ef39..0000000000 --- a/ompi/mca/btl/sicortex/configure.params +++ /dev/null @@ -1,15 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2008-2009 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# Specific to this module - -PARAM_CONFIG_FILES="Makefile"