From bac67e0d81bc1888a8d122e49f5e2e56a97f6411 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Thu, 12 Dec 2013 18:54:47 +0000 Subject: [PATCH] Per discussion @Chicago OMPI dev meeting Dec 2013: remove all MX support. This commit was SVN r29873. --- ompi/mca/btl/mx/Makefile.am | 48 -- ompi/mca/btl/mx/btl_mx.c | 701 -------------------------- ompi/mca/btl/mx/btl_mx.h | 304 ------------ ompi/mca/btl/mx/btl_mx_component.c | 764 ----------------------------- ompi/mca/btl/mx/btl_mx_endpoint.c | 55 --- ompi/mca/btl/mx/btl_mx_endpoint.h | 92 ---- ompi/mca/btl/mx/btl_mx_frag.c | 71 --- ompi/mca/btl/mx/btl_mx_frag.h | 99 ---- ompi/mca/btl/mx/btl_mx_proc.c | 243 --------- ompi/mca/btl/mx/btl_mx_proc.h | 58 --- ompi/mca/btl/mx/configure.m4 | 42 -- ompi/mca/common/mx/Makefile.am | 94 ---- ompi/mca/common/mx/common_mx.c | 115 ----- ompi/mca/common/mx/common_mx.h | 43 -- ompi/mca/common/mx/configure.m4 | 42 -- ompi/mca/mtl/mx/Makefile.am | 60 --- ompi/mca/mtl/mx/configure.m4 | 49 -- ompi/mca/mtl/mx/mtl_mx.c | 278 ----------- ompi/mca/mtl/mx/mtl_mx.h | 87 ---- ompi/mca/mtl/mx/mtl_mx_cancel.c | 47 -- ompi/mca/mtl/mx/mtl_mx_component.c | 168 ------- ompi/mca/mtl/mx/mtl_mx_endpoint.c | 105 ---- ompi/mca/mtl/mx/mtl_mx_endpoint.h | 71 --- ompi/mca/mtl/mx/mtl_mx_probe.c | 86 ---- ompi/mca/mtl/mx/mtl_mx_recv.c | 89 ---- ompi/mca/mtl/mx/mtl_mx_request.h | 40 -- ompi/mca/mtl/mx/mtl_mx_send.c | 191 -------- ompi/mca/mtl/mx/mtl_mx_types.h | 128 ----- ompi/mca/mtl/mx/post_configure.sh | 1 - 29 files changed, 4171 deletions(-) delete mode 100644 ompi/mca/btl/mx/Makefile.am delete mode 100644 ompi/mca/btl/mx/btl_mx.c delete mode 100644 ompi/mca/btl/mx/btl_mx.h delete mode 100644 ompi/mca/btl/mx/btl_mx_component.c delete mode 100644 ompi/mca/btl/mx/btl_mx_endpoint.c delete mode 100644 ompi/mca/btl/mx/btl_mx_endpoint.h delete mode 100644 ompi/mca/btl/mx/btl_mx_frag.c delete mode 100644 ompi/mca/btl/mx/btl_mx_frag.h delete mode 100644 ompi/mca/btl/mx/btl_mx_proc.c delete mode 100644 ompi/mca/btl/mx/btl_mx_proc.h delete mode 100644 ompi/mca/btl/mx/configure.m4 delete mode 100644 ompi/mca/common/mx/Makefile.am delete mode 100644 ompi/mca/common/mx/common_mx.c delete mode 100644 ompi/mca/common/mx/common_mx.h delete mode 100644 ompi/mca/common/mx/configure.m4 delete mode 100644 ompi/mca/mtl/mx/Makefile.am delete mode 100644 ompi/mca/mtl/mx/configure.m4 delete mode 100644 ompi/mca/mtl/mx/mtl_mx.c delete mode 100644 ompi/mca/mtl/mx/mtl_mx.h delete mode 100644 ompi/mca/mtl/mx/mtl_mx_cancel.c delete mode 100644 ompi/mca/mtl/mx/mtl_mx_component.c delete mode 100644 ompi/mca/mtl/mx/mtl_mx_endpoint.c delete mode 100644 ompi/mca/mtl/mx/mtl_mx_endpoint.h delete mode 100644 ompi/mca/mtl/mx/mtl_mx_probe.c delete mode 100644 ompi/mca/mtl/mx/mtl_mx_recv.c delete mode 100644 ompi/mca/mtl/mx/mtl_mx_request.h delete mode 100644 ompi/mca/mtl/mx/mtl_mx_send.c delete mode 100644 ompi/mca/mtl/mx/mtl_mx_types.h delete mode 100644 ompi/mca/mtl/mx/post_configure.sh diff --git a/ompi/mca/btl/mx/Makefile.am b/ompi/mca/btl/mx/Makefile.am deleted file mode 100644 index f241064859..0000000000 --- a/ompi/mca/btl/mx/Makefile.am +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -AM_CPPFLAGS = $(btl_mx_CPPFLAGS) - -btl_mx_sources = btl_mx.c btl_mx.h btl_mx_component.c btl_mx_endpoint.c \ - btl_mx_endpoint.h btl_mx_frag.c btl_mx_frag.h btl_mx_proc.c btl_mx_proc.h - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_ompi_btl_mx_DSO -component_noinst = -component_install = mca_btl_mx.la -else -component_noinst = libmca_btl_mx.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_btl_mx_la_SOURCES = $(btl_mx_sources) -mca_btl_mx_la_LIBADD = \ - $(btl_mx_LIBS) \ - $(top_ompi_builddir)/ompi/mca/common/mx/libmca_common_mx.la -mca_btl_mx_la_LDFLAGS = -module -avoid-version $(btl_mx_LDFLAGS) - -noinst_LTLIBRARIES = $(component_noinst) -libmca_btl_mx_la_SOURCES = $(btl_mx_sources) -libmca_btl_mx_la_LIBADD = $(btl_mx_LIBS) -libmca_btl_mx_la_LDFLAGS = -module -avoid-version $(btl_mx_LDFLAGS) diff --git a/ompi/mca/btl/mx/btl_mx.c b/ompi/mca/btl/mx/btl_mx.c deleted file mode 100644 index 021599d0a7..0000000000 --- a/ompi/mca/btl/mx/btl_mx.c +++ /dev/null @@ -1,701 +0,0 @@ -/* - * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "opal/class/opal_bitmap.h" - -#if OPAL_ENABLE_FT_CR == 1 -#include "ompi/runtime/ompi_cr.h" -#endif - -#include "btl_mx.h" -#include "btl_mx_frag.h" -#include "btl_mx_proc.h" -#include "btl_mx_endpoint.h" -#include "opal/datatype/opal_convertor.h" -#include "opal/prefetch.h" - -/** - * - */ -int mca_btl_mx_add_procs( struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t** ompi_procs, - struct mca_btl_base_endpoint_t** peers, - opal_bitmap_t* reachable ) -{ - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl; - int i, rc; - - for( i = 0; i < (int) nprocs; i++ ) { - - struct ompi_proc_t* ompi_proc = ompi_procs[i]; - mca_btl_mx_proc_t* mx_proc; - mca_btl_base_endpoint_t* mx_endpoint; - - /** - * By default don't allow communications with self nor with any - * other processes on the same node. The BTL self and sm are - * supposed to take care of such communications. - */ - if( OPAL_PROC_ON_LOCAL_NODE(ompi_procs[i]->proc_flags) ) { - if( ompi_procs[i] == ompi_proc_local_proc ) { - if( 0 == mca_btl_mx_component.mx_support_self ) - continue; - } else { - if( 0 == mca_btl_mx_component.mx_support_sharedmem ) - continue; - } - } - if( NULL == (mx_proc = mca_btl_mx_proc_create(ompi_proc)) ) { - continue; - } - - OPAL_THREAD_LOCK(&mx_proc->proc_lock); - - /* The btl_proc datastructure is shared by all MX BTL - * instances that are trying to reach this destination. - * Cache the peer instance on the btl_proc. - */ - mx_endpoint = OBJ_NEW(mca_btl_mx_endpoint_t); - if(NULL == mx_endpoint) { - OPAL_THREAD_UNLOCK(&mx_proc->proc_lock); - return OMPI_ERR_OUT_OF_RESOURCE; - } - - mx_endpoint->endpoint_btl = mx_btl; - rc = mca_btl_mx_proc_insert( mx_proc, mx_endpoint ); - if( rc != OMPI_SUCCESS ) { - OBJ_RELEASE(mx_endpoint); - OBJ_RELEASE(mx_proc); - OPAL_THREAD_UNLOCK(&mx_proc->proc_lock); - continue; - } - opal_bitmap_set_bit(reachable, i); - OPAL_THREAD_UNLOCK(&mx_proc->proc_lock); - peers[i] = mx_endpoint; - } - - return OMPI_SUCCESS; -} - -int mca_btl_mx_del_procs( struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t** procs, - struct mca_btl_base_endpoint_t** peers ) -{ - opal_output( 0, "MX BTL delete procs\n" ); - /* TODO */ - return OMPI_SUCCESS; -} - - -/** - * Register callback function to support send/recv semantics - */ - -int mca_btl_mx_register( struct mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_module_recv_cb_fn_t cbfunc, - void* cbdata ) -{ - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl; - -#if 0 - if( (NULL != cbfunc) && ( 0 == mca_btl_mx_component.mx_use_unexpected) ) { -#endif - if( NULL != cbfunc ) { - mca_btl_mx_frag_t* frag = NULL; - mx_return_t mx_return; - mx_segment_t mx_segment; - int i; - - /* Post the receives if there is no unexpected handler */ - for( i = 0; i < mca_btl_mx_component.mx_max_posted_recv; i++ ) { - MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag); - if( NULL == frag ) { - opal_output( 0, "mca_btl_mx_register: unable to allocate more eager fragments\n" ); - if( 0 == i ) { - return OMPI_ERROR; - } - break; /* some fragments are already registered. Try to continue... */ - } - frag->base.des_dst = (mca_btl_base_segment_t*)frag->segment; - frag->base.des_dst_cnt = 1; - frag->base.des_src = NULL; - frag->base.des_src_cnt = 0; - frag->mx_frag_list = NULL; - frag->type = MCA_BTL_MX_RECV; - - mx_segment.segment_ptr = (void*)(frag+1); - mx_segment.segment_length = mx_btl->super.btl_eager_limit; - mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1, - 0x01ULL, BTL_MX_RECV_MASK, - frag, &(frag->mx_request) ); - if( MX_SUCCESS != mx_return ) { - opal_output( 0, "mca_btl_mx_register: mx_irecv failed with status %d (%s)\n", - mx_return, mx_strerror(mx_return) ); - MCA_BTL_MX_FRAG_RETURN( mx_btl, frag ); - return OMPI_ERROR; - } - } - } - - return OMPI_SUCCESS; -} - - -/** - * Allocate a segment. - * - * @param btl (IN) BTL module - * @param size (IN) Request segment size. - */ - -mca_btl_base_descriptor_t* mca_btl_mx_alloc( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - uint8_t order, - size_t size, - uint32_t flags) -{ - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl; - mca_btl_mx_frag_t* frag = NULL; - - MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag); - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - frag->segment[0].base.seg_len = - size <= mx_btl->super.btl_eager_limit ? - size : mx_btl->super.btl_eager_limit ; - frag->segment[0].base.seg_addr.pval = (void*)(frag+1); - frag->base.des_src = (mca_btl_base_segment_t*)frag->segment; - frag->base.des_src_cnt = 1; - frag->base.des_flags = flags; - frag->base.order = MCA_BTL_NO_ORDER; - - return (mca_btl_base_descriptor_t*)frag; -} - - -/** - * Return a segment - */ -int mca_btl_mx_free( struct mca_btl_base_module_t* btl, - mca_btl_base_descriptor_t* des ) -{ - mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)des; - - assert( MCA_BTL_MX_SEND == frag->type ); - MCA_BTL_MX_FRAG_RETURN(btl, frag); - - return OMPI_SUCCESS; -} - -/** - * Pack data and return a descriptor that can be - * used for send/put. - * - * @param btl (IN) BTL module - * @param peer (IN) BTL peer addressing - */ -mca_btl_base_descriptor_t* -mca_btl_mx_prepare_src( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags) -{ - mca_btl_mx_frag_t* frag = NULL; - struct iovec iov; - uint32_t iov_count = 1; - size_t max_data; - - max_data = btl->btl_eager_limit - reserve; - if( (*size) < max_data ) { - max_data = *size; - } - /* If the data is contiguous we can use directly the pointer - * to the user memory. - */ - if( 0 == opal_convertor_need_buffers(convertor) ) { - /** - * let the convertor figure out the correct pointer depending - * on the data layout - */ - iov.iov_base = NULL; - if( 0 == reserve ) { - MCA_BTL_MX_FRAG_ALLOC_USER(btl, frag); - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - max_data = *size; - frag->base.des_src_cnt = 1; - } else { - MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag); - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - frag->base.des_src_cnt = 2; - } - } else { - MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag); - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - frag->base.des_src_cnt = 1; - iov.iov_base = (void*)((unsigned char*)frag->segment[0].base.seg_addr.pval + reserve); - } - - iov.iov_len = max_data; - (void)opal_convertor_pack(convertor, &iov, &iov_count, &max_data ); - *size = max_data; - - if( 1 == frag->base.des_src_cnt ) { - frag->segment[0].base.seg_len = reserve + max_data; - if( 0 == reserve ) - frag->segment[0].base.seg_addr.pval = iov.iov_base; - } else { - frag->segment[0].base.seg_len = reserve; - frag->segment[1].base.seg_len = max_data; - frag->segment[1].base.seg_addr.pval = iov.iov_base; - } - frag->base.des_src = (mca_btl_base_segment_t*)frag->segment; - frag->base.des_flags = flags; - frag->base.order = MCA_BTL_NO_ORDER; - - return &frag->base; -} - -/** - * Prepare a descriptor for send/rdma using the supplied - * convertor. If the convertor references data that is contigous, - * the descriptor may simply point to the user buffer. Otherwise, - * this routine is responsible for allocating buffer space and - * packing if required. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL peer addressing - * @param convertor (IN) Data type convertor - * @param reserve (IN) Additional bytes requested by upper layer to precede user data - * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) - */ - -mca_btl_base_descriptor_t* mca_btl_mx_prepare_dst( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_mpool_base_registration_t* registration, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags) -{ - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl; - mca_btl_mx_frag_t* frag = NULL; - mx_return_t mx_return; - mx_segment_t mx_segment; - - MCA_BTL_MX_FRAG_ALLOC_USER(btl, frag); - if( OPAL_UNLIKELY(NULL == frag) ) { - return NULL; - } - - frag->segment[0].base.seg_len = *size; - opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment[0].base.seg_addr.pval) ); - frag->segment[0].key = (uint64_t)(intptr_t)frag; - - mx_segment.segment_ptr = frag->segment[0].base.seg_addr.pval; - mx_segment.segment_length = frag->segment[0].base.seg_len; - - mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1, - frag->segment[0].key, BTL_MX_PUT_MASK, - NULL, &(frag->mx_request) ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "Fail to re-register a fragment with the MX NIC ...\n" ); - MCA_BTL_MX_FRAG_RETURN( btl, frag ); - return NULL; - } - -#ifdef HAVE_MX_FORGET - { - mx_return = mx_forget( mx_btl->mx_endpoint, &(frag->mx_request) ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_forget failed in mca_btl_mx_prepare_dst with error %d (%s)\n", - mx_return, mx_strerror(mx_return) ); - return NULL; - } - } -#endif - - /* Allow the fragment to be recycled using the mca_btl_mx_free function */ - frag->type = MCA_BTL_MX_SEND; - frag->base.des_dst = (mca_btl_base_segment_t*)frag->segment; - frag->base.des_dst_cnt = 1; - frag->base.des_flags = flags; - frag->base.order = MCA_BTL_NO_ORDER; - - return &frag->base; -} - - -/** - * Initiate an asynchronous put. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transferred - */ -static int mca_btl_mx_put( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_btl_base_descriptor_t* descriptor ) -{ - mca_btl_mx_segment_t *src_seg = (mca_btl_mx_segment_t *) descriptor->des_src; - mca_btl_mx_segment_t *dst_seg = (mca_btl_mx_segment_t *) descriptor->des_dst; - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl; - mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor; - mx_segment_t mx_segment[2]; - mx_return_t mx_return; - uint32_t i = 0; - - if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) { - if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status ) - return OMPI_ERROR; - if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status ) - return OMPI_ERR_OUT_OF_RESOURCE; - if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) ) - return OMPI_ERROR; - } - - frag->endpoint = endpoint; - frag->type = MCA_BTL_MX_SEND; - descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; - - do { - mx_segment[i].segment_ptr = src_seg[i].base.seg_addr.pval; - mx_segment[i].segment_length = src_seg[i].base.seg_len; - } while (++i < descriptor->des_src_cnt); - - mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt, - endpoint->mx_peer_addr, dst_seg->key, frag, - &frag->mx_request ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) ); - return OMPI_ERROR; - } - return OMPI_SUCCESS; -} - - -/** - * Initiate an inline send to the peer. If failure then return a descriptor. - * - * @param btl (IN) BTL module - * @param peer (IN) BTL peer addressing - */ -static int mca_btl_mx_sendi( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct opal_convertor_t* convertor, - void* header, - size_t header_size, - size_t payload_size, - uint8_t order, - uint32_t flags, - mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t** descriptor ) -{ - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl; - size_t max_data; - - if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) { - if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status ) - return OMPI_ERROR; - if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status ) - return OMPI_ERR_OUT_OF_RESOURCE; - if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) ) - return OMPI_ERROR; - } - - if( !opal_convertor_need_buffers(convertor) ) { - uint32_t mx_segment_count = 0; - uint64_t tag64 = 0x01ULL | (((uint64_t)tag) << 8); - mx_return_t mx_return; - mx_request_t mx_request; - mx_segment_t mx_segments[2], *mx_segment = mx_segments; - - if( 0 != header_size ) { - mx_segment->segment_ptr = header; - mx_segment->segment_length = header_size; - mx_segment++; - mx_segment_count++; - } - if( 0 != payload_size ) { - struct iovec iov; - uint32_t iov_count = 1; - - iov.iov_base = NULL; - iov.iov_len = payload_size; - - (void)opal_convertor_pack( convertor, &iov, &iov_count, &max_data ); - assert( max_data == payload_size ); - - mx_segment->segment_ptr = iov.iov_base; - mx_segment->segment_length = max_data; - mx_segment_count++; - } - - mx_return = mx_isend( mx_btl->mx_endpoint, mx_segments, mx_segment_count, - endpoint->mx_peer_addr, tag64, NULL, &mx_request ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) ); - return OMPI_ERROR; - } -#ifdef HAVE_MX_FORGET - { - uint32_t mx_result; - mx_return = mx_ibuffered( mx_btl->mx_endpoint, &mx_request, &mx_result ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_ibuffered failed with error %d (%s)\n", - mx_return, mx_strerror(mx_return) ); - return OMPI_SUCCESS; - } - if( mx_result ) { - mx_return = mx_forget( mx_btl->mx_endpoint, &mx_request ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_forget failed with error %d (%s)\n", - mx_return, mx_strerror(mx_return) ); - } - } - return OMPI_SUCCESS; - } -#endif - } - /* No optimization on this path. Just allocate a descriptor and return it - * to the user. - */ - *descriptor = mca_btl_mx_alloc( btl, endpoint, order, - header_size + payload_size, flags ); - return OMPI_ERR_RESOURCE_BUSY; -} - -/** - * Initiate an asynchronous send. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transfered - * @param tag (IN) The tag value used to notify the peer. - */ - -int mca_btl_mx_send( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - struct mca_btl_base_descriptor_t* descriptor, - mca_btl_base_tag_t tag ) - -{ - mca_btl_mx_segment_t *src_seg = (mca_btl_mx_segment_t *) descriptor->des_src; - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl; - mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor; - mx_segment_t mx_segment[2]; - mx_return_t mx_return; - uint64_t total_length = 0, tag64; - uint32_t i = 0; - int btl_ownership = (descriptor->des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - - if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) { - if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status ) - return OMPI_ERROR; - if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status ) - return OMPI_ERR_OUT_OF_RESOURCE; - if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) ) - return OMPI_ERROR; - } - - frag->endpoint = endpoint; - frag->type = MCA_BTL_MX_SEND; - - do { - mx_segment[i].segment_ptr = src_seg[i].base.seg_addr.pval; - mx_segment[i].segment_length = src_seg[i].base.seg_len; - total_length += src_seg[i].base.seg_len; - } while (++i < descriptor->des_src_cnt); - - tag64 = 0x01ULL | (((uint64_t)tag) << 8); - mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt, - endpoint->mx_peer_addr, tag64, frag, &frag->mx_request ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) ); - return OMPI_ERROR; - } - -#ifdef HAVE_MX_FORGET - { - uint32_t mx_result; - mx_return = mx_ibuffered( mx_btl->mx_endpoint, &(frag->mx_request), &mx_result ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_ibuffered failed with error %d (%s)\n", - mx_return, mx_strerror(mx_return) ); - frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; - return OMPI_ERROR; - } - if( mx_result ) { - mx_return = mx_forget( mx_btl->mx_endpoint, &(frag->mx_request) ); - if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) { - opal_output( 0, "mx_forget failed with error %d (%s)\n", - mx_return, mx_strerror(mx_return) ); - frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; - return OMPI_SUCCESS; - } - - if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) { - frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint, - &(frag->base), OMPI_SUCCESS); - } - if( btl_ownership ) { - MCA_BTL_MX_FRAG_RETURN( mx_btl, frag ); - } - return 1; - } - } -#endif - if( 2048 > total_length ) { - mx_status_t mx_status; - uint32_t mx_result; - - /* let's check for completness */ - mx_return = mx_test( mx_btl->mx_endpoint, &(frag->mx_request), - &mx_status, &mx_result ); - if( OPAL_LIKELY(MX_SUCCESS == mx_return) ) { - if( mx_result ) { - if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) { - frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint, - &(frag->base), OMPI_SUCCESS); - } - if( btl_ownership ) { - MCA_BTL_MX_FRAG_RETURN( mx_btl, frag ); - } - return 1; - } - } - } - frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; - - return OMPI_SUCCESS; -} - - -/* - * Cleanup/release module resources. - */ - -int mca_btl_mx_finalize( struct mca_btl_base_module_t* btl ) -{ - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl; - - if( NULL != mx_btl->mx_endpoint ) - mx_close_endpoint(mx_btl->mx_endpoint); - - OBJ_DESTRUCT( &mx_btl->mx_lock ); - OBJ_DESTRUCT( &mx_btl->mx_peers ); - free(mx_btl); - return OMPI_SUCCESS; -} - - -#if OPAL_ENABLE_FT_CR == 0 -int mca_btl_mx_ft_event(int state) { - return OMPI_SUCCESS; -} -#else -int mca_btl_mx_ft_event(int state) { - mca_btl_mx_module_t* mx_btl; - int i; - - if(OPAL_CRS_CHECKPOINT == state) { - /* Continue must reconstruct the routes (including modex), since we - * have to tear down the devices completely. - * We have to do this because the MX driver can be checkpointed, but - * cannot be restarted with BLCR due to an mmap problem. If we do not - * close MX then BLCR throws the following error in /var/log/messages: - * kernel: do_mmap(, 00002aaab0aac000, 0000000000400000, ...) failed: ffffffffffffffff - * kernel: vmadump: mmap failed: /dev/mx0 - * kernel: blcr: thaw_threads returned error, aborting. -1 - * JJH: It may be possible to, instead of restarting the entire driver, just reconnect endpoints - */ - orte_cr_continue_like_restart = true; - - for( i = 0; i < mca_btl_mx_component.mx_num_btls; i++ ) { - mx_btl = mca_btl_mx_component.mx_btls[i]; - - if( NULL != mx_btl->mx_endpoint ) { - mx_close_endpoint(mx_btl->mx_endpoint); - mx_btl->mx_endpoint = NULL; - } - } - } - else if(OPAL_CRS_CONTINUE == state) { - ; - } - else if(OPAL_CRS_RESTART == state) { - ; - } - else if(OPAL_CRS_TERM == state ) { - ; - } - else { - ; - } - - return OMPI_SUCCESS; -} -#endif /* OPAL_ENABLE_FT_CR */ - -mca_btl_mx_module_t mca_btl_mx_module = { - { - &mca_btl_mx_component.super, - 0, /* max size of first fragment */ - 0, /* min send fragment size */ - 0, /* max send fragment size */ - 0, /* btl_rdma_pipeline_send_length */ - 0, /* btl_rdma_pipeline_frag_size */ - 0, /* btl_min_rdma_pipeline_size */ - 0, /* exclusivity */ - 0, /* latency */ - 0, /* bandwidth */ - 0, /* segment size */ - MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT, /* flags */ - mca_btl_mx_add_procs, - mca_btl_mx_del_procs, - mca_btl_mx_register, - mca_btl_mx_finalize, - mca_btl_mx_alloc, - mca_btl_mx_free, - mca_btl_mx_prepare_src, - mca_btl_mx_prepare_dst, - mca_btl_mx_send, - mca_btl_mx_sendi, /* send immediate */ - mca_btl_mx_put, /* put */ - NULL, /* get */ - mca_btl_base_dump, - NULL, /* mpool */ - NULL, /* register error */ - mca_btl_mx_ft_event - } -}; diff --git a/ompi/mca/btl/mx/btl_mx.h b/ompi/mca/btl/mx/btl_mx.h deleted file mode 100644 index 9e2dc4d891..0000000000 --- a/ompi/mca/btl/mx/btl_mx.h +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2009 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -/** - * @file - */ -#ifndef MCA_PTL_MX_H -#define MCA_PTL_MX_H - -#include "ompi_config.h" -#include -#include - -/* Open MPI includes */ -#include "ompi/class/ompi_free_list.h" -#include "opal/mca/event/event.h" -#include "ompi/mca/btl/btl.h" -#include "ompi/mca/btl/base/base.h" -#include "ompi/mca/mpool/mpool.h" - -#include - -#ifdef HAVE_MX_EXTENSIONS_H -#include -#endif /* HAVE_MX_EXTENSIONS_H */ - -BEGIN_C_DECLS - -/** - * The mask used for receive and for the PUT protocol - */ -#define BTL_MX_RECV_MASK 0x00000000000000ffULL -#define BTL_MX_PUT_MASK 0xffffffffffffffffULL - -/** - * MX BTL component. - */ - -struct mca_btl_mx_component_t { - mca_btl_base_component_2_0_0_t super; /**< base BTL component */ - - int32_t mx_num_btls; - int32_t mx_max_btls; - /**< number of hcas available to the MX component */ - - struct mca_btl_mx_module_t** mx_btls; - /**< array of available BTL modules */ - - int32_t mx_free_list_num; - /**< initial size of free lists */ - - int32_t mx_free_list_max; - /**< maximum size of free lists */ - - int32_t mx_max_posted_recv; - /**< number of posted receives on each NIC */ - - int32_t mx_free_list_inc; - /**< number of elements to alloc when growing free lists */ - - int32_t mx_support_sharedmem; - /**< true if we want to activate the MX support for shared memory */ - int32_t mx_support_self; - /**< true if we want to activate the MX support for self communications */ - int32_t mx_bonding; - /**< true if MX is in charge of doing the device bonding */ - int32_t mx_use_unexpected; - /**< true if Open MPI is allowed to register an unexpected handler with the MX library */ - - opal_list_t mx_procs; /**< list of mx proc structures */ - - int32_t mx_filter; - int32_t mx_timeout; - int32_t mx_connection_retries; - - ompi_free_list_t mx_send_eager_frags; /**< free list of mx eager send fragments */ - ompi_free_list_t mx_send_user_frags; /**< free list of mx user send fragments */ - - opal_mutex_t mx_lock; /**< lock for accessing module state */ - -#if MX_HAVE_MAPPER_STATE - char* mx_if_include; /**< include the following NICs */ - char* mx_if_exclude; /**< Exclude the following NICs. These - * values are based on the last 6 - * digits in hexadecimal of the MAC - * address of the mapper. - */ -#endif /* MX_HAVE_MAPPER_STATE */ -}; -typedef struct mca_btl_mx_component_t mca_btl_mx_component_t; - -OMPI_MODULE_DECLSPEC extern mca_btl_mx_component_t mca_btl_mx_component; - -/** - * BTL Module Interface. - * Each BTL correspond to a high level vision of a network interface. The - * current version of the MX BTL is not able to handle stripping of the - * messages by itself. Therefore, it rely on the PML layer for that. - */ -struct mca_btl_mx_module_t { - mca_btl_base_module_t super; /**< base BTL interface */ - mx_endpoint_t mx_endpoint; /**< local MX endpoint */ - mx_endpoint_addr_t mx_endpoint_addr; /**< local MX endpoint address */ - uint32_t mx_unique_network_id; /**< unique identifier for this BTL, - * based on the MAC address of the - * mapper used to route messages. - */ - opal_list_t mx_peers; /**< list of peers */ - - int32_t mx_posted_request; /**< number of posted MX request */ - opal_mutex_t mx_lock; /**< lock for accessing module state */ -}; -typedef struct mca_btl_mx_module_t mca_btl_mx_module_t; -extern mca_btl_mx_module_t mca_btl_mx_module; - -/** - * MX component initialization. - * - * @param num_btl_modules (OUT) Number of BTLs returned in BTL array. - * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE) - * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE) - */ -extern mca_btl_base_module_t** mca_btl_mx_component_init( - int *num_btl_modules, - bool allow_multi_user_threads, - bool have_hidden_threads -); - - -/** - * MX component progress. - */ -extern int mca_btl_mx_component_progress(void); - - - -/** - * Cleanup any resources held by the BTL. - * - * @param btl BTL instance. - * @return OMPI_SUCCESS or error status on failure. - */ - -extern int mca_btl_mx_finalize( - struct mca_btl_base_module_t* btl -); - - -/** - * PML->BTL notification of change in the process list. - * - * @param btl (IN) - * @param nprocs (IN) Number of processes - * @param procs (IN) Set of processes - * @param peers (OUT) Set of (optional) peer addressing info. - * @param peers (IN/OUT) Set of processes that are reachable via this BTL. - * @return OMPI_SUCCESS or error status on failure. - * - */ - -extern int mca_btl_mx_add_procs( - struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t **procs, - struct mca_btl_base_endpoint_t** peers, - opal_bitmap_t* reachable -); - -/** - * PML->BTL notification of change in the process list. - * - * @param btl (IN) BTL instance - * @param nproc (IN) Number of processes. - * @param procs (IN) Set of processes. - * @param peers (IN) Set of peer data structures. - * @return Status indicating if cleanup was successful - * - */ - -extern int mca_btl_mx_del_procs( - struct mca_btl_base_module_t* btl, - size_t nprocs, - struct ompi_proc_t **procs, - struct mca_btl_base_endpoint_t** peers -); - - -/** - * Initiate an asynchronous send. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL addressing information - * @param descriptor (IN) Description of the data to be transfered - * @param tag (IN) The tag value used to notify the peer. - */ - -extern int mca_btl_mx_send( - struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* btl_peer, - struct mca_btl_base_descriptor_t* descriptor, - mca_btl_base_tag_t tag -); - - -/** - * Register a callback function that is called on receipt - * of a fragment. - * - * @param btl (IN) BTL module - * @return Status indicating if registration was successful - * - */ - -extern int mca_btl_mx_register( - struct mca_btl_base_module_t* btl, - mca_btl_base_tag_t tag, - mca_btl_base_module_recv_cb_fn_t cbfunc, - void* cbdata); - -/** - * Allocate a descriptor with a segment of the requested size. - * Note that the BTL layer may choose to return a smaller size - * if it cannot support the request. - * - * @param btl (IN) BTL module - * @param size (IN) Request segment size. - */ - -mca_btl_base_descriptor_t* mca_btl_mx_alloc( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* endpoint, - uint8_t order, - size_t size, - uint32_t flags); - - -/** - * Return a segment allocated by this BTL. - * - * @param btl (IN) BTL module - * @param descriptor (IN) Allocated descriptor. - */ - -int mca_btl_mx_free( struct mca_btl_base_module_t* btl, - mca_btl_base_descriptor_t* des ); - - -/** - * Prepare a descriptor for send/rdma using the supplied - * convertor. If the convertor references data that is contigous, - * the descriptor may simply point to the user buffer. Otherwise, - * this routine is responsible for allocating buffer space and - * packing if required. - * - * @param btl (IN) BTL module - * @param endpoint (IN) BTL peer addressing - * @param convertor (IN) Data type convertor - * @param reserve (IN) Additional bytes requested by upper layer to precede user data - * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) - */ -mca_btl_base_descriptor_t* -mca_btl_mx_prepare_src( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* peer, - struct mca_mpool_base_registration_t*, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags); - -mca_btl_base_descriptor_t* -mca_btl_mx_prepare_dst( struct mca_btl_base_module_t* btl, - struct mca_btl_base_endpoint_t* peer, - struct mca_mpool_base_registration_t*, - struct opal_convertor_t* convertor, - uint8_t order, - size_t reserve, - size_t* size, - uint32_t flags); - -/** - * Fault Tolerance Event Notification Function - * @param state Checkpoint Stae - * @return OMPI_SUCCESS or failure status - */ -int mca_btl_mx_ft_event(int state); - -END_C_DECLS - -#endif diff --git a/ompi/mca/btl/mx/btl_mx_component.c b/ompi/mca/btl/mx/btl_mx_component.c deleted file mode 100644 index f6e3626372..0000000000 --- a/ompi/mca/btl/mx/btl_mx_component.c +++ /dev/null @@ -1,764 +0,0 @@ -/* - * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include "opal/prefetch.h" -#include "opal/util/opal_environ.h" -#include "ompi/constants.h" - -#include "ompi/runtime/ompi_module_exchange.h" -#include "ompi/mca/btl/base/btl_base_error.h" -#include "ompi/mca/common/mx/common_mx.h" - -#include "btl_mx.h" -#include "btl_mx_frag.h" -#include "btl_mx_endpoint.h" - -#if MX_HAVE_MAPPER_STATE -#include "mx_io.h" -#include "mx_internals/mx__fops.h" -#include "mx_internals/mx__driver_interface.h" -#endif /* MX_HAVE_MAPPER_STATE */ - -static int mca_btl_mx_component_register(void); -static int mca_btl_mx_component_open(void); -static int mca_btl_mx_component_close(void); - - -mca_btl_mx_component_t mca_btl_mx_component = { - { - /* First, the mca_base_component_t struct containing meta information - about the component itself */ - - { - MCA_BTL_BASE_VERSION_2_0_0, - - "mx", /* MCA component name */ - OMPI_MAJOR_VERSION, /* MCA component major version */ - OMPI_MINOR_VERSION, /* MCA component minor version */ - OMPI_RELEASE_VERSION, /* MCA component release version */ - mca_btl_mx_component_open, /* component open */ - mca_btl_mx_component_close, /* component close */ - NULL, /* component query */ - mca_btl_mx_component_register, /* component register */ - }, - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - }, - - mca_btl_mx_component_init, - mca_btl_mx_component_progress, - } -}; - -static int mca_btl_mx_component_verify(void) -{ - if( 0 >= mca_btl_mx_component.mx_bonding ) { - char* value = getenv("MX_BONDING"); - if( NULL == value ) { - mca_btl_mx_component.mx_bonding = 1; - } else { - mca_btl_mx_component.mx_bonding = atoi(value); - if( 0 >= mca_btl_mx_component.mx_bonding ) - mca_btl_mx_component.mx_bonding = 1; - } - } else if( 1 != mca_btl_mx_component.mx_bonding ) { - char value[8]; - snprintf( value, 8, "%d\n", mca_btl_mx_component.mx_bonding ); - opal_setenv( "MX_BONDING", value, true, &environ ); - } - - return mca_btl_base_param_verify(&mca_btl_mx_module.super); -} - -static int mca_btl_mx_component_register(void) -{ - mca_btl_mx_component.mx_max_btls = 8; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "max_btls", - "Maximum number of accepted Myrinet cards", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_max_btls); - mca_btl_mx_component.mx_timeout = MX_INFINITE; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "timeout", - "Timeout for connections", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_timeout); - mca_btl_mx_component.mx_connection_retries = 20; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "retries", - "Number of retries for each new connection before considering the peer as unreacheable", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_connection_retries); - mca_btl_mx_component.mx_filter = 0xdeadbeef; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "filter", - "Unique ID for the application (used to connect to the peers)", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_filter); - mca_btl_mx_component.mx_support_self = 0; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "self", - "Enable the MX support for self communications", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_support_self); - mca_btl_mx_component.mx_support_sharedmem = 0; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "shared_mem", - "Enable the MX support for shared memory", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_support_sharedmem); - mca_btl_mx_component.mx_bonding = 1; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "bonding", - "Integrate MX library bonding. Less than 0 is system default, everything else will set the MX_BONDING to the value.", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_bonding); - -#ifdef HAVE_MX_REGISTER_UNEXP_HANDLER - mca_btl_mx_component.mx_use_unexpected = 0; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "register_unexp", - "Enable the MX support for the unexpected request handler (Open MPI matching)", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_use_unexpected); -#endif /* HAVE_MX_REGISTER_UNEXP_HANDLER */ - mca_btl_mx_component.mx_free_list_num = 8; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "free_list_num", - "Number of allocated default request", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_free_list_num); - mca_btl_mx_component.mx_free_list_inc = 32; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "free_list_inc", - "Number of request we allocate each time we miss some", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_free_list_inc); - mca_btl_mx_component.mx_free_list_max = 1024; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "free_list_max", - "Maximum number of request this device is allowed to allocate", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_free_list_max); - - mca_btl_mx_component.mx_max_posted_recv = 16; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "max_posted_recv", - "Number of received posted in advance. Increasing this number for communication " - "bound application can lead to visible improvement in performances", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_max_posted_recv); - -#if MX_HAVE_MAPPER_STATE - mca_btl_mx_component.mx_if_include = NULL; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "if_include", - "Myrinet card to use (last 6 digits from the mapper MAC)", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_if_include); - mca_btl_mx_component.mx_if_exclude = NULL; - (void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "if_exclude", - "Myrinet card to avoid (last 6 digits from the mapper MAC)", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_mx_component.mx_if_exclude); -#endif /* MX_HAVE_MAPPER_STATE */ - - mca_btl_mx_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT; - if( mca_btl_mx_component.mx_use_unexpected ) { - mca_btl_mx_module.super.btl_eager_limit = 1024; - mca_btl_mx_module.super.btl_rndv_eager_limit = 1024; - } else { - mca_btl_mx_module.super.btl_eager_limit = 4*1024; - mca_btl_mx_module.super.btl_rndv_eager_limit = 4*1024; - } - mca_btl_mx_module.super.btl_max_send_size = 32*1024; - mca_btl_mx_module.super.btl_rdma_pipeline_send_length = 256*1024; - mca_btl_mx_module.super.btl_rdma_pipeline_frag_size = 8*1024*1024; - mca_btl_mx_module.super.btl_min_rdma_pipeline_size = 0; - mca_btl_mx_module.super.btl_flags = (MCA_BTL_FLAGS_SEND_INPLACE | - MCA_BTL_FLAGS_PUT | - MCA_BTL_FLAGS_SEND | - MCA_BTL_FLAGS_RDMA_MATCHED); - mca_btl_mx_module.super.btl_seg_size = sizeof (mca_btl_mx_segment_t); - mca_btl_mx_module.super.btl_bandwidth = 2000; - mca_btl_mx_module.super.btl_latency = 5; - - mca_btl_base_param_register(&mca_btl_mx_component.super.btl_version, - &mca_btl_mx_module.super); - - return mca_btl_mx_component_verify(); -} - - -/* - * Called by MCA framework to open the component, registers - * component parameters. - */ - -static int mca_btl_mx_component_open(void) -{ - int ret; - - /* initialize state */ - mca_btl_mx_component.mx_num_btls = 0; - mca_btl_mx_component.mx_btls = NULL; - mca_btl_mx_component.mx_use_unexpected = 0; - - ret = mca_btl_mx_component_verify(); - if (OMPI_SUCCESS != ret) { - return ret; - } - - /* initialize objects */ - OBJ_CONSTRUCT(&mca_btl_mx_component.mx_procs, opal_list_t); - if( 0 == mca_btl_mx_component.mx_support_sharedmem ) - opal_setenv( "MX_DISABLE_SHMEM", "1", true, &environ ); - if( 0 == mca_btl_mx_component.mx_support_self ) - opal_setenv( "MX_DISABLE_SELF", "1", true, &environ ); - /* Force the long pipeline (up to 4Kb fragments) */ - opal_setenv( "MX_PIPELINE_LOG", "0", true, &environ ); - - return OMPI_SUCCESS; -} - -/* - * component cleanup - sanity checking of queue lengths - */ - -static int mca_btl_mx_component_close(void) -{ - - if(OMPI_SUCCESS != ompi_common_mx_finalize()) { - return OMPI_ERROR; - } - - if( NULL == mca_btl_mx_component.mx_btls ) - return OMPI_SUCCESS; - - /* release resources */ - OBJ_DESTRUCT(&mca_btl_mx_component.mx_procs); - OBJ_DESTRUCT(&mca_btl_mx_component.mx_send_eager_frags); - OBJ_DESTRUCT(&mca_btl_mx_component.mx_send_user_frags); - OBJ_DESTRUCT(&mca_btl_mx_component.mx_procs); - OBJ_DESTRUCT(&mca_btl_mx_component.mx_lock); - -#if MX_HAVE_MAPPER_STATE - if( NULL != mca_btl_mx_component.mx_if_include ) { - free( mca_btl_mx_component.mx_if_include ); - mca_btl_mx_component.mx_if_include = NULL; - } - if( NULL != mca_btl_mx_component.mx_if_exclude ) { - free( mca_btl_mx_component.mx_if_exclude ); - mca_btl_mx_component.mx_if_exclude = NULL; - } -#endif /* MX_HAVE_MAPPER_STATE */ - return OMPI_SUCCESS; -} - -static int __counter = 0; -#ifdef HAVE_MX_REGISTER_UNEXP_HANDLER - -/** - * In order to avoid useless memcpy, the unexpected handler will be called - * by the MX library before doing any match in the MX internal queues. Here - * we have a chance to match the message using our own matching logic from - * the PML. If the match is realized, we will return MX_RECV_FINISHED (the - * MX request will vanish in the MX library). If the match do not succeed - * MX_RECV_CONTINUE have to be returned and the MX library will do the - * match itself. - */ -static mx_unexp_handler_action_t -mca_btl_mx_unexpected_handler( void *context, mx_endpoint_addr_t source, - uint64_t match_value, uint32_t length, - void * data_if_available ) -{ - mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)context; - mca_btl_active_message_callback_t* reg; - mca_btl_base_tag_t tag; - mca_btl_base_descriptor_t descriptor; - mca_btl_base_segment_t segment; - - if( 0 == __counter ) { - return MX_RECV_CONTINUE; - } - /*opal_output( 0, "Get unexpected handler context %p source %lld match_value %lld\n" - "\tlength %d data %p\n", context, source.stuff[0], match_value, length, - data_if_available );*/ - if( !(0x01 & match_value) ) { - return MX_RECV_CONTINUE; - } - - tag = (match_value >> 8) & 0xff; - reg = mca_btl_base_active_message_trigger + tag; - - segment.seg_addr.pval = data_if_available; - segment.seg_len = length; - descriptor.des_dst = &segment; - descriptor.des_dst_cnt = 1; - reg->cbfunc( &(mx_btl->super), tag, &descriptor, reg->cbdata ); - - return MX_RECV_FINISHED; -} -#endif /* HAVE_MX_REGISTER_UNEXP_HANDLER */ - -/* - * Create and initialize an MX BTL module, where each module - * represents a specific NIC or a specific bonded set of NICS. - */ -static mca_btl_mx_module_t* mca_btl_mx_create(uint32_t board_num) -{ - mca_btl_mx_module_t* mx_btl; - mx_endpoint_t mx_endpoint; - mx_endpoint_addr_t mx_endpoint_addr; - mx_return_t status; - uint32_t endpoint_id, mx_unique_network_id = 0; - uint64_t nic_id; - - /* open local endpoint */ - status = mx_open_endpoint( board_num, MX_ANY_ENDPOINT, - mca_btl_mx_component.mx_filter, - NULL, 0, &mx_endpoint); - if(status != MX_SUCCESS) { - opal_output( 0, "mca_btl_mx_init: mx_open_endpoint() failed with status %d (%s)\n", - status, mx_strerror(status) ); - return NULL; - } - - /* query the endpoint address */ - if((status = mx_get_endpoint_addr( mx_endpoint, - &mx_endpoint_addr)) != MX_SUCCESS) { - opal_output( 0, "mca_btl_mx_init: mx_get_endpoint_addr() failed with status %d (%s)\n", - status, mx_strerror(status) ); - mx_close_endpoint(mx_endpoint); - return NULL; - } - - status = mx_decompose_endpoint_addr(mx_endpoint_addr, &nic_id, &endpoint_id); - if( MX_SUCCESS != status ) { - opal_output( 0, "mca_btl_mx_init: mx_decompose_endpoint_addr() failed with status %d (%s)\n", - status, mx_strerror(status) ); - mx_close_endpoint(mx_endpoint); - return NULL; - } - status = mx_nic_id_to_board_number(nic_id, &board_num); - if( MX_SUCCESS != status ) { - opal_output( 0, "mca_btl_mx_init: mx_nic_id_to_board_number() failed with status %d (%s)\n", - status, mx_strerror(status) ); - mx_close_endpoint(mx_endpoint); - return NULL; - } - -#if MX_HAVE_MAPPER_STATE - { - mx_endpt_handle_t endp_handle; - mx_mapper_state_t ms; - char mapper_mac[7], *where; - - status = mx_open_board( board_num, &endp_handle ); - if( MX_SUCCESS != status ) { - opal_output( 0, "Unable to open board %d: %s\n", board_num, mx_strerror(status) ); - mx_close_endpoint(mx_endpoint); - return NULL; - } - - ms.board_number = board_num; - ms.iport = 0; - status = mx__get_mapper_state( endp_handle, &ms ); - if( MX_SUCCESS != status ) { - opal_output( 0, "get_mapper_state failed for board %d: %s\n", - board_num, mx_strerror(status) ); - mx_close_endpoint(mx_endpoint); - return NULL; - } - /* Keep the first 4 bytes for the network speed */ - mx_unique_network_id = ((ms.mapper_mac[3] << 16) + - (ms.mapper_mac[4] << 8) + - (ms.mapper_mac[5])); - - /* Try to figure out if we are allowed to use this network */ - snprintf( mapper_mac, 7, "%6x", mx_unique_network_id ); - - if( (NULL != mca_btl_mx_component.mx_if_exclude) && - (NULL != (where = strstr(mca_btl_mx_component.mx_if_exclude, mapper_mac))) ) { - mx_close_endpoint(mx_endpoint); - return NULL; - } - else if( (NULL != mca_btl_mx_component.mx_if_include) && - (NULL == (where = strstr(mca_btl_mx_component.mx_if_include, mapper_mac))) ) { - mx_close_endpoint(mx_endpoint); - return NULL; - } - } -#endif /* MX_HAVE_MAPPER_STATE */ - - mx_btl = malloc(sizeof(mca_btl_mx_module_t)); - if( NULL == mx_btl ) { - opal_output( 0, "mca_btl_mx_init: unable to allocate %lu bytes of memory\n", - sizeof(mca_btl_mx_module_t) ); - mx_close_endpoint(mx_endpoint); - return NULL; - } - - /* copy over default settings */ - memcpy( mx_btl, &mca_btl_mx_module, sizeof(mca_btl_mx_module_t) ); - OBJ_CONSTRUCT( &mx_btl->mx_peers, opal_list_t ); - OBJ_CONSTRUCT( &mx_btl->mx_lock, opal_mutex_t ); - mx_btl->mx_endpoint = mx_endpoint; - mx_btl->mx_endpoint_addr = mx_endpoint_addr; - - mx_btl->super.btl_bandwidth = 2000; /* whatever */ - mx_btl->super.btl_latency = 10; -#if defined(MX_HAS_NET_TYPE) - { - int value, board = board_num; - if( (status = mx_get_info( mx_btl->mx_endpoint, MX_LINE_SPEED, - &board, sizeof(board), - &value, sizeof(int))) != MX_SUCCESS ) { - opal_output( 0, "mx_get_info(MX_LINE_SPEED) failed with status %d (%s)\n", - status, mx_strerror(status) ); - } else { - if( MX_SPEED_2G == value ) { - mx_unique_network_id |= 0xaa000000; - mx_btl->super.btl_bandwidth = 2000; - mx_btl->super.btl_latency = 5; - } else if( MX_SPEED_10G == value ) { - mx_unique_network_id |= 0xbb000000; - mx_btl->super.btl_bandwidth = 10000; - mx_btl->super.btl_latency = 3; - } else { - mx_unique_network_id |= 0xcc000000; - } - } - } -#endif /* defined(MX_HAS_NET_TYPE) */ - mx_btl->super.btl_bandwidth *= mca_btl_mx_component.mx_bonding; - mx_btl->mx_unique_network_id = mx_unique_network_id; - -#ifdef HAVE_MX_REGISTER_UNEXP_HANDLER - if( mca_btl_mx_component.mx_use_unexpected ) { - status = mx_register_unexp_handler( mx_btl->mx_endpoint, mca_btl_mx_unexpected_handler, - (void*)mx_btl ); - if( MX_SUCCESS != status ) { - opal_output( 0, "mca_btl_mx_init: mx_register_unexp_handler() failed with status %d (%s)\n", - status, mx_strerror(status) ); - /* switch to a mode without the unexpected handler */ - mca_btl_mx_component.mx_use_unexpected = 0; - } - } -#endif /* HAVE_MX_REGISTER_UNEXP_HANDLER */ - return mx_btl; -} - -/* - * MX component initialization: - * - check if MX can be initialized. - * - and construct all static objects. - */ - -mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules, - bool enable_progress_threads, - bool enable_mpi_threads) -{ - mca_btl_base_module_t** btls; - mx_return_t status; - uint32_t count; - int32_t i; - mca_btl_mx_addr_t *mx_addrs; - - *num_btl_modules = 0; - - /* First check if MX is available ... */ - if( OMPI_SUCCESS != ompi_common_mx_initialize() ) { - ompi_modex_send(&mca_btl_mx_component.super.btl_version, NULL, 0); - return NULL; - } - - /* get the number of card available on the system */ - if( MX_SUCCESS != (status = mx_get_info( NULL, MX_NIC_COUNT, NULL, 0, - &mca_btl_mx_component.mx_num_btls, - sizeof(uint32_t))) ) { - opal_output( 0, "mca_btl_mx_component_init: mx_get_info(MX_NIC_COUNT) failed with status %d(%s)\n", - status, mx_strerror(status) ); - return NULL; - } - /* Don't forget the bonding rules ... */ - assert( mca_btl_mx_component.mx_bonding >= 1 ); - mca_btl_mx_component.mx_num_btls /= mca_btl_mx_component.mx_bonding; - - if (0 == mca_btl_mx_component.mx_num_btls) { - mca_btl_base_error_no_nics("Myrinet/MX", "NIC"); - return NULL; - } - /* Limit ourselves to the number of devices requested by the users. */ - if( mca_btl_mx_component.mx_num_btls > mca_btl_mx_component.mx_max_btls ) { - mca_btl_mx_component.mx_num_btls = mca_btl_mx_component.mx_max_btls; - } - - /* initialize objects */ - OBJ_CONSTRUCT(&mca_btl_mx_component.mx_send_eager_frags, ompi_free_list_t); - OBJ_CONSTRUCT(&mca_btl_mx_component.mx_send_user_frags, ompi_free_list_t); - OBJ_CONSTRUCT(&mca_btl_mx_component.mx_procs, opal_list_t); - OBJ_CONSTRUCT(&mca_btl_mx_component.mx_lock, opal_mutex_t); - - ompi_free_list_init_new( &mca_btl_mx_component.mx_send_eager_frags, - sizeof(mca_btl_mx_frag_t) + mca_btl_mx_module.super.btl_eager_limit, - opal_cache_line_size, - OBJ_CLASS(mca_btl_mx_frag_t), - 0,opal_cache_line_size, - mca_btl_mx_component.mx_free_list_num, - mca_btl_mx_component.mx_free_list_max, - mca_btl_mx_component.mx_free_list_inc, - NULL ); /* use default allocator */ - - ompi_free_list_init_new( &mca_btl_mx_component.mx_send_user_frags, - sizeof(mca_btl_mx_frag_t), - opal_cache_line_size, - OBJ_CLASS(mca_btl_mx_frag_t), - 0,opal_cache_line_size, - mca_btl_mx_component.mx_free_list_num, - mca_btl_mx_component.mx_free_list_max, - mca_btl_mx_component.mx_free_list_inc, - NULL ); /* use default allocator */ - - /* intialize process hash table */ - OBJ_CONSTRUCT( &mca_btl_mx_component.mx_procs, opal_list_t ); - - /* Now we know how many NIC are available on the system. We will create a BTL - * for each one and then give a pointer to the BTL to the upper level. - */ - mca_btl_mx_component.mx_btls = malloc( mca_btl_mx_component.mx_num_btls * - sizeof(mca_btl_base_module_t*) ); - if( NULL == mca_btl_mx_component.mx_btls ) { - opal_output( 0, "MX BTL unable to allocate memory\n" ); - return NULL; - } - - mx_addrs = (mca_btl_mx_addr_t*)calloc( mca_btl_mx_component.mx_num_btls, - sizeof(mca_btl_mx_addr_t) ); - if( NULL == mx_addrs ) { - opal_output( 0, "MX BTL unable to allocate memory\n" ); - free(mca_btl_mx_component.mx_btls); - mca_btl_mx_component.mx_btls = NULL; - return NULL; - } - - /* create a btl for each NIC */ - for( i = count = 0; i < mca_btl_mx_component.mx_num_btls; i++ ) { - mca_btl_mx_module_t* mx_btl = mca_btl_mx_create(MX_ANY_NIC); - if( NULL == mx_btl ) { - continue; - } - status = mx_decompose_endpoint_addr( mx_btl->mx_endpoint_addr, - &(mx_addrs[count].nic_id), - &(mx_addrs[count].endpoint_id) ); - if( MX_SUCCESS != status ) { - mca_btl_mx_finalize( &mx_btl->super ); - continue; - } - mx_addrs[count].unique_network_id = mx_btl->mx_unique_network_id; - -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - BTL_MX_ADDR_HTON(mx_addrs[count]); -#endif - mca_btl_mx_component.mx_btls[count] = mx_btl; - count++; /* one more succesfully initialized MX interface */ - } - mca_btl_mx_component.mx_num_btls = count; - *num_btl_modules = count; - if( 0 == count ) { - /* No active BTL module */ - free(mx_addrs); - free(mca_btl_mx_component.mx_btls); - mca_btl_mx_component.mx_btls = NULL; - return NULL; - } - - /* publish the MX addresses via the MCA framework */ - ompi_modex_send(&mca_btl_mx_component.super.btl_version, mx_addrs, - sizeof(mca_btl_mx_addr_t) * mca_btl_mx_component.mx_num_btls); - - free( mx_addrs ); - - btls = malloc( mca_btl_mx_component.mx_num_btls * sizeof(mca_btl_base_module_t*) ); - if( NULL == btls ) { - free( mca_btl_mx_component.mx_btls ); - mca_btl_mx_component.mx_num_btls = 0; /* no active BTL modules */ - return NULL; - } - memcpy( btls, mca_btl_mx_component.mx_btls, - mca_btl_mx_component.mx_num_btls*sizeof(mca_btl_mx_module_t*) ); - - return btls; -} - -/* - * MX component progress. - */ -int mca_btl_mx_component_progress(void) -{ - int32_t num_progressed = 0, i; - mx_status_t mx_status; - mx_return_t mx_return; - mx_request_t mx_request; - mca_btl_mx_frag_t* frag; - - ++__counter; - for( i = 0; i < mca_btl_mx_component.mx_num_btls; i++ ) { - mca_btl_mx_module_t* mx_btl = mca_btl_mx_component.mx_btls[i]; - uint32_t mx_result = 0; - - recheck_device: - mx_return = mx_ipeek( mx_btl->mx_endpoint, &mx_request, &mx_result ); - if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) { - opal_output( 0, "mca_btl_mx_component_progress: mx_ipeek() failed with status %d (%s)\n", - mx_return, mx_strerror(mx_return) ); - continue; - } - if( OPAL_LIKELY(mx_result == 0) ) { - continue; - } - - mx_return = mx_test( mx_btl->mx_endpoint, &mx_request, &mx_status, &mx_result); - if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) { - opal_output(0, "mca_btl_mx_progress: mx_test() failed with status %d (%s)\n", - mx_return, mx_strerror(mx_return)); - continue; - } - /* on the mx_status we have now the pointer attached to the request. - * This pointer indicate which fragment we are working on. On the - * status we have the status of the operation, so we know what we - * are supposed to do next. - */ - frag = mx_status.context; - num_progressed++; - /* If the context is NULL then we are facing a send immediate request. Therefore, - * nothing special should be done, just keep going. - */ - if( NULL == frag ) goto recheck_device; - if( MCA_BTL_MX_SEND == frag->type ) { /* it's a send */ - int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); - /* call the completion callback */ - if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) { - frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint, - &(frag->base), OMPI_SUCCESS ); - } - if( btl_ownership ) { - MCA_BTL_MX_FRAG_RETURN( mx_btl, frag ); - } - goto recheck_device; - } else { - mca_btl_active_message_callback_t* reg; - mx_segment_t mx_segment; - uint8_t tag = (mx_status.match_info >> 8) & 0xff; - - reg = mca_btl_base_active_message_trigger + tag; - frag->base.des_dst->seg_len = mx_status.msg_length; - reg->cbfunc( &(mx_btl->super), tag, &(frag->base), reg->cbdata ); - /** - * The upper level extract the data from the fragment. - * Now we can register the fragment - * again with the MX BTL. - */ - mx_segment.segment_ptr = frag->base.des_dst->seg_addr.pval; - mx_segment.segment_length = mca_btl_mx_module.super.btl_eager_limit; - mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1, - 0x01ULL, BTL_MX_RECV_MASK, - frag, &(frag->mx_request) ); - if( MX_SUCCESS != mx_return ) { - opal_output( 0, "Fail to re-register a fragment with the MX NIC ... (%s)\n", - mx_strerror(mx_return) ); - } - } - } - __counter--; - return num_progressed; -} - -#if 0 - { - int counters, board, i, value, *counters_value; - char text[MX_MAX_STR_LEN]; - char *counters_name; - if( (status = mx_get_info( mx_btl->mx_endpoint, MX_PIO_SEND_MAX, NULL, 0, - &value, sizeof(int))) != MX_SUCCESS ) { - opal_output( 0, "mx_get_info(MX_PIO_SEND_MAX) failed with status %d (%s)\n", - status, mx_strerror(status) ); - } - printf( "MX_PIO_SEND_MAX = %d\n", value ); - if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COPY_SEND_MAX, NULL, 0, - &value, sizeof(int))) != MX_SUCCESS ) { - opal_output( 0, "mx_get_info(MX_COPY_SEND_MAX) failed with status %d (%s)\n", - status, mx_strerror(status) ); - } - printf( "MX_COPY_SEND_MAX = %d\n", value ); - - board = 0; - if( (status = mx_get_info( mx_btl->mx_endpoint, MX_PRODUCT_CODE, &board, sizeof(int), - text, MX_MAX_STR_LEN)) != MX_SUCCESS ) { - opal_output( 0, "mx_get_info(MX_PRODUCT_CODE) failed with status %d (%s)\n", - status, mx_strerror(status) ); - } - printf( "product code %s\n", text ); - - if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COUNTERS_COUNT, &board, sizeof(int), - &counters, sizeof(int))) != MX_SUCCESS ) { - opal_output( 0, "mx_get_info(MX_COUNTERS_COUNT) failed with status %d (%s)\n", - status, mx_strerror(status) ); - } - printf( "counters = %d\n", counters ); - counters_name = (char*)malloc( counters * MX_MAX_STR_LEN ); - if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COUNTERS_LABELS, &board, sizeof(int), - counters_name, counters * MX_MAX_STR_LEN)) != MX_SUCCESS ) { - opal_output( 0, "mx_get_info(MX_COUNTERS_LABELS) failed with status %d (%s)\n", - status, mx_strerror(status) ); - } - counters_value = (int*)malloc( counters * sizeof(int) ); - if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COUNTERS_VALUES, &board, sizeof(int), - counters_value, counters * sizeof(int))) != MX_SUCCESS ) { - opal_output( 0, "mx_get_info(MX_COUNTERS_VALUES) failed with status %d (%s)\n", - status, mx_strerror(status) ); - } - for( i = 0; i < counters; i++ ) - printf( "%d -> %s = %d\n", i, counters_name + i * MX_MAX_STR_LEN, - counters_value[i] ); - free( counters_name ); - free( counters_value ); - } -#endif diff --git a/ompi/mca/btl/mx/btl_mx_endpoint.c b/ompi/mca/btl/mx/btl_mx_endpoint.c deleted file mode 100644 index 2b749f52ee..0000000000 --- a/ompi/mca/btl/mx/btl_mx_endpoint.c +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include -#include -#include "ompi/types.h" -#include "btl_mx.h" -#include "btl_mx_endpoint.h" -#include "btl_mx_proc.h" -#include "btl_mx_frag.h" - - -/* - * Initialize state of the endpoint instance. - * - */ -static void mca_btl_mx_endpoint_construct(mca_btl_base_endpoint_t* endpoint) -{ - endpoint->endpoint_btl = NULL; - endpoint->endpoint_proc = NULL; - endpoint->status = MCA_BTL_MX_NOT_CONNECTED; -} - -/* - * Destroy a endpoint - * - */ -static void mca_btl_mx_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) -{ -} - - -OBJ_CLASS_INSTANCE( - mca_btl_mx_endpoint_t, - opal_list_item_t, - mca_btl_mx_endpoint_construct, - mca_btl_mx_endpoint_destruct); - diff --git a/ompi/mca/btl/mx/btl_mx_endpoint.h b/ompi/mca/btl/mx/btl_mx_endpoint.h deleted file mode 100644 index cb14c1cd3c..0000000000 --- a/ompi/mca/btl/mx/btl_mx_endpoint.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_MX_ENDPOINT_H -#define MCA_BTL_MX_ENDPOINT_H - -#include "opal/class/opal_list.h" -#include "opal/mca/event/event.h" -#include "btl_mx_frag.h" -#include "btl_mx.h" - -#include - -BEGIN_C_DECLS - -#define MCA_BTL_MX_NOT_CONNECTED 0x0000 -#define MCA_BTL_MX_NOT_REACHEABLE 0x0001 -#define MCA_BTL_MX_CONNECTION_PENDING 0x0002 -#define MCA_BTL_MX_CONNECTED 0x0004 - -/** - * Structure used to publish MX information to peers - */ -struct mca_btl_mx_addr_t { - uint64_t nic_id; - uint32_t endpoint_id; - uint32_t unique_network_id; /* Unique identifier for each MX network */ -}; -typedef struct mca_btl_mx_addr_t mca_btl_mx_addr_t; - -#define BTL_MX_ADDR_HTON(h) \ - do { \ - h.nic_id = hton64(h.nic_id); \ - h.endpoint_id = htonl(h.endpoint_id); \ - h.unique_network_id = htonl(h.unique_network_id); \ - } while (0) - -#define BTL_MX_ADDR_NTOH(h) \ - do { \ - h.nic_id = ntoh64(h.nic_id); \ - h.endpoint_id = ntohl(h.endpoint_id); \ - h.unique_network_id = ntohl(h.unique_network_id); \ - } while (0) - - -/** - * An abstraction that represents a connection to a endpoint process. - * An instance of mca_btl_base_endpoint_t is associated w/ each process - * and BTL pair at startup. However, connections to the endpoint - * are established dynamically on an as-needed basis: - */ - -struct mca_btl_base_endpoint_t { - opal_list_item_t super; - - struct mca_btl_mx_module_t* endpoint_btl; - /**< BTL instance that created this connection */ - - struct mca_btl_mx_proc_t* endpoint_proc; - /**< proc structure corresponding to endpoint */ - - struct mca_btl_mx_addr_t* mx_peer; - /** the address as reported by the peer */ - - mx_endpoint_addr_t mx_peer_addr; - /** the remote MX endpoint address */ - - int status; /**< status of the endpoint */ -}; - -typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t; -typedef mca_btl_base_endpoint_t mca_btl_mx_endpoint_t; - -OBJ_CLASS_DECLARATION(mca_btl_mx_endpoint_t); - -END_C_DECLS -#endif diff --git a/ompi/mca/btl/mx/btl_mx_frag.c b/ompi/mca/btl/mx/btl_mx_frag.c deleted file mode 100644 index a514eac6c2..0000000000 --- a/ompi/mca/btl/mx/btl_mx_frag.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "btl_mx_frag.h" - -static void mca_btl_mx_frag_common_constructor(mca_btl_mx_frag_t* frag) -{ - frag->base.des_src = NULL; - frag->base.des_src_cnt = 0; - frag->base.des_dst = NULL; - frag->base.des_dst_cnt = 0; -} - -static void mca_btl_mx_frag_eager_constructor(mca_btl_mx_frag_t* frag) -{ - frag->size = mca_btl_mx_module.super.btl_eager_limit; - mca_btl_mx_frag_common_constructor(frag); -} - -static void mca_btl_mx_frag_max_constructor(mca_btl_mx_frag_t* frag) -{ - frag->size = mca_btl_mx_module.super.btl_max_send_size; - mca_btl_mx_frag_common_constructor(frag); -} - -static void mca_btl_mx_frag_user_constructor(mca_btl_mx_frag_t* frag) -{ - frag->size = 0; - mca_btl_mx_frag_common_constructor(frag); -} - - -OBJ_CLASS_INSTANCE( - mca_btl_mx_frag_t, - mca_btl_base_descriptor_t, - NULL, - NULL); - -OBJ_CLASS_INSTANCE( - mca_btl_mx_frag_eager_t, - mca_btl_base_descriptor_t, - mca_btl_mx_frag_eager_constructor, - NULL); - -OBJ_CLASS_INSTANCE( - mca_btl_mx_frag_max_t, - mca_btl_base_descriptor_t, - mca_btl_mx_frag_max_constructor, - NULL); - -OBJ_CLASS_INSTANCE( - mca_btl_mx_frag_user_t, - mca_btl_base_descriptor_t, - mca_btl_mx_frag_user_constructor, - NULL); - diff --git a/ompi/mca/btl/mx/btl_mx_frag.h b/ompi/mca/btl/mx/btl_mx_frag.h deleted file mode 100644 index 332c7379cb..0000000000 --- a/ompi/mca/btl/mx/btl_mx_frag.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_MX_FRAG_H -#define MCA_BTL_MX_FRAG_H - - -#include "ompi_config.h" -#include "ompi/class/ompi_free_list.h" -#include "btl_mx.h" - -#define MCA_BTL_MX_SEND 0x01 -#define MCA_BTL_MX_RECV 0x02 - -BEGIN_C_DECLS - -struct mca_btl_mx_segment_t { - mca_btl_base_segment_t base; - uint64_t key; -}; -typedef struct mca_btl_mx_segment_t mca_btl_mx_segment_t; - -/** - * MX send framxent derived type. - */ -struct mca_btl_mx_frag_t { - mca_btl_base_descriptor_t base; - mca_btl_mx_segment_t segment[2]; - struct mca_btl_base_endpoint_t* endpoint; - uint8_t type; - mx_request_t mx_request; - size_t size; - ompi_free_list_t* mx_frag_list; -}; -typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_t; -OBJ_CLASS_DECLARATION(mca_btl_mx_frag_t); - -typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_eager_t; - -OBJ_CLASS_DECLARATION(mca_btl_mx_frag_eager_t); - -typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_max_t; - -OBJ_CLASS_DECLARATION(mca_btl_mx_frag_max_t); - -typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_user_t; - -OBJ_CLASS_DECLARATION(mca_btl_mx_frag_user_t); - -/* - * Macros to allocate/return descriptors from module specific - * free list(s). - */ - -#define MCA_BTL_MX_FRAG_ALLOC_EAGER(btl, frag) \ -do { \ - ompi_free_list_item_t *item; \ - OMPI_FREE_LIST_GET_MT( &mca_btl_mx_component.mx_send_eager_frags, item); \ - if( OPAL_LIKELY(NULL != item) ) { \ - frag = (mca_btl_mx_frag_t*) item; \ - frag->mx_frag_list = &(mca_btl_mx_component.mx_send_eager_frags); \ - frag->segment[0].base.seg_addr.pval = (void*)(frag+1); \ - } \ -} while(0) - -#define MCA_BTL_MX_FRAG_ALLOC_USER(btl, frag) \ -do { \ - ompi_free_list_item_t *item; \ - OMPI_FREE_LIST_GET_MT( &mca_btl_mx_component.mx_send_user_frags, item); \ - if( OPAL_LIKELY(NULL != item) ) { \ - frag = (mca_btl_mx_frag_t*) item; \ - frag->mx_frag_list = &(mca_btl_mx_component.mx_send_user_frags); \ - } \ -} while(0) - -#define MCA_BTL_MX_FRAG_RETURN(btl, frag) \ -do { \ - OMPI_FREE_LIST_RETURN_MT( frag->mx_frag_list, \ - (ompi_free_list_item_t*)(frag)); \ -} while(0) - -END_C_DECLS - -#endif diff --git a/ompi/mca/btl/mx/btl_mx_proc.c b/ompi/mca/btl/mx/btl_mx_proc.c deleted file mode 100644 index 330a2fcdda..0000000000 --- a/ompi/mca/btl/mx/btl_mx_proc.c +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "ompi/runtime/ompi_module_exchange.h" - -#include "btl_mx.h" -#include "btl_mx_proc.h" - -static void mca_btl_mx_proc_construct(mca_btl_mx_proc_t* proc); -static void mca_btl_mx_proc_destruct(mca_btl_mx_proc_t* proc); - -OBJ_CLASS_INSTANCE(mca_btl_mx_proc_t, - opal_list_item_t, mca_btl_mx_proc_construct, - mca_btl_mx_proc_destruct); - -void mca_btl_mx_proc_construct(mca_btl_mx_proc_t* mx_proc) -{ - mx_proc->proc_ompi = 0; - mx_proc->mx_peers_count = 0; - mx_proc->mx_peers = NULL; - mx_proc->mx_routing = NULL; - OBJ_CONSTRUCT(&mx_proc->proc_lock, opal_mutex_t); - /* add to list of all proc instance */ - OPAL_THREAD_LOCK(&mca_btl_mx_component.mx_lock); - opal_list_append(&mca_btl_mx_component.mx_procs, &mx_proc->super); - OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock); -} - -/* - * Cleanup MX proc instance - */ - -void mca_btl_mx_proc_destruct(mca_btl_mx_proc_t* mx_proc) -{ - /* remove from list of all proc instances */ - OPAL_THREAD_LOCK(&mca_btl_mx_component.mx_lock); - opal_list_remove_item(&mca_btl_mx_component.mx_procs, &mx_proc->super); - OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock); - - /* release resources */ - if( NULL != mx_proc->mx_peers ) { - free(mx_proc->mx_peers); - mx_proc->mx_peers = NULL; - } - if( NULL != mx_proc->mx_routing ) { - free(mx_proc->mx_routing); - mx_proc->mx_routing = NULL; - } - OBJ_DESTRUCT(&mx_proc->proc_lock); -} - - -/* - * Look for an existing MX process instances based on the associated - * ompi_proc_t instance. - */ -static mca_btl_mx_proc_t* mca_btl_mx_proc_lookup_ompi(ompi_proc_t* ompi_proc) -{ - mca_btl_mx_proc_t* mx_proc; - - OPAL_THREAD_LOCK(&mca_btl_mx_component.mx_lock); - - for( mx_proc = (mca_btl_mx_proc_t*)opal_list_get_first(&mca_btl_mx_component.mx_procs); - mx_proc != (mca_btl_mx_proc_t*)opal_list_get_end(&mca_btl_mx_component.mx_procs); - mx_proc = (mca_btl_mx_proc_t*)opal_list_get_next(mx_proc) ) { - - if(mx_proc->proc_ompi == ompi_proc) { - OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock); - return mx_proc; - } - - } - - OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock); - - return NULL; -} - -/** - * Create a MX process structure. There is a one-to-one correspondence - * between a ompi_proc_t and a mca_btl_mx_proc_t instance. We cache - * additional data (specifically the list of mca_btl_mx_endpoint_t instances, - * and published addresses) associated w/ a given destination on this - * datastructure. - */ -mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc) -{ - mca_btl_mx_proc_t* module_proc = NULL; - mca_btl_mx_addr_t *mx_peers; - int i, j, rc, mx_peers_count, *mx_routing; - bool at_least_one_route = false; - size_t size; - - /* Check if we have already created a MX proc - * structure for this ompi process */ - module_proc = mca_btl_mx_proc_lookup_ompi(ompi_proc); - if( module_proc != NULL ) { - return module_proc; /* Gotcha! */ - } - - /* query for the peer address info */ - rc = ompi_modex_recv( &mca_btl_mx_component.super.btl_version, - ompi_proc, (void*)&mx_peers, &size ); - if( OMPI_SUCCESS != rc ) { - opal_output( 0, "mca_pml_base_modex_recv failed for peer %s", - OMPI_NAME_PRINT(&ompi_proc->proc_name) ); - return NULL; - } - - if( size < sizeof(mca_btl_mx_addr_t) ) { /* no available connection */ - return NULL; - } - if( (size % sizeof(mca_btl_mx_addr_t)) != 0 ) { - opal_output( 0, "invalid mx address for peer %s", - OMPI_NAME_PRINT(&ompi_proc->proc_name) ); - return NULL; - } - /* Let's see if we have a way to connect to the remote proc using MX. - * Without the routing information from the mapper, it is pretty - * to do this. Right now, we base this connection detection on the last - * 6 digits of the mapper MAC. - */ - mx_peers_count = size / sizeof(mca_btl_mx_addr_t); - mx_routing = (int*)malloc( mx_peers_count * sizeof(int) ); - for( i = 0; i < mx_peers_count; mx_routing[i++] = -1 ); - - for( i = 0; i < mx_peers_count; i++ ) { - mca_btl_mx_module_t* mx_btl; -#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT - BTL_MX_ADDR_NTOH(mx_peers[rc]); -#endif - for( j = 0; j < mca_btl_mx_component.mx_num_btls; j++ ) { - mx_btl = mca_btl_mx_component.mx_btls[j]; - if( mx_btl->mx_unique_network_id == mx_peers[j].unique_network_id ) { - /* There is at least one connection between these two nodes */ - if( -1 == mx_routing[j] ) { - /* First connection */ - mx_routing[j] = i; - at_least_one_route = true; - break; - } - /* If multiple remote endpoints match mine, we keep going. As a - * result we will match them in order, i.e. remote endpoint 0 - * will be connected to local endpoint 0. - */ - } - } - } - if( false == at_least_one_route ) { - free(mx_routing); - return NULL; - } - - module_proc = OBJ_NEW(mca_btl_mx_proc_t); - module_proc->proc_ompi = ompi_proc; - module_proc->mx_peers_count = mx_peers_count; - module_proc->mx_peers = mx_peers; - module_proc->mx_routing = mx_routing; - return module_proc; -} - - -/** - * Note that this routine must be called with the lock on the process - * already held. Insert a btl instance into the proc array and assign - * it an address. - */ -int mca_btl_mx_proc_insert( mca_btl_mx_proc_t* module_proc, - mca_btl_mx_endpoint_t* module_endpoint ) -{ - mca_btl_mx_module_t* mx_btl; - int btl_index, peer_endpoint_index; - - for( btl_index = 0; btl_index < mca_btl_mx_component.mx_num_btls; btl_index++ ) { - mx_btl = mca_btl_mx_component.mx_btls[btl_index]; - peer_endpoint_index = module_proc->mx_routing[btl_index]; - if( (-1 != peer_endpoint_index) && (mx_btl == module_endpoint->endpoint_btl) ) { - module_endpoint->mx_peer = module_proc->mx_peers + peer_endpoint_index; - module_endpoint->endpoint_proc = module_proc; - return OMPI_SUCCESS; - } - } - module_proc->mx_peers_count = 0; - /** - * No Myrinet connectivity. Let the PML layer figure out another - * way to communicate with the peer. - */ - return OMPI_ERROR; -} - -int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint ) -{ - int num_retry = 0; - mx_return_t mx_status; - mx_endpoint_addr_t mx_remote_addr; - - module_endpoint->status = MCA_BTL_MX_CONNECTION_PENDING; - - retry_connect: - mx_status = mx_connect( module_endpoint->endpoint_btl->mx_endpoint, - module_endpoint->mx_peer->nic_id, module_endpoint->mx_peer->endpoint_id, - mca_btl_mx_component.mx_filter, mca_btl_mx_component.mx_timeout, &mx_remote_addr ); - if( MX_SUCCESS != mx_status ) { - if( MX_TIMEOUT == mx_status ) - if( num_retry++ < mca_btl_mx_component.mx_connection_retries ) - goto retry_connect; - { - char peer_name[MX_MAX_HOSTNAME_LEN]; - - if( MX_SUCCESS != mx_nic_id_to_hostname( module_endpoint->mx_peer->nic_id, peer_name ) ) - sprintf( peer_name, "unknown %lx nic_id", (long)module_endpoint->mx_peer->nic_id ); - - opal_output( 0, "mx_connect fail for %s with key %x (error %s)\n\tUnique ID (local %x remote %x)\n", - peer_name, mca_btl_mx_component.mx_filter, mx_strerror(mx_status), - module_endpoint->endpoint_btl->mx_unique_network_id, - module_endpoint->mx_peer->unique_network_id ); - } - module_endpoint->status = MCA_BTL_MX_NOT_REACHEABLE; - return OMPI_ERROR; - } - module_endpoint->mx_peer_addr = mx_remote_addr; - module_endpoint->status = MCA_BTL_MX_CONNECTED; - - return OMPI_SUCCESS; -} - diff --git a/ompi/mca/btl/mx/btl_mx_proc.h b/ompi/mca/btl/mx/btl_mx_proc.h deleted file mode 100644 index 355d0cbe23..0000000000 --- a/ompi/mca/btl/mx/btl_mx_proc.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2008 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_MX_PROC_H -#define MCA_BTL_MX_PROC_H - -#include "opal/class/opal_object.h" -#include "ompi/proc/proc.h" -#include "btl_mx.h" -#include "btl_mx_endpoint.h" - -BEGIN_C_DECLS - -/** - * Represents the state of a remote process and the set of addresses - * that it exports. Also cache an instance of mca_btl_base_endpoint_t for - * each BTL instance that attempts to open a connection to the process. - */ -struct mca_btl_mx_proc_t { - opal_list_item_t super; - /**< allow proc to be placed on a list */ - - ompi_proc_t *proc_ompi; - /**< pointer to corresponding ompi_proc_t */ - - mca_btl_mx_addr_t *mx_peers; /**< peers addresses */ - int mx_peers_count; - int* mx_routing; /**< peer routing information */ - - opal_mutex_t proc_lock; - /**< lock to protect against concurrent access to proc state */ -}; -typedef struct mca_btl_mx_proc_t mca_btl_mx_proc_t; - -OBJ_CLASS_DECLARATION(mca_btl_mx_proc_t); - -mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc); -int mca_btl_mx_proc_insert(mca_btl_mx_proc_t*, mca_btl_base_endpoint_t*); -int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint ); - -END_C_DECLS - -#endif /* MCA_BTL_MX_PROC_H */ diff --git a/ompi/mca/btl/mx/configure.m4 b/ompi/mca/btl/mx/configure.m4 deleted file mode 100644 index a6b94be574..0000000000 --- a/ompi/mca/btl/mx/configure.m4 +++ /dev/null @@ -1,42 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2007 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - - -# MCA_btl_mx_CONFIG([action-if-can-compile], -# [action-if-cant-compile]) -# ------------------------------------------------ -AC_DEFUN([MCA_ompi_btl_mx_CONFIG],[ - AC_CONFIG_FILES([ompi/mca/btl/mx/Makefile]) - - OMPI_CHECK_MX([btl_mx], - [btl_mx_happy="yes"], - [btl_mx_happy="no"]) - - AS_IF([test "$btl_mx_happy" = "yes"], - [$1], - [$2]) - - # substitute in the things needed to build mx - AC_SUBST([btl_mx_CFLAGS]) - AC_SUBST([btl_mx_CPPFLAGS]) - AC_SUBST([btl_mx_LDFLAGS]) - AC_SUBST([btl_mx_LIBS]) -])dnl - diff --git a/ompi/mca/common/mx/Makefile.am b/ompi/mca/common/mx/Makefile.am deleted file mode 100644 index e717ab9c7d..0000000000 --- a/ompi/mca/common/mx/Makefile.am +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# A word of explanation... -# -# This library is linked against various MCA components because all -# shared-memory based components (e.g., mpool, ptl, etc.) need to -# share some common code and data. There's two cases: -# -# 1. libmca_common_portals.la is a shared library. By linking that shared -# library to all components that need it, the OS linker will -# automatically load it into the process as necessary, and there will -# only be one copy (i.e., all the components will share *one* copy of -# the code and data). -# 2. libmca_common_portals.la is a static library. In this case, it -# will be rolled up into the top-level libmpi.la. It will also be -# rolled into each component, but then the component will also be -# rolled up into the upper-level libmpi.la. Libtool sorts this all -# out and it all works out in the end. -# -# Note that building this common component statically and linking -# against other dynamic components is *not* supported! - - -AM_CPPFLAGS = $(common_mx_CPPFLAGS) - -# Header files - -headers = \ - common_mx.h - -# Source files - -sources = \ - common_mx.c - - -lib_LTLIBRARIES = -noinst_LTLIBRARIES = -comp_inst = libmca_common_mx.la -comp_noinst = libmca_common_mx_noinst.la - -if MCA_BUILD_ompi_common_mx_DSO -lib_LTLIBRARIES += $(comp_inst) -else -noinst_LTLIBRARIES += $(comp_noinst) -endif - -libmca_common_mx_la_SOURCES = $(headers) $(sources) -libmca_common_mx_la_LDFLAGS = -version-info $(libmca_common_mx_so_version) $(common_mx_LDFLAGS) -libmca_common_mx_la_LIBADD = $(common_mx_LIBS) - -libmca_common_mx_noinst_la_SOURCES = $(libmca_common_mx_la_SOURCES) -libmca_common_mx_noinst_la_LDFLAGS = $(common_mx_LDFLAGS) -libmca_common_mx_noinst_la_LIBADD = $(common_mx_LIBS) - - -# Conditionally install the header files - -if WANT_INSTALL_HEADERS -ompidir = $(includedir)/openmpi/$(subdir) -ompi_HEADERS = $(headers) -endif - -# These two rules will sym link the "noinst" libtool library filename -# to the installable libtool library filename in the case where we are -# compiling this component statically (case 2), described above). - -all-local: - if test -z "$(lib_LTLIBRARIES)"; then \ - rm -f "$(comp_inst)"; \ - $(LN_S) "$(comp_noinst)" "$(comp_inst)"; \ - fi - -clean-local: - if test -z "$(lib_LTLIBRARIES)"; then \ - rm -f "$(comp_inst)"; \ - fi diff --git a/ompi/mca/common/mx/common_mx.c b/ompi/mca/common/mx/common_mx.c deleted file mode 100644 index 949a383354..0000000000 --- a/ompi/mca/common/mx/common_mx.c +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2008 Myricom. All rights reserved. - * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" -#include "ompi/constants.h" -#include "common_mx.h" - -#include -#include "opal/memoryhooks/memory.h" -#include "opal/util/output.h" -#include "ompi/runtime/params.h" - - -int mx__regcache_clean(void *ptr, size_t size); - -static int ompi_common_mx_initialize_ref_cnt = 0; -static int ompi_common_mx_available = 0; -static int ompi_common_mx_memory_cb_registered = 0; - -static void -memory_release_cb(void *buf, size_t length, void *cbdata, bool from_alloc) -{ - mx__regcache_clean(buf, length); -} - - -int -ompi_common_mx_initialize(void) -{ - mx_return_t mx_return; - int ret = OMPI_SUCCESS; - - ompi_common_mx_initialize_ref_cnt++; - - if(ompi_common_mx_initialize_ref_cnt == 1) { - /* set the MX error handle to always return. This function is the - * only MX function allowed to be called before mx_init in order - * to make sure that if the MX is not up and running the MX - * library does not exit the application. - */ - mx_set_error_handler(MX_ERRORS_RETURN); - - /* If we have a memory manager available, then tell MX to use - the rcache */ - if (0 != (OPAL_MEMORY_FREE_SUPPORT & opal_mem_hooks_support_level())) { - if (OMPI_SUCCESS == - opal_mem_hooks_register_release(memory_release_cb, NULL)) { - ompi_common_mx_memory_cb_registered = 1; - setenv("MX_RCACHE", "2", 1); - } - } - - /* initialize the mx library */ - mx_return = mx_init(); - - if(MX_SUCCESS != mx_return) { - ompi_common_mx_available = -1; - if (0 != ompi_common_mx_memory_cb_registered) { - opal_mem_hooks_unregister_release(memory_release_cb); - ompi_common_mx_memory_cb_registered = 0; - } - opal_output(0, - "Error in mx_init (error %s)\n", - mx_strerror(mx_return)); - /* We did not succeed to initialize the MX device */ - ompi_common_mx_initialize_ref_cnt = 0; - return OMPI_ERR_NOT_AVAILABLE; - } - ompi_common_mx_available = 1; - } else if (ompi_common_mx_available < 0) { - ret = OMPI_ERR_NOT_AVAILABLE; - } - - return ret; -} - -int -ompi_common_mx_finalize(void) -{ - ompi_common_mx_initialize_ref_cnt--; - if( 0 == ompi_common_mx_initialize_ref_cnt ) { - mx_return_t mx_return; - - if (0 != ompi_common_mx_memory_cb_registered) { - opal_mem_hooks_unregister_release(memory_release_cb); - ompi_common_mx_memory_cb_registered = 0; - } - - mx_return = mx_finalize(); - if(mx_return != MX_SUCCESS){ - opal_output(0, "Error in mx_finalize (error %s)\n", mx_strerror(mx_return)); - return OMPI_ERROR; - } - } - return OMPI_SUCCESS; -} diff --git a/ompi/mca/common/mx/common_mx.h b/ompi/mca/common/mx/common_mx.h deleted file mode 100644 index 722f187a13..0000000000 --- a/ompi/mca/common/mx/common_mx.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OMPI_MCA_COMMON_MX_H -#define OMPI_MCA_COMMON_MX_H - -#include "ompi_config.h" -#include "myriexpress.h" - - -/** - * Initialize mx library - * * - * @retval OMPI_SUCCESS MX successfully initialized - * @retval OMPI_ERR_NOT_AVAILABLE MX could not be initialized - */ -OMPI_DECLSPEC int ompi_common_mx_initialize(void); - - -/** - * Shut down mx library - * - */ -OMPI_DECLSPEC int ompi_common_mx_finalize(void); - - - -#endif /* OMPI_MCA_COMMON_PORTALS_H */ diff --git a/ompi/mca/common/mx/configure.m4 b/ompi/mca/common/mx/configure.m4 deleted file mode 100644 index 04bbfa0b68..0000000000 --- a/ompi/mca/common/mx/configure.m4 +++ /dev/null @@ -1,42 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - - -# MCA_mtl_mx_CONFIG([action-if-can-compile], -# [action-if-cant-compile]) -# ------------------------------------------------ -AC_DEFUN([MCA_ompi_common_mx_CONFIG],[ - AC_CONFIG_FILES([ompi/mca/common/mx/Makefile]) - - OMPI_CHECK_MX([common_mx], - [common_mx_happy="yes"], - [common_mx_happy="no"]) - - AS_IF([test "$common_mx_happy" = "yes"], - [$1], - [$2]) - - # substitute in the things needed to build mx - AC_SUBST([common_mx_CFLAGS]) - AC_SUBST([common_mx_CPPFLAGS]) - AC_SUBST([common_mx_LDFLAGS]) - AC_SUBST([common_mx_LIBS]) -])dnl - diff --git a/ompi/mca/mtl/mx/Makefile.am b/ompi/mca/mtl/mx/Makefile.am deleted file mode 100644 index 93715f5419..0000000000 --- a/ompi/mca/mtl/mx/Makefile.am +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2006 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -EXTRA_DIST = post_configure.sh - -AM_CPPFLAGS = $(mtl_mx_CPPFLAGS) - -mtl_mx_sources = \ - mtl_mx.c \ - mtl_mx.h \ - mtl_mx_cancel.c \ - mtl_mx_component.c \ - mtl_mx_endpoint.c \ - mtl_mx_endpoint.h \ - mtl_mx_probe.c \ - mtl_mx_recv.c \ - mtl_mx_request.h \ - mtl_mx_send.c \ - mtl_mx_types.h - -# Make the output library in this directory, and name it either -# mca__.la (for DSO builds) or libmca__.la -# (for static builds). - -if MCA_BUILD_ompi_mtl_mx_DSO -component_noinst = -component_install = mca_mtl_mx.la -else -component_noinst = libmca_mtl_mx.la -component_install = -endif - -mcacomponentdir = $(pkglibdir) -mcacomponent_LTLIBRARIES = $(component_install) -mca_mtl_mx_la_SOURCES = $(mtl_mx_sources) -mca_mtl_mx_la_LIBADD = \ - $(mtl_mx_LIBS) \ - $(top_ompi_builddir)/ompi/mca/common/mx/libmca_common_mx.la -mca_mtl_mx_la_LDFLAGS = -module -avoid-version $(mtl_mx_LDFLAGS) - -noinst_LTLIBRARIES = $(component_noinst) -libmca_mtl_mx_la_SOURCES = $(mtl_mx_sources) -libmca_mtl_mx_la_LIBADD = $(mtl_mx_LIBS) -libmca_mtl_mx_la_LDFLAGS = -module -avoid-version $(mtl_mx_LDFLAGS) diff --git a/ompi/mca/mtl/mx/configure.m4 b/ompi/mca/mtl/mx/configure.m4 deleted file mode 100644 index c45815c858..0000000000 --- a/ompi/mca/mtl/mx/configure.m4 +++ /dev/null @@ -1,49 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2013 Sandia National Laboratories. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# MCA_ompi_mtl_mx_POST_CONFIG(will_build) -# ---------------------------------------- -# Only require the tag if we're actually going to be built -AC_DEFUN([MCA_ompi_mtl_mx_POST_CONFIG], [ - AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([MTL])]) -])dnl - -# MCA_mtl_mx_CONFIG([action-if-can-compile], -# [action-if-cant-compile]) -# ------------------------------------------------ -AC_DEFUN([MCA_ompi_mtl_mx_CONFIG],[ - AC_CONFIG_FILES([ompi/mca/mtl/mx/Makefile]) - - OMPI_CHECK_MX([mtl_mx], - [mtl_mx_happy="yes"], - [mtl_mx_happy="no"]) - - AS_IF([test "$mtl_mx_happy" = "yes"], - [$1], - [$2]) - - # substitute in the things needed to build mx - AC_SUBST([mtl_mx_CFLAGS]) - AC_SUBST([mtl_mx_CPPFLAGS]) - AC_SUBST([mtl_mx_LDFLAGS]) - AC_SUBST([mtl_mx_LIBS]) -])dnl - diff --git a/ompi/mca/mtl/mx/mtl_mx.c b/ompi/mca/mtl/mx/mtl_mx.c deleted file mode 100644 index d7010b8dd8..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx.c +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2010 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "opal/prefetch.h" - -#include "ompi/mca/mtl/mtl.h" -#include "ompi/runtime/ompi_module_exchange.h" -#include "ompi/mca/mtl/base/mtl_base_datatype.h" -#include "ompi/mca/common/mx/common_mx.h" -#include "mtl_mx.h" -#include "mtl_mx_types.h" -#include "mtl_mx_endpoint.h" -#include "mtl_mx_request.h" - -static int -ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl, - size_t nprocs, - struct ompi_proc_t** procs); -static int -ompi_mtl_mx_del_procs(struct mca_mtl_base_module_t *mtl, - size_t nprocs, - struct ompi_proc_t** procs); - -static int -ompi_mtl_mx_add_comm(struct mca_mtl_base_module_t *mtl, - struct ompi_communicator_t *comm); - -static int -ompi_mtl_mx_del_comm(struct mca_mtl_base_module_t *mtl, - struct ompi_communicator_t *comm); - -mca_mtl_mx_module_t ompi_mtl_mx = { - { - 8191, /* max cid - 2^13 - 1 */ - (1UL << 30), /* max tag value - must allow negatives */ - 0, /* request reserve space */ - 0, /* flags */ - - - ompi_mtl_mx_add_procs, - ompi_mtl_mx_del_procs, - ompi_mtl_mx_finalize, - - ompi_mtl_mx_send, - ompi_mtl_mx_isend, - - ompi_mtl_mx_irecv, - ompi_mtl_mx_iprobe, - ompi_mtl_mx_imrecv, - ompi_mtl_mx_improbe, - - ompi_mtl_mx_cancel, - ompi_mtl_mx_add_comm, - ompi_mtl_mx_del_comm - } -}; - -int ompi_mtl_mx_progress( void ); - -int ompi_mtl_mx_module_init(){ - mx_param_t mx_param; - mx_return_t mx_return; - int32_t nic, ep; - - /* setup params */ - mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX; - mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max; - - /* get a local endpoint */ - nic = ompi_mtl_mx.mx_board_num; - if (nic < 0) { - nic = MX_ANY_NIC; - } - ep = ompi_mtl_mx.mx_endpoint_num; - if (ep < 0) { - ep = MX_ANY_ENDPOINT; - } - mx_return = mx_open_endpoint(nic, - ep, - ompi_mtl_mx.mx_filter, - NULL, - 0, - &ompi_mtl_mx.mx_endpoint); - - if(mx_return != MX_SUCCESS) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return)); - return OMPI_ERROR; - } - - /* get the endpoint address */ - mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint, - &ompi_mtl_mx.mx_endpoint_addr); - - if(mx_return != MX_SUCCESS) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return)); - return OMPI_ERROR; - } - - mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id), - &(ompi_mtl_mx.mx_addr.endpoint_id) ); - - if(mx_return != MX_SUCCESS) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return)); - return OMPI_ERROR; - } - opal_output_verbose(10, ompi_mtl_base_framework.framework_output, - "mtl:mx: local nic %d, endpoint %d, got nic %d, ep %d\n", nic, ep, - (int)ompi_mtl_mx.mx_addr.nic_id, - ompi_mtl_mx.mx_addr.endpoint_id); - - ompi_modex_send( &mca_mtl_mx_component.super.mtl_version, - &ompi_mtl_mx.mx_addr, - sizeof(mca_mtl_mx_addr_t)); - - /* register the mtl mx progress function */ - opal_progress_register(ompi_mtl_mx_progress); - - return OMPI_SUCCESS; -} - -int -ompi_mtl_mx_finalize(struct mca_mtl_base_module_t* mtl) { - mx_return_t mx_return; - - opal_progress_unregister(ompi_mtl_mx_progress); - - /* free resources */ - mx_return = mx_close_endpoint(ompi_mtl_mx.mx_endpoint); - if(mx_return != MX_SUCCESS){ - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_close_endpoint (error %s)\n", mx_strerror(mx_return)); - return OMPI_ERROR; - } - - return ompi_common_mx_finalize(); - -} - -int -ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl, - size_t nprocs, - struct ompi_proc_t** procs) -{ - int i; - assert(mtl == &ompi_mtl_mx.super); - - for( i = 0; i < (int) nprocs; i++ ){ - mca_mtl_mx_endpoint_t* mtl_mx_endpoint = - mca_mtl_mx_endpoint_create(procs[i]); - if(NULL == mtl_mx_endpoint) { - return OMPI_ERROR; - } - procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] = - mtl_mx_endpoint; - } - - /* because mx_connect isn't an interupting function, need to - progress MX as often as possible during the stage gate 2. This - would have happened after the stage gate anyway, so we're just - speeding things up a bit. */ -#if OMPI_ENABLE_PROGRESS_THREADS == 0 - /* switch from letting us sit in the event library for a bit each - time through opal_progress() to completely non-blocking */ - opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK); -#endif - - return OMPI_SUCCESS; -} - - -int -ompi_mtl_mx_del_procs(struct mca_mtl_base_module_t *mtl, - size_t nprocs, - struct ompi_proc_t** procs) -{ - return OMPI_SUCCESS; -} - - -int -ompi_mtl_mx_add_comm(struct mca_mtl_base_module_t *mtl, - struct ompi_communicator_t *comm) -{ - return OMPI_SUCCESS; -} - -int -ompi_mtl_mx_del_comm(struct mca_mtl_base_module_t *mtl, - struct ompi_communicator_t *comm) -{ - return OMPI_SUCCESS; -} - - -int ompi_mtl_mx_progress( void ) { - mx_return_t mx_return; - mx_request_t mx_request; - mx_status_t mx_status; - uint32_t result; - mca_mtl_mx_request_t* mtl_mx_request; - int completed = 0; - - while(1){ - mx_return = mx_ipeek(ompi_mtl_mx.mx_endpoint, - &mx_request, - &result); - - if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_ipeek (error %s)\n", mx_strerror(mx_return)); - } - if(result) { - completed++; - mx_return = mx_test(ompi_mtl_mx.mx_endpoint, - &mx_request, - &mx_status, - &result); - if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_test (error %s)\n", mx_strerror(mx_return)); - abort(); - } - if( OPAL_UNLIKELY(0 == result) ) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in ompi_mtl_mx_progress, mx_ipeek returned a request, mx_test on the request resulted failure.\n"); - abort(); - } - mtl_mx_request = (mca_mtl_mx_request_t*) mx_status.context; - if(OMPI_MTL_MX_ISEND == mtl_mx_request->type) { - if(mtl_mx_request->free_after) { - free(mtl_mx_request->mx_segment[0].segment_ptr); - } - } else { - assert( OMPI_MTL_MX_IRECV == mtl_mx_request->type ); - - ompi_mtl_datatype_unpack(mtl_mx_request->convertor, - mtl_mx_request->mx_segment[0].segment_ptr, - mx_status.xfer_length); - /* set the status */ - MX_GET_SRC(mx_status.match_info, - mtl_mx_request->super.ompi_req->req_status.MPI_SOURCE); - MX_GET_TAG(mx_status.match_info, - mtl_mx_request->super.ompi_req->req_status.MPI_TAG); - mtl_mx_request->super.ompi_req->req_status._ucount = - mx_status.xfer_length; - } - /* suppose everything went just fine ... */ - mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS; - if( OPAL_UNLIKELY(MX_STATUS_SUCCESS != mx_status.code) ) { - if( MX_STATUS_TRUNCATED == mx_status.code ) { - mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE; - } else { - mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_INTERN; - } - return completed; - } - mtl_mx_request->super.completion_callback(&mtl_mx_request->super); - return completed; - } else { - return completed; - } - } -} - diff --git a/ompi/mca/mtl/mx/mtl_mx.h b/ompi/mca/mtl/mx/mtl_mx.h deleted file mode 100644 index 08c8742bdb..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2007 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MTL_MX_H_HAS_BEEN_INCLUDED -#define MTL_MX_H_HAS_BEEN_INCLUDED - -#include "ompi/mca/mtl/mtl.h" -#include "ompi/mca/mtl/base/base.h" -#include "opal/datatype/opal_convertor.h" - - -BEGIN_C_DECLS - -int -ompi_mtl_mx_send(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t* comm, - int dest, - int tag, - struct opal_convertor_t *convertor, - mca_pml_base_send_mode_t mode); - -extern int ompi_mtl_mx_isend(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t* comm, - int dest, - int tag, - struct opal_convertor_t *convertor, - mca_pml_base_send_mode_t mode, - bool blocking, - mca_mtl_request_t * mtl_request); - -extern int ompi_mtl_mx_irecv(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t *comm, - int src, - int tag, - struct opal_convertor_t *convertor, - struct mca_mtl_request_t *mtl_request); - - -extern int ompi_mtl_mx_iprobe(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t *comm, - int src, - int tag, - int *flag, - struct ompi_status_public_t *status); - -extern int ompi_mtl_mx_imrecv(struct mca_mtl_base_module_t* mtl, - struct opal_convertor_t *convertor, - struct ompi_message_t **message, - struct mca_mtl_request_t *mtl_request); - -extern int ompi_mtl_mx_improbe(struct mca_mtl_base_module_t *mtl, - struct ompi_communicator_t *comm, - int src, - int tag, - int *matched, - struct ompi_message_t **message, - struct ompi_status_public_t *status); - -extern int ompi_mtl_mx_cancel(struct mca_mtl_base_module_t* mtl, - struct mca_mtl_request_t *mtl_request, - int flag); - -extern int ompi_mtl_mx_finalize(struct mca_mtl_base_module_t* mtl); - -int ompi_mtl_mx_module_init(void); - - - -END_C_DECLS - -#endif /* MTL_MX_H_HAS_BEEN_INCLUDED */ - diff --git a/ompi/mca/mtl/mx/mtl_mx_cancel.c b/ompi/mca/mtl/mx/mtl_mx_cancel.c deleted file mode 100644 index edfc2ca476..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_cancel.c +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - - -#include "mtl_mx.h" -#include "mtl_mx_types.h" -#include "mtl_mx_request.h" - -int -ompi_mtl_mx_cancel(struct mca_mtl_base_module_t* mtl, - struct mca_mtl_request_t *mtl_request, - int flag) -{ - mca_mtl_mx_request_t *mtl_mx_request = - (mca_mtl_mx_request_t*) mtl_request; - uint32_t result; - - mx_cancel(ompi_mtl_mx.mx_endpoint, - &mtl_mx_request->mx_request, - &result); - - /* If MX canceled the request, mark it as canceled and complete - it. Otherwise, keep on going */ - if (result) { - mtl_request->ompi_req->req_status._cancelled = true; - mtl_mx_request->super.completion_callback(&mtl_mx_request->super); - } - - return OMPI_SUCCESS; -} diff --git a/ompi/mca/mtl/mx/mtl_mx_component.c b/ompi/mca/mtl/mx/mtl_mx_component.c deleted file mode 100644 index e04d0387cc..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_component.c +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "opal/mca/event/event.h" -#include "ompi/mca/mtl/base/base.h" -#include "ompi/mca/common/mx/common_mx.h" - -#include "mtl_mx.h" -#include "mtl_mx_types.h" -#include "mtl_mx_request.h" - -#include "myriexpress.h" - -#define MCA_MTL_MX_QUEUE_LENGTH_MAX 2*1024*1024 -static int ompi_mtl_mx_component_register(void); -static int ompi_mtl_mx_component_open(void); -static int ompi_mtl_mx_component_close(void); -static int ompi_mtl_mx_component_initialized = 0; - -static mca_mtl_base_module_t* ompi_mtl_mx_component_init( bool enable_progress_threads, - bool enable_mpi_threads ); - -mca_mtl_mx_component_t mca_mtl_mx_component = { - - { - /* First, the mca_base_component_t struct containing meta - * information about the component itself */ - - { - MCA_MTL_BASE_VERSION_2_0_0, - - "mx", /* MCA component name */ - OMPI_MAJOR_VERSION, /* MCA component major version */ - OMPI_MINOR_VERSION, /* MCA component minor version */ - OMPI_RELEASE_VERSION, /* MCA component release version */ - ompi_mtl_mx_component_open, /* component open */ - ompi_mtl_mx_component_close, /* component close */ - NULL, - ompi_mtl_mx_component_register - }, - { - /* The component is not checkpoint ready */ - MCA_BASE_METADATA_PARAM_NONE - }, - - ompi_mtl_mx_component_init /* component init */ - } -}; - -static int -ompi_mtl_mx_component_register(void) -{ - ompi_mtl_mx.mx_filter = 0xaaaaffff; - (void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "filter", - "user assigned value used to filter incomming messages", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_filter); - - ompi_mtl_mx.mx_timeout = MX_INFINITE; - (void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "timeout", - "Timeout for connections", MCA_BASE_VAR_TYPE_INT, - NULL, 0, 0, OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_timeout); - - ompi_mtl_mx.mx_retries = 20; - (void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "retries", - "Number of retries for each new connection before considering the peer as unreacheable", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_retries); - - ompi_mtl_mx.mx_support_sharedmem = 1; - (void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "shared_mem", - "Enable the MX support for shared memory", MCA_BASE_VAR_TYPE_INT, - NULL, 0, 0, OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_support_sharedmem); - - ompi_mtl_mx.mx_unexp_queue_max = MCA_MTL_MX_QUEUE_LENGTH_MAX; - (void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "unexpected_queue_length", - "Length of MX unexpected message queue", MCA_BASE_VAR_TYPE_INT, - NULL, 0, 0, OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_unexp_queue_max); - - ompi_mtl_mx.mx_board_num = -1; - (void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "board", - "Which MX board number to use (<0 = any)", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_board_num); - - ompi_mtl_mx.mx_endpoint_num = -1; - (void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "endpoint", - "Which MX endpoint number to use (<0 = any)", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_endpoint_num); - - return OMPI_SUCCESS; -} - -static int -ompi_mtl_mx_component_open(void) -{ - if(ompi_mtl_mx.mx_unexp_queue_max > MCA_MTL_MX_QUEUE_LENGTH_MAX) { - ompi_mtl_mx.mx_unexp_queue_max = MCA_MTL_MX_QUEUE_LENGTH_MAX; - } - - return OMPI_SUCCESS; -} - - -static int -ompi_mtl_mx_component_close(void) -{ - --ompi_mtl_mx_component_initialized; - if( 0 == ompi_mtl_mx_component_initialized ) { - int ret = ompi_common_mx_finalize(); - if(OMPI_SUCCESS != ret) { - return OMPI_ERROR; - } - } - return OMPI_SUCCESS; -} - - -static mca_mtl_base_module_t* -ompi_mtl_mx_component_init(bool enable_progress_threads, - bool enable_mpi_threads) -{ - int ret; - - ret = ompi_common_mx_initialize(); - if(OMPI_SUCCESS != ret) { - return NULL; - } - ompi_mtl_mx_component_initialized++; - - ret = ompi_mtl_mx_module_init(); - if (OMPI_SUCCESS != ret) { - return NULL; - } - - ompi_mtl_mx.super.mtl_request_size = - sizeof(mca_mtl_mx_request_t) - - sizeof(struct mca_mtl_request_t); - - return &ompi_mtl_mx.super; -} - diff --git a/ompi/mca/mtl/mx/mtl_mx_endpoint.c b/ompi/mca/mtl/mx/mtl_mx_endpoint.c deleted file mode 100644 index ae1c794ef9..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_endpoint.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include "opal/util/output.h" -#include -#include -#include "ompi/types.h" -#include "mtl_mx.h" -#include "mtl_mx_types.h" -#include "mtl_mx_endpoint.h" -#include "ompi/runtime/ompi_module_exchange.h" - -/* - * Initialize state of the endpoint instance. - * - */ - -static void mca_mtl_mx_endpoint_construct(mca_mtl_mx_endpoint_t* endpoint) -{ - endpoint->mtl_mx_module = NULL; - endpoint->mx_peer = NULL; -} - -/* - * Destroy a endpoint - * - */ - -static void mca_mtl_mx_endpoint_destruct(mca_mtl_mx_endpoint_t* endpoint) -{ -} - - -OBJ_CLASS_INSTANCE( - mca_mtl_mx_endpoint_t, - opal_list_item_t, - mca_mtl_mx_endpoint_construct, - mca_mtl_mx_endpoint_destruct); - - - - - -mca_mtl_mx_endpoint_t* mca_mtl_mx_endpoint_create(ompi_proc_t* ompi_proc) { - mca_mtl_mx_endpoint_t* mtl_mx_endpoint = NULL; - int rc; - mca_mtl_mx_addr_t *mx_peer; - size_t size; - mx_return_t mx_return; - int num_retry = 0; - /* get the remote proc's address (only one) */ - rc = ompi_modex_recv(&mca_mtl_mx_component.super.mtl_version, - ompi_proc, (void**)&mx_peer, &size); - if( rc != OMPI_SUCCESS || size != sizeof(mca_mtl_mx_addr_t)) { - return NULL; - } - - mtl_mx_endpoint = (mca_mtl_mx_endpoint_t*) OBJ_NEW(mca_mtl_mx_endpoint_t); - mtl_mx_endpoint->mx_peer = mx_peer; - - retry_connect: - mx_return = mx_connect(ompi_mtl_mx.mx_endpoint, - mx_peer->nic_id, - mx_peer->endpoint_id, - ompi_mtl_mx.mx_filter, - ompi_mtl_mx.mx_timeout, - &mtl_mx_endpoint->mx_peer_addr); - if(MX_SUCCESS != mx_return) { - char peer_name[MX_MAX_HOSTNAME_LEN]; - if(MX_TIMEOUT == mx_return) { - if( num_retry++ < ompi_mtl_mx.mx_retries ) { - goto retry_connect; - } - } - - if(MX_SUCCESS != mx_nic_id_to_hostname( mx_peer->nic_id, peer_name)) { - sprintf( peer_name, "unknown %lx nic_id", (long)mx_peer->nic_id ); - } - opal_output(ompi_mtl_base_framework.framework_output, - "mx_connect fail for %s with key %x (error %s)\n", - peer_name, ompi_mtl_mx.mx_filter, mx_strerror(mx_return) ); - return NULL; - } - - - return mtl_mx_endpoint; - -} diff --git a/ompi/mca/mtl/mx/mtl_mx_endpoint.h b/ompi/mca/mtl/mx/mtl_mx_endpoint.h deleted file mode 100644 index 8d174f8449..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_endpoint.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_MTL_MX_ENDPOINT_H -#define MCA_MTL_MX_ENDPOINT_H - -#include "opal/class/opal_list.h" -#include "opal/mca/event/event.h" -#include "ompi/proc/proc.h" -#include "ompi/mca/mtl/mtl.h" -#include "mtl_mx.h" - -#include "myriexpress.h" - -BEGIN_C_DECLS - -OBJ_CLASS_DECLARATION(mca_mtl_mx_endpoint_t); - -/** - * Structure used to publish MX information to peers - */ -struct mca_mtl_mx_addr_t { - uint64_t nic_id; - uint32_t endpoint_id; -}; -typedef struct mca_mtl_mx_addr_t mca_mtl_mx_addr_t; - -/** - * An abstraction that represents a connection to a endpoint process. - * An instance of mca_mtl_mx_endpoint_t is associated w/ each process - * and MTL pair at startup. However, connections to the endpoint - * are established dynamically on an as-needed basis: - */ - -struct mca_mtl_mx_endpoint_t { - opal_list_item_t super; - - struct mca_mtl_mx_module_t* mtl_mx_module; - /**< MTL instance that created this connection */ - - struct mca_mtl_mx_addr_t* mx_peer; - /** the address as reported by the peer */ - - mx_endpoint_addr_t mx_peer_addr; - /** the remote MX endpoint address */ - - -}; -typedef struct mca_mtl_mx_endpoint_t mca_mtl_mx_endpoint_t; -OBJ_CLASS_DECLARATION(mca_mtl_mx_endpoint); - -mca_mtl_mx_endpoint_t* mca_mtl_mx_endpoint_create(ompi_proc_t*); - - -END_C_DECLS -#endif diff --git a/ompi/mca/mtl/mx/mtl_mx_probe.c b/ompi/mca/mtl/mx/mtl_mx_probe.c deleted file mode 100644 index af30f3451b..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_probe.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2010 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "ompi/communicator/communicator.h" -#include "ompi/mca/mtl/base/base.h" - -#include "mtl_mx.h" -#include "mtl_mx_types.h" -#include "mtl_mx_request.h" - - -int -ompi_mtl_mx_iprobe(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t *comm, - int src, - int tag, - int *flag, - struct ompi_status_public_t *status) -{ - uint32_t result; - mx_return_t ret; - mx_status_t mx_status; - uint64_t match_bits; - uint64_t mask_bits; - - MX_SET_RECV_BITS(match_bits, - mask_bits, - comm->c_contextid, - src, - tag); - - ret = mx_iprobe(ompi_mtl_mx.mx_endpoint, - match_bits, - mask_bits, - &mx_status, - &result); - if (MX_SUCCESS != ret) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_iprobe (error %s)\n", mx_strerror(ret)); - return OMPI_ERROR; - } - - if (result) { - if(MPI_STATUS_IGNORE != status) { - MX_GET_SRC(mx_status.match_info, status->MPI_SOURCE); - MX_GET_TAG(mx_status.match_info, status->MPI_TAG); - status->_ucount = mx_status.msg_length; - } - *flag = 1; - } else { - *flag = 0; - } - - return OMPI_SUCCESS; -} - - -int -ompi_mtl_mx_improbe(struct mca_mtl_base_module_t *mtl, - struct ompi_communicator_t *comm, - int src, - int tag, - int *matched, - struct ompi_message_t **message, - struct ompi_status_public_t *status) -{ - return OMPI_ERR_NOT_IMPLEMENTED; -} diff --git a/ompi/mca/mtl/mx/mtl_mx_recv.c b/ompi/mca/mtl/mx/mtl_mx_recv.c deleted file mode 100644 index 1d8e5682ab..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_recv.c +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#include "ompi_config.h" -#include "ompi/communicator/communicator.h" -#include "opal/datatype/opal_convertor.h" -#include "ompi/mca/mtl/base/mtl_base_datatype.h" - -#include "mtl_mx.h" -#include "mtl_mx_types.h" -#include "mtl_mx_request.h" - -int -ompi_mtl_mx_irecv(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t *comm, - int src, - int tag, - struct opal_convertor_t *convertor, - struct mca_mtl_request_t *mtl_request) -{ - int ret; - mx_return_t mx_return; - mca_mtl_mx_request_t * mtl_mx_request = (mca_mtl_mx_request_t*) mtl_request; - uint64_t match_bits; - uint64_t mask_bits; - size_t length; - - ret = ompi_mtl_datatype_recv_buf(convertor, - &mtl_mx_request->mx_segment[0].segment_ptr, - &length, - &mtl_mx_request->free_after); - - mtl_mx_request->mx_segment[0].segment_length = length; - mtl_mx_request->convertor = convertor; - mtl_mx_request->type = OMPI_MTL_MX_IRECV; - - if(OMPI_SUCCESS != ret) return ret; - - - MX_SET_RECV_BITS(match_bits, - mask_bits, - comm->c_contextid, - src, - tag); - - OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, - "recv bits: 0x%016" PRIu64 " 0x%016" PRIu64 "\n", - match_bits, mask_bits)); - - mx_return = mx_irecv( ompi_mtl_mx.mx_endpoint, - mtl_mx_request->mx_segment, - 1, - match_bits, - mask_bits, - mtl_mx_request, - &mtl_mx_request->mx_request); - if(mx_return != MX_SUCCESS) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_irecv (error %s)\n", mx_strerror(mx_return)); - return OMPI_ERROR; - } - - return OMPI_SUCCESS; -} - - -int -ompi_mtl_mx_imrecv(struct mca_mtl_base_module_t* mtl, - struct opal_convertor_t *convertor, - struct ompi_message_t **message, - struct mca_mtl_request_t *mtl_request) -{ - return OMPI_ERR_NOT_IMPLEMENTED; -} diff --git a/ompi/mca/mtl/mx/mtl_mx_request.h b/ompi/mca/mtl/mx/mtl_mx_request.h deleted file mode 100644 index a92bf69bca..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_request.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OMPI_MTL_MX_REQUEST_H -#define OMPI_MTL_MX_REQUEST_H - -#include "opal/datatype/opal_convertor.h" - - -typedef enum { - OMPI_MTL_MX_ISEND, - OMPI_MTL_MX_IRECV -} mca_mtl_mx_request_type_t; - -struct mca_mtl_mx_request_t { - struct mca_mtl_request_t super; - mx_request_t mx_request; - mx_segment_t mx_segment[1]; - struct opal_convertor_t *convertor; - bool free_after; - mca_mtl_mx_request_type_t type; -}; -typedef struct mca_mtl_mx_request_t mca_mtl_mx_request_t; - -#endif diff --git a/ompi/mca/mtl/mx/mtl_mx_send.c b/ompi/mca/mtl/mx/mtl_mx_send.c deleted file mode 100644 index 22cc00d50b..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_send.c +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "ompi_config.h" - -#include "opal/prefetch.h" - -#include "ompi/communicator/communicator.h" -#include "opal/datatype/opal_convertor.h" - -#include "mtl_mx.h" -#include "mtl_mx_types.h" -#include "mtl_mx_request.h" -#include "ompi/mca/mtl/base/mtl_base_datatype.h" - -int -ompi_mtl_mx_send(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t* comm, - int dest, - int tag, - struct opal_convertor_t *convertor, - mca_pml_base_send_mode_t mode) -{ - mx_return_t mx_return; - uint64_t match_bits; - mca_mtl_mx_request_t mtl_mx_request; - size_t length; - mx_status_t mx_status; - uint32_t result; - ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest ); - mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL]; - char* where; - - assert(mtl == &ompi_mtl_mx.super); - - MX_SET_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag); - - ompi_mtl_datatype_pack(convertor, - &mtl_mx_request.mx_segment[0].segment_ptr, - &length, - &mtl_mx_request.free_after); - - mtl_mx_request.mx_segment[0].segment_length = length; - mtl_mx_request.convertor = convertor; - mtl_mx_request.type = OMPI_MTL_MX_ISEND; - - OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, - "issend bits: 0x%016" PRIu64 "\n", - match_bits)); - - if(mode == MCA_PML_BASE_SEND_SYNCHRONOUS) { - mx_return = mx_issend( ompi_mtl_mx.mx_endpoint, - mtl_mx_request.mx_segment, - 1, - mx_endpoint->mx_peer_addr, - match_bits, - &mtl_mx_request, - &mtl_mx_request.mx_request - ); - where = "mx_issend"; - } else { - mx_return = mx_isend( ompi_mtl_mx.mx_endpoint, - mtl_mx_request.mx_segment, - 1, - mx_endpoint->mx_peer_addr, - match_bits, - &mtl_mx_request, - &mtl_mx_request.mx_request - ); - where = "mx_isend"; - } - if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) { - char peer_name[MX_MAX_HOSTNAME_LEN]; - if(MX_SUCCESS != mx_nic_id_to_hostname( mx_endpoint->mx_peer->nic_id, peer_name)) { - sprintf( peer_name, "unknown %lx nic_id", (long)mx_endpoint->mx_peer->nic_id ); - } - opal_output(ompi_mtl_base_framework.framework_output, "Error in %s (error %s) sending to %s\n", - where, mx_strerror(mx_return), peer_name); - - /* Free buffer if needed */ - if(mtl_mx_request.free_after) { - free(mtl_mx_request.mx_segment[0].segment_ptr); - } - return OMPI_ERROR; - } - - do { - mx_return = mx_test(ompi_mtl_mx.mx_endpoint, - &mtl_mx_request.mx_request, - &mx_status, - &result); - if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) { - opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_wait (error %s)\n", mx_strerror(mx_return)); - abort(); - } - if( OPAL_UNLIKELY(result && mx_status.code != MX_STATUS_SUCCESS) ) { - opal_output(ompi_mtl_base_framework.framework_output, - "Error in ompi_mtl_mx_send, mx_wait returned something other than MX_STATUS_SUCCESS: mx_status.code = %d.\n", - mx_status.code); - abort(); - } - } while(!result); - - /* Free buffer if needed */ - if(mtl_mx_request.free_after) { - free(mtl_mx_request.mx_segment[0].segment_ptr); - } - - return OMPI_SUCCESS; -} - -int -ompi_mtl_mx_isend(struct mca_mtl_base_module_t* mtl, - struct ompi_communicator_t* comm, - int dest, - int tag, - struct opal_convertor_t *convertor, - mca_pml_base_send_mode_t mode, - bool blocking, - mca_mtl_request_t * mtl_request) -{ - mx_return_t mx_return; - uint64_t match_bits; - mca_mtl_mx_request_t * mtl_mx_request = (mca_mtl_mx_request_t*) mtl_request; - size_t length; - ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest ); - mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL]; - char* where; - - assert(mtl == &ompi_mtl_mx.super); - - MX_SET_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag); - - ompi_mtl_datatype_pack(convertor, - &mtl_mx_request->mx_segment[0].segment_ptr, - &length, - &mtl_mx_request->free_after); - mtl_mx_request->mx_segment[0].segment_length = length; - mtl_mx_request->convertor = convertor; - mtl_mx_request->type = OMPI_MTL_MX_ISEND; - - OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, - "issend bits: 0x%016" PRIu64 "\n", match_bits)); - - if(mode == MCA_PML_BASE_SEND_SYNCHRONOUS) { - mx_return = mx_issend( ompi_mtl_mx.mx_endpoint, - mtl_mx_request->mx_segment, - 1, - mx_endpoint->mx_peer_addr, - match_bits, - mtl_mx_request, - &mtl_mx_request->mx_request - ); - where = "mx_issend"; - } else { - mx_return = mx_isend( ompi_mtl_mx.mx_endpoint, - mtl_mx_request->mx_segment, - 1, - mx_endpoint->mx_peer_addr, - match_bits, - mtl_mx_request, - &mtl_mx_request->mx_request - ); - where = "mx_isend"; - } - if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) { - char peer_name[MX_MAX_HOSTNAME_LEN]; - if(MX_SUCCESS != mx_nic_id_to_hostname( mx_endpoint->mx_peer->nic_id, peer_name)) { - sprintf( peer_name, "unknown %lx nic_id", (long)mx_endpoint->mx_peer->nic_id ); - } - opal_output(ompi_mtl_base_framework.framework_output, "Error in %s (error %s) sending to %s\n", - where, mx_strerror(mx_return), peer_name); - return OMPI_ERROR; - } - return OMPI_SUCCESS; -} diff --git a/ompi/mca/mtl/mx/mtl_mx_types.h b/ompi/mca/mtl/mx/mtl_mx_types.h deleted file mode 100644 index d5cfc5268a..0000000000 --- a/ompi/mca/mtl/mx/mtl_mx_types.h +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2007 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2006 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MTL_MX_TYPES_H_HAS_BEEN_INCLUDED -#define MTL_MX_TYPES_H_HAS_BEEN_INCLUDED - -#include "ompi_config.h" -#include "mtl_mx.h" - -#include "ompi/mca/mtl/mtl.h" -#include "ompi/mca/mtl/base/base.h" -#include "mtl_mx_endpoint.h" - -#include "myriexpress.h" - - -BEGIN_C_DECLS - -/** - * MTL Module Interface - */ -struct mca_mtl_mx_module_t { - mca_mtl_base_module_t super; /**< base MTL interface */ - int32_t mx_unexp_queue_max; /**< maximium size of the MX unexpected message queue */ - int32_t mx_filter; /**< user assigned value used to filter incomming messages */ - int32_t mx_timeout; - int32_t mx_retries; - int32_t mx_support_sharedmem; - int mx_board_num; - int mx_endpoint_num; - mx_endpoint_t mx_endpoint; /**< mx data structure for local endpoint */ - mx_endpoint_addr_t mx_endpoint_addr; /**< mx data structure for local endpoint address */ - mca_mtl_mx_addr_t mx_addr; -}; -typedef struct mca_mtl_mx_module_t mca_mtl_mx_module_t; - -extern mca_mtl_mx_module_t ompi_mtl_mx; - -struct mca_mtl_mx_component_t{ - mca_mtl_base_component_2_0_0_t super; /**< base MTL component */ -}; -typedef struct mca_mtl_mx_component_t mca_mtl_mx_component_t; - -OMPI_MODULE_DECLSPEC extern mca_mtl_mx_component_t mca_mtl_mx_component; - - -/* match/ignore bit manipulation - * - * 01234567 01234567 01234567 01234567 01234567 01234567 01234567 01234567 - * | | - * context id | source | message tag - * | | - */ - -#define MX_SOURCE_MASK 0x0000FFFF00000000ULL -#define MX_TAG_MASK 0x00000000FFFFFFFFULL - -#define MX_SOURCE_IGNR ~MX_SOURCE_MASK - /* we need to keep top bit (sign bit) of the tag - collectives use this to distinguish the message */ -#define MX_TAG_IGNR 0xFFFFFFFF80000000ULL - -/* get the tag from the bits */ -#define MX_GET_TAG(match_bits, tag) \ -{ \ - tag = (int) (match_bits & MX_TAG_MASK); \ -} - - -/* get the tag from the bits */ -#define MX_GET_SRC(match_bits, src) \ -{ \ - src = (int) ((match_bits & MX_SOURCE_MASK) >> 32); \ -} - -/* send posting */ -#define MX_SET_SEND_BITS(match_bits, contextid, source, tag) \ -{ \ - match_bits = contextid; \ - match_bits = (match_bits << 16); \ - match_bits |= source; \ - match_bits = (match_bits << 32); \ - match_bits |= (MX_TAG_MASK & tag); \ -} - -/* receive posting */ -#define MX_SET_RECV_BITS(match_bits, mask_bits, contextid, source, tag) \ -{ \ - match_bits = contextid; \ - match_bits = (match_bits << 16); \ - \ - if (MPI_ANY_SOURCE == source) { \ - mask_bits = MX_SOURCE_IGNR; \ - } else { \ - mask_bits = ~0; \ - match_bits |= source; \ - } \ - match_bits = (match_bits << 32); \ - \ - if (MPI_ANY_TAG == tag) { \ - mask_bits &= MX_TAG_IGNR; \ - } else { \ - match_bits |= (MX_TAG_MASK & tag); \ - } \ -} - - - -END_C_DECLS - -#endif /* MTL_MX_TYPES_H_HAS_BEEN_INCLUDED */ - diff --git a/ompi/mca/mtl/mx/post_configure.sh b/ompi/mca/mtl/mx/post_configure.sh deleted file mode 100644 index 187ae63f35..0000000000 --- a/ompi/mca/mtl/mx/post_configure.sh +++ /dev/null @@ -1 +0,0 @@ -DIRECT_CALL_HEADER="ompi/mca/mtl/mx/mtl_mx.h"