ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Per discussion @Chicago OMPI dev meeting Dec 2013: remove all MX support.

Просмотр исходного кода 
This commit was SVN r29873.
Этот коммит содержится в:
Jeff Squyres 2013-12-12 18:54:47 +00:00
родитель 0e81959aae
Коммит bac67e0d81
29 изменённых файлов: 0 добавлений и 4171 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

48
ompi/mca/btl/mx/Makefile.am
Просмотреть файл

@ -1,48 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CPPFLAGS = $(btl_mx_CPPFLAGS)
btl_mx_sources = btl_mx.c btl_mx.h btl_mx_component.c btl_mx_endpoint.c \
btl_mx_endpoint.h btl_mx_frag.c btl_mx_frag.h btl_mx_proc.c btl_mx_proc.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_ompi_btl_mx_DSO
component_noinst =
component_install = mca_btl_mx.la
else
component_noinst = libmca_btl_mx.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_btl_mx_la_SOURCES = $(btl_mx_sources)
mca_btl_mx_la_LIBADD = \
$(btl_mx_LIBS) \
$(top_ompi_builddir)/ompi/mca/common/mx/libmca_common_mx.la
mca_btl_mx_la_LDFLAGS = -module -avoid-version $(btl_mx_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_btl_mx_la_SOURCES = $(btl_mx_sources)
libmca_btl_mx_la_LIBADD = $(btl_mx_LIBS)
libmca_btl_mx_la_LDFLAGS = -module -avoid-version $(btl_mx_LDFLAGS)

701
ompi/mca/btl/mx/btl_mx.c
Просмотреть файл

@ -1,701 +0,0 @@
/*
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2013 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/class/opal_bitmap.h"
#if OPAL_ENABLE_FT_CR == 1
#include "ompi/runtime/ompi_cr.h"
#endif
#include "btl_mx.h"
#include "btl_mx_frag.h"
#include "btl_mx_proc.h"
#include "btl_mx_endpoint.h"
#include "opal/datatype/opal_convertor.h"
#include "opal/prefetch.h"
/**
*
*/
int mca_btl_mx_add_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t** ompi_procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
int i, rc;
for( i = 0; i < (int) nprocs; i++ ) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_btl_mx_proc_t* mx_proc;
mca_btl_base_endpoint_t* mx_endpoint;
/**
* By default don't allow communications with self nor with any
* other processes on the same node. The BTL self and sm are
* supposed to take care of such communications.
*/
if( OPAL_PROC_ON_LOCAL_NODE(ompi_procs[i]->proc_flags) ) {
if( ompi_procs[i] == ompi_proc_local_proc ) {
if( 0 == mca_btl_mx_component.mx_support_self )
continue;
} else {
if( 0 == mca_btl_mx_component.mx_support_sharedmem )
continue;
}
}
if( NULL == (mx_proc = mca_btl_mx_proc_create(ompi_proc)) ) {
continue;
}
OPAL_THREAD_LOCK(&mx_proc->proc_lock);
/* The btl_proc datastructure is shared by all MX BTL
* instances that are trying to reach this destination.
* Cache the peer instance on the btl_proc.
*/
mx_endpoint = OBJ_NEW(mca_btl_mx_endpoint_t);
if(NULL == mx_endpoint) {
OPAL_THREAD_UNLOCK(&mx_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
mx_endpoint->endpoint_btl = mx_btl;
rc = mca_btl_mx_proc_insert( mx_proc, mx_endpoint );
if( rc != OMPI_SUCCESS ) {
OBJ_RELEASE(mx_endpoint);
OBJ_RELEASE(mx_proc);
OPAL_THREAD_UNLOCK(&mx_proc->proc_lock);
continue;
}
opal_bitmap_set_bit(reachable, i);
OPAL_THREAD_UNLOCK(&mx_proc->proc_lock);
peers[i] = mx_endpoint;
}
return OMPI_SUCCESS;
}
int mca_btl_mx_del_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t** procs,
struct mca_btl_base_endpoint_t** peers )
{
opal_output( 0, "MX BTL delete procs\n" );
/* TODO */
return OMPI_SUCCESS;
}
/**
* Register callback function to support send/recv semantics
*/
int mca_btl_mx_register( struct mca_btl_base_module_t* btl,
mca_btl_base_tag_t tag,
mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl;
#if 0
if( (NULL != cbfunc) && ( 0 == mca_btl_mx_component.mx_use_unexpected) ) {
#endif
if( NULL != cbfunc ) {
mca_btl_mx_frag_t* frag = NULL;
mx_return_t mx_return;
mx_segment_t mx_segment;
int i;
/* Post the receives if there is no unexpected handler */
for( i = 0; i < mca_btl_mx_component.mx_max_posted_recv; i++ ) {
MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag);
if( NULL == frag ) {
opal_output( 0, "mca_btl_mx_register: unable to allocate more eager fragments\n" );
if( 0 == i ) {
return OMPI_ERROR;
}
break; /* some fragments are already registered. Try to continue... */
}
frag->base.des_dst = (mca_btl_base_segment_t*)frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->mx_frag_list = NULL;
frag->type = MCA_BTL_MX_RECV;
mx_segment.segment_ptr = (void*)(frag+1);
mx_segment.segment_length = mx_btl->super.btl_eager_limit;
mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1,
0x01ULL, BTL_MX_RECV_MASK,
frag, &(frag->mx_request) );
if( MX_SUCCESS != mx_return ) {
opal_output( 0, "mca_btl_mx_register: mx_irecv failed with status %d (%s)\n",
mx_return, mx_strerror(mx_return) );
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
return OMPI_ERROR;
}
}
}
return OMPI_SUCCESS;
}
/**
* Allocate a segment.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*/
mca_btl_base_descriptor_t* mca_btl_mx_alloc( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
uint8_t order,
size_t size,
uint32_t flags)
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl;
mca_btl_mx_frag_t* frag = NULL;
MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->segment[0].base.seg_len =
size <= mx_btl->super.btl_eager_limit ?
size : mx_btl->super.btl_eager_limit ;
frag->segment[0].base.seg_addr.pval = (void*)(frag+1);
frag->base.des_src = (mca_btl_base_segment_t*)frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_flags = flags;
frag->base.order = MCA_BTL_NO_ORDER;
return (mca_btl_base_descriptor_t*)frag;
}
/**
* Return a segment
*/
int mca_btl_mx_free( struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des )
{
mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)des;
assert( MCA_BTL_MX_SEND == frag->type );
MCA_BTL_MX_FRAG_RETURN(btl, frag);
return OMPI_SUCCESS;
}
/**
* Pack data and return a descriptor that can be
* used for send/put.
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
*/
mca_btl_base_descriptor_t*
mca_btl_mx_prepare_src( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
mca_btl_mx_frag_t* frag = NULL;
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data;
max_data = btl->btl_eager_limit - reserve;
if( (*size) < max_data ) {
max_data = *size;
}
/* If the data is contiguous we can use directly the pointer
* to the user memory.
*/
if( 0 == opal_convertor_need_buffers(convertor) ) {
/**
* let the convertor figure out the correct pointer depending
* on the data layout
*/
iov.iov_base = NULL;
if( 0 == reserve ) {
MCA_BTL_MX_FRAG_ALLOC_USER(btl, frag);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
max_data = *size;
frag->base.des_src_cnt = 1;
} else {
MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->base.des_src_cnt = 2;
}
} else {
MCA_BTL_MX_FRAG_ALLOC_EAGER(mx_btl, frag);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->base.des_src_cnt = 1;
iov.iov_base = (void*)((unsigned char*)frag->segment[0].base.seg_addr.pval + reserve);
}
iov.iov_len = max_data;
(void)opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
*size = max_data;
if( 1 == frag->base.des_src_cnt ) {
frag->segment[0].base.seg_len = reserve + max_data;
if( 0 == reserve )
frag->segment[0].base.seg_addr.pval = iov.iov_base;
} else {
frag->segment[0].base.seg_len = reserve;
frag->segment[1].base.seg_len = max_data;
frag->segment[1].base.seg_addr.pval = iov.iov_base;
}
frag->base.des_src = (mca_btl_base_segment_t*)frag->segment;
frag->base.des_flags = flags;
frag->base.order = MCA_BTL_NO_ORDER;
return &frag->base;
}
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
mca_btl_base_descriptor_t* mca_btl_mx_prepare_dst( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
mca_btl_mx_frag_t* frag = NULL;
mx_return_t mx_return;
mx_segment_t mx_segment;
MCA_BTL_MX_FRAG_ALLOC_USER(btl, frag);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
frag->segment[0].base.seg_len = *size;
opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment[0].base.seg_addr.pval) );
frag->segment[0].key = (uint64_t)(intptr_t)frag;
mx_segment.segment_ptr = frag->segment[0].base.seg_addr.pval;
mx_segment.segment_length = frag->segment[0].base.seg_len;
mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1,
frag->segment[0].key, BTL_MX_PUT_MASK,
NULL, &(frag->mx_request) );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "Fail to re-register a fragment with the MX NIC ...\n" );
MCA_BTL_MX_FRAG_RETURN( btl, frag );
return NULL;
}
#ifdef HAVE_MX_FORGET
{
mx_return = mx_forget( mx_btl->mx_endpoint, &(frag->mx_request) );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_forget failed in mca_btl_mx_prepare_dst with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
return NULL;
}
}
#endif
/* Allow the fragment to be recycled using the mca_btl_mx_free function */
frag->type = MCA_BTL_MX_SEND;
frag->base.des_dst = (mca_btl_base_segment_t*)frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_flags = flags;
frag->base.order = MCA_BTL_NO_ORDER;
return &frag->base;
}
/**
* Initiate an asynchronous put.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
static int mca_btl_mx_put( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor )
{
mca_btl_mx_segment_t *src_seg = (mca_btl_mx_segment_t *) descriptor->des_src;
mca_btl_mx_segment_t *dst_seg = (mca_btl_mx_segment_t *) descriptor->des_dst;
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor;
mx_segment_t mx_segment[2];
mx_return_t mx_return;
uint32_t i = 0;
if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) {
if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERROR;
if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERR_OUT_OF_RESOURCE;
if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) )
return OMPI_ERROR;
}
frag->endpoint = endpoint;
frag->type = MCA_BTL_MX_SEND;
descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
do {
mx_segment[i].segment_ptr = src_seg[i].base.seg_addr.pval;
mx_segment[i].segment_length = src_seg[i].base.seg_len;
} while (++i < descriptor->des_src_cnt);
mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt,
endpoint->mx_peer_addr, dst_seg->key, frag,
&frag->mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/**
* Initiate an inline send to the peer. If failure then return a descriptor.
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
*/
static int mca_btl_mx_sendi( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct opal_convertor_t* convertor,
void* header,
size_t header_size,
size_t payload_size,
uint8_t order,
uint32_t flags,
mca_btl_base_tag_t tag,
mca_btl_base_descriptor_t** descriptor )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl;
size_t max_data;
if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) {
if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERROR;
if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERR_OUT_OF_RESOURCE;
if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) )
return OMPI_ERROR;
}
if( !opal_convertor_need_buffers(convertor) ) {
uint32_t mx_segment_count = 0;
uint64_t tag64 = 0x01ULL | (((uint64_t)tag) << 8);
mx_return_t mx_return;
mx_request_t mx_request;
mx_segment_t mx_segments[2], *mx_segment = mx_segments;
if( 0 != header_size ) {
mx_segment->segment_ptr = header;
mx_segment->segment_length = header_size;
mx_segment++;
mx_segment_count++;
}
if( 0 != payload_size ) {
struct iovec iov;
uint32_t iov_count = 1;
iov.iov_base = NULL;
iov.iov_len = payload_size;
(void)opal_convertor_pack( convertor, &iov, &iov_count, &max_data );
assert( max_data == payload_size );
mx_segment->segment_ptr = iov.iov_base;
mx_segment->segment_length = max_data;
mx_segment_count++;
}
mx_return = mx_isend( mx_btl->mx_endpoint, mx_segments, mx_segment_count,
endpoint->mx_peer_addr, tag64, NULL, &mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
return OMPI_ERROR;
}
#ifdef HAVE_MX_FORGET
{
uint32_t mx_result;
mx_return = mx_ibuffered( mx_btl->mx_endpoint, &mx_request, &mx_result );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_ibuffered failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
return OMPI_SUCCESS;
}
if( mx_result ) {
mx_return = mx_forget( mx_btl->mx_endpoint, &mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_forget failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
}
}
return OMPI_SUCCESS;
}
#endif
}
/* No optimization on this path. Just allocate a descriptor and return it
* to the user.
*/
*descriptor = mca_btl_mx_alloc( btl, endpoint, order,
header_size + payload_size, flags );
return OMPI_ERR_RESOURCE_BUSY;
}
/**
* Initiate an asynchronous send.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transfered
* @param tag (IN) The tag value used to notify the peer.
*/
int mca_btl_mx_send( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag )
{
mca_btl_mx_segment_t *src_seg = (mca_btl_mx_segment_t *) descriptor->des_src;
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor;
mx_segment_t mx_segment[2];
mx_return_t mx_return;
uint64_t total_length = 0, tag64;
uint32_t i = 0;
int btl_ownership = (descriptor->des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) {
if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERROR;
if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERR_OUT_OF_RESOURCE;
if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) )
return OMPI_ERROR;
}
frag->endpoint = endpoint;
frag->type = MCA_BTL_MX_SEND;
do {
mx_segment[i].segment_ptr = src_seg[i].base.seg_addr.pval;
mx_segment[i].segment_length = src_seg[i].base.seg_len;
total_length += src_seg[i].base.seg_len;
} while (++i < descriptor->des_src_cnt);
tag64 = 0x01ULL | (((uint64_t)tag) << 8);
mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt,
endpoint->mx_peer_addr, tag64, frag, &frag->mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
return OMPI_ERROR;
}
#ifdef HAVE_MX_FORGET
{
uint32_t mx_result;
mx_return = mx_ibuffered( mx_btl->mx_endpoint, &(frag->mx_request), &mx_result );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_ibuffered failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_ERROR;
}
if( mx_result ) {
mx_return = mx_forget( mx_btl->mx_endpoint, &(frag->mx_request) );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_forget failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_SUCCESS;
}
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS);
}
if( btl_ownership ) {
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
}
return 1;
}
}
#endif
if( 2048 > total_length ) {
mx_status_t mx_status;
uint32_t mx_result;
/* let's check for completness */
mx_return = mx_test( mx_btl->mx_endpoint, &(frag->mx_request),
&mx_status, &mx_result );
if( OPAL_LIKELY(MX_SUCCESS == mx_return) ) {
if( mx_result ) {
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS);
}
if( btl_ownership ) {
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
}
return 1;
}
}
}
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_SUCCESS;
}
/*
* Cleanup/release module resources.
*/
int mca_btl_mx_finalize( struct mca_btl_base_module_t* btl )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl;
if( NULL != mx_btl->mx_endpoint )
mx_close_endpoint(mx_btl->mx_endpoint);
OBJ_DESTRUCT( &mx_btl->mx_lock );
OBJ_DESTRUCT( &mx_btl->mx_peers );
free(mx_btl);
return OMPI_SUCCESS;
}
#if OPAL_ENABLE_FT_CR == 0
int mca_btl_mx_ft_event(int state) {
return OMPI_SUCCESS;
}
#else
int mca_btl_mx_ft_event(int state) {
mca_btl_mx_module_t* mx_btl;
int i;
if(OPAL_CRS_CHECKPOINT == state) {
/* Continue must reconstruct the routes (including modex), since we
* have to tear down the devices completely.
* We have to do this because the MX driver can be checkpointed, but
* cannot be restarted with BLCR due to an mmap problem. If we do not
* close MX then BLCR throws the following error in /var/log/messages:
* kernel: do_mmap(<file>, 00002aaab0aac000, 0000000000400000, ...) failed: ffffffffffffffff
* kernel: vmadump: mmap failed: /dev/mx0
* kernel: blcr: thaw_threads returned error, aborting. -1
* JJH: It may be possible to, instead of restarting the entire driver, just reconnect endpoints
*/
orte_cr_continue_like_restart = true;
for( i = 0; i < mca_btl_mx_component.mx_num_btls; i++ ) {
mx_btl = mca_btl_mx_component.mx_btls[i];
if( NULL != mx_btl->mx_endpoint ) {
mx_close_endpoint(mx_btl->mx_endpoint);
mx_btl->mx_endpoint = NULL;
}
}
}
else if(OPAL_CRS_CONTINUE == state) {
;
}
else if(OPAL_CRS_RESTART == state) {
;
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}
#endif /* OPAL_ENABLE_FT_CR */
mca_btl_mx_module_t mca_btl_mx_module = {
{
&mca_btl_mx_component.super,
0, /* max size of first fragment */
0, /* min send fragment size */
0, /* max send fragment size */
0, /* btl_rdma_pipeline_send_length */
0, /* btl_rdma_pipeline_frag_size */
0, /* btl_min_rdma_pipeline_size */
0, /* exclusivity */
0, /* latency */
0, /* bandwidth */
0, /* segment size */
MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT, /* flags */
mca_btl_mx_add_procs,
mca_btl_mx_del_procs,
mca_btl_mx_register,
mca_btl_mx_finalize,
mca_btl_mx_alloc,
mca_btl_mx_free,
mca_btl_mx_prepare_src,
mca_btl_mx_prepare_dst,
mca_btl_mx_send,
mca_btl_mx_sendi, /* send immediate */
mca_btl_mx_put, /* put */
NULL, /* get */
mca_btl_base_dump,
NULL, /* mpool */
NULL, /* register error */
mca_btl_mx_ft_event
}
};

304
ompi/mca/btl/mx/btl_mx.h
Просмотреть файл

@ -1,304 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_MX_H
#define MCA_PTL_MX_H
#include "ompi_config.h"
#include <sys/types.h>
#include <string.h>
/* Open MPI includes */
#include "ompi/class/ompi_free_list.h"
#include "opal/mca/event/event.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include <myriexpress.h>
#ifdef HAVE_MX_EXTENSIONS_H
#include <mx_extensions.h>
#endif /* HAVE_MX_EXTENSIONS_H */
BEGIN_C_DECLS
/**
* The mask used for receive and for the PUT protocol
*/
#define BTL_MX_RECV_MASK 0x00000000000000ffULL
#define BTL_MX_PUT_MASK 0xffffffffffffffffULL
/**
* MX BTL component.
*/
struct mca_btl_mx_component_t {
mca_btl_base_component_2_0_0_t super; /**< base BTL component */
int32_t mx_num_btls;
int32_t mx_max_btls;
/**< number of hcas available to the MX component */
struct mca_btl_mx_module_t** mx_btls;
/**< array of available BTL modules */
int32_t mx_free_list_num;
/**< initial size of free lists */
int32_t mx_free_list_max;
/**< maximum size of free lists */
int32_t mx_max_posted_recv;
/**< number of posted receives on each NIC */
int32_t mx_free_list_inc;
/**< number of elements to alloc when growing free lists */
int32_t mx_support_sharedmem;
/**< true if we want to activate the MX support for shared memory */
int32_t mx_support_self;
/**< true if we want to activate the MX support for self communications */
int32_t mx_bonding;
/**< true if MX is in charge of doing the device bonding */
int32_t mx_use_unexpected;
/**< true if Open MPI is allowed to register an unexpected handler with the MX library */
opal_list_t mx_procs; /**< list of mx proc structures */
int32_t mx_filter;
int32_t mx_timeout;
int32_t mx_connection_retries;
ompi_free_list_t mx_send_eager_frags; /**< free list of mx eager send fragments */
ompi_free_list_t mx_send_user_frags; /**< free list of mx user send fragments */
opal_mutex_t mx_lock; /**< lock for accessing module state */
#if MX_HAVE_MAPPER_STATE
char* mx_if_include; /**< include the following NICs */
char* mx_if_exclude; /**< Exclude the following NICs. These
* values are based on the last 6
* digits in hexadecimal of the MAC
* address of the mapper.
*/
#endif /* MX_HAVE_MAPPER_STATE */
};
typedef struct mca_btl_mx_component_t mca_btl_mx_component_t;
OMPI_MODULE_DECLSPEC extern mca_btl_mx_component_t mca_btl_mx_component;
/**
* BTL Module Interface.
* Each BTL correspond to a high level vision of a network interface. The
* current version of the MX BTL is not able to handle stripping of the
* messages by itself. Therefore, it rely on the PML layer for that.
*/
struct mca_btl_mx_module_t {
mca_btl_base_module_t super; /**< base BTL interface */
mx_endpoint_t mx_endpoint; /**< local MX endpoint */
mx_endpoint_addr_t mx_endpoint_addr; /**< local MX endpoint address */
uint32_t mx_unique_network_id; /**< unique identifier for this BTL,
* based on the MAC address of the
* mapper used to route messages.
*/
opal_list_t mx_peers; /**< list of peers */
int32_t mx_posted_request; /**< number of posted MX request */
opal_mutex_t mx_lock; /**< lock for accessing module state */
};
typedef struct mca_btl_mx_module_t mca_btl_mx_module_t;
extern mca_btl_mx_module_t mca_btl_mx_module;
/**
* MX component initialization.
*
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
*/
extern mca_btl_base_module_t** mca_btl_mx_component_init(
int *num_btl_modules,
bool allow_multi_user_threads,
bool have_hidden_threads
);
/**
* MX component progress.
*/
extern int mca_btl_mx_component_progress(void);
/**
* Cleanup any resources held by the BTL.
*
* @param btl BTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_btl_mx_finalize(
struct mca_btl_base_module_t* btl
);
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN)
* @param nprocs (IN) Number of processes
* @param procs (IN) Set of processes
* @param peers (OUT) Set of (optional) peer addressing info.
* @param peers (IN/OUT) Set of processes that are reachable via this BTL.
* @return OMPI_SUCCESS or error status on failure.
*
*/
extern int mca_btl_mx_add_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable
);
/**
* PML->BTL notification of change in the process list.
*
* @param btl (IN) BTL instance
* @param nproc (IN) Number of processes.
* @param procs (IN) Set of processes.
* @param peers (IN) Set of peer data structures.
* @return Status indicating if cleanup was successful
*
*/
extern int mca_btl_mx_del_procs(
struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t** peers
);
/**
* Initiate an asynchronous send.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transfered
* @param tag (IN) The tag value used to notify the peer.
*/
extern int mca_btl_mx_send(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* btl_peer,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag
);
/**
* Register a callback function that is called on receipt
* of a fragment.
*
* @param btl (IN) BTL module
* @return Status indicating if registration was successful
*
*/
extern int mca_btl_mx_register(
struct mca_btl_base_module_t* btl,
mca_btl_base_tag_t tag,
mca_btl_base_module_recv_cb_fn_t cbfunc,
void* cbdata);
/**
* Allocate a descriptor with a segment of the requested size.
* Note that the BTL layer may choose to return a smaller size
* if it cannot support the request.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*/
mca_btl_base_descriptor_t* mca_btl_mx_alloc( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
uint8_t order,
size_t size,
uint32_t flags);
/**
* Return a segment allocated by this BTL.
*
* @param btl (IN) BTL module
* @param descriptor (IN) Allocated descriptor.
*/
int mca_btl_mx_free( struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des );
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
mca_btl_base_descriptor_t*
mca_btl_mx_prepare_src( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags);
mca_btl_base_descriptor_t*
mca_btl_mx_prepare_dst( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* peer,
struct mca_mpool_base_registration_t*,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags);
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae
* @return OMPI_SUCCESS or failure status
*/
int mca_btl_mx_ft_event(int state);
END_C_DECLS
#endif

764
ompi/mca/btl/mx/btl_mx_component.c
Просмотреть файл

@ -1,764 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/prefetch.h"
#include "opal/util/opal_environ.h"
#include "ompi/constants.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "ompi/mca/common/mx/common_mx.h"
#include "btl_mx.h"
#include "btl_mx_frag.h"
#include "btl_mx_endpoint.h"
#if MX_HAVE_MAPPER_STATE
#include "mx_io.h"
#include "mx_internals/mx__fops.h"
#include "mx_internals/mx__driver_interface.h"
#endif /* MX_HAVE_MAPPER_STATE */
static int mca_btl_mx_component_register(void);
static int mca_btl_mx_component_open(void);
static int mca_btl_mx_component_close(void);
mca_btl_mx_component_t mca_btl_mx_component = {
{
/* First, the mca_base_component_t struct containing meta information
about the component itself */
{
MCA_BTL_BASE_VERSION_2_0_0,
"mx", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_btl_mx_component_open, /* component open */
mca_btl_mx_component_close, /* component close */
NULL, /* component query */
mca_btl_mx_component_register, /* component register */
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
},
mca_btl_mx_component_init,
mca_btl_mx_component_progress,
}
};
static int mca_btl_mx_component_verify(void)
{
if( 0 >= mca_btl_mx_component.mx_bonding ) {
char* value = getenv("MX_BONDING");
if( NULL == value ) {
mca_btl_mx_component.mx_bonding = 1;
} else {
mca_btl_mx_component.mx_bonding = atoi(value);
if( 0 >= mca_btl_mx_component.mx_bonding )
mca_btl_mx_component.mx_bonding = 1;
}
} else if( 1 != mca_btl_mx_component.mx_bonding ) {
char value[8];
snprintf( value, 8, "%d\n", mca_btl_mx_component.mx_bonding );
opal_setenv( "MX_BONDING", value, true, &environ );
}
return mca_btl_base_param_verify(&mca_btl_mx_module.super);
}
static int mca_btl_mx_component_register(void)
{
mca_btl_mx_component.mx_max_btls = 8;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "max_btls",
"Maximum number of accepted Myrinet cards",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_max_btls);
mca_btl_mx_component.mx_timeout = MX_INFINITE;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "timeout",
"Timeout for connections",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_timeout);
mca_btl_mx_component.mx_connection_retries = 20;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "retries",
"Number of retries for each new connection before considering the peer as unreacheable",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_connection_retries);
mca_btl_mx_component.mx_filter = 0xdeadbeef;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "filter",
"Unique ID for the application (used to connect to the peers)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_filter);
mca_btl_mx_component.mx_support_self = 0;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "self",
"Enable the MX support for self communications",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_support_self);
mca_btl_mx_component.mx_support_sharedmem = 0;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "shared_mem",
"Enable the MX support for shared memory",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_support_sharedmem);
mca_btl_mx_component.mx_bonding = 1;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "bonding",
"Integrate MX library bonding. Less than 0 is system default, everything else will set the MX_BONDING to the value.",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_bonding);
#ifdef HAVE_MX_REGISTER_UNEXP_HANDLER
mca_btl_mx_component.mx_use_unexpected = 0;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "register_unexp",
"Enable the MX support for the unexpected request handler (Open MPI matching)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_use_unexpected);
#endif /* HAVE_MX_REGISTER_UNEXP_HANDLER */
mca_btl_mx_component.mx_free_list_num = 8;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "free_list_num",
"Number of allocated default request",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_free_list_num);
mca_btl_mx_component.mx_free_list_inc = 32;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "free_list_inc",
"Number of request we allocate each time we miss some",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_free_list_inc);
mca_btl_mx_component.mx_free_list_max = 1024;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "free_list_max",
"Maximum number of request this device is allowed to allocate",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_free_list_max);
mca_btl_mx_component.mx_max_posted_recv = 16;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "max_posted_recv",
"Number of received posted in advance. Increasing this number for communication "
"bound application can lead to visible improvement in performances",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_max_posted_recv);
#if MX_HAVE_MAPPER_STATE
mca_btl_mx_component.mx_if_include = NULL;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "if_include",
"Myrinet card to use (last 6 digits from the mapper MAC)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_if_include);
mca_btl_mx_component.mx_if_exclude = NULL;
(void) mca_base_component_var_register(&mca_btl_mx_component.btl_version, "if_exclude",
"Myrinet card to avoid (last 6 digits from the mapper MAC)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_mx_component.mx_if_exclude);
#endif /* MX_HAVE_MAPPER_STATE */
mca_btl_mx_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
if( mca_btl_mx_component.mx_use_unexpected ) {
mca_btl_mx_module.super.btl_eager_limit = 1024;
mca_btl_mx_module.super.btl_rndv_eager_limit = 1024;
} else {
mca_btl_mx_module.super.btl_eager_limit = 4*1024;
mca_btl_mx_module.super.btl_rndv_eager_limit = 4*1024;
}
mca_btl_mx_module.super.btl_max_send_size = 32*1024;
mca_btl_mx_module.super.btl_rdma_pipeline_send_length = 256*1024;
mca_btl_mx_module.super.btl_rdma_pipeline_frag_size = 8*1024*1024;
mca_btl_mx_module.super.btl_min_rdma_pipeline_size = 0;
mca_btl_mx_module.super.btl_flags = (MCA_BTL_FLAGS_SEND_INPLACE |
MCA_BTL_FLAGS_PUT |
MCA_BTL_FLAGS_SEND |
MCA_BTL_FLAGS_RDMA_MATCHED);
mca_btl_mx_module.super.btl_seg_size = sizeof (mca_btl_mx_segment_t);
mca_btl_mx_module.super.btl_bandwidth = 2000;
mca_btl_mx_module.super.btl_latency = 5;
mca_btl_base_param_register(&mca_btl_mx_component.super.btl_version,
&mca_btl_mx_module.super);
return mca_btl_mx_component_verify();
}
/*
* Called by MCA framework to open the component, registers
* component parameters.
*/
static int mca_btl_mx_component_open(void)
{
int ret;
/* initialize state */
mca_btl_mx_component.mx_num_btls = 0;
mca_btl_mx_component.mx_btls = NULL;
mca_btl_mx_component.mx_use_unexpected = 0;
ret = mca_btl_mx_component_verify();
if (OMPI_SUCCESS != ret) {
return ret;
}
/* initialize objects */
OBJ_CONSTRUCT(&mca_btl_mx_component.mx_procs, opal_list_t);
if( 0 == mca_btl_mx_component.mx_support_sharedmem )
opal_setenv( "MX_DISABLE_SHMEM", "1", true, &environ );
if( 0 == mca_btl_mx_component.mx_support_self )
opal_setenv( "MX_DISABLE_SELF", "1", true, &environ );
/* Force the long pipeline (up to 4Kb fragments) */
opal_setenv( "MX_PIPELINE_LOG", "0", true, &environ );
return OMPI_SUCCESS;
}
/*
* component cleanup - sanity checking of queue lengths
*/
static int mca_btl_mx_component_close(void)
{
if(OMPI_SUCCESS != ompi_common_mx_finalize()) {
return OMPI_ERROR;
}
if( NULL == mca_btl_mx_component.mx_btls )
return OMPI_SUCCESS;
/* release resources */
OBJ_DESTRUCT(&mca_btl_mx_component.mx_procs);
OBJ_DESTRUCT(&mca_btl_mx_component.mx_send_eager_frags);
OBJ_DESTRUCT(&mca_btl_mx_component.mx_send_user_frags);
OBJ_DESTRUCT(&mca_btl_mx_component.mx_procs);
OBJ_DESTRUCT(&mca_btl_mx_component.mx_lock);
#if MX_HAVE_MAPPER_STATE
if( NULL != mca_btl_mx_component.mx_if_include ) {
free( mca_btl_mx_component.mx_if_include );
mca_btl_mx_component.mx_if_include = NULL;
}
if( NULL != mca_btl_mx_component.mx_if_exclude ) {
free( mca_btl_mx_component.mx_if_exclude );
mca_btl_mx_component.mx_if_exclude = NULL;
}
#endif /* MX_HAVE_MAPPER_STATE */
return OMPI_SUCCESS;
}
static int __counter = 0;
#ifdef HAVE_MX_REGISTER_UNEXP_HANDLER
/**
* In order to avoid useless memcpy, the unexpected handler will be called
* by the MX library before doing any match in the MX internal queues. Here
* we have a chance to match the message using our own matching logic from
* the PML. If the match is realized, we will return MX_RECV_FINISHED (the
* MX request will vanish in the MX library). If the match do not succeed
* MX_RECV_CONTINUE have to be returned and the MX library will do the
* match itself.
*/
static mx_unexp_handler_action_t
mca_btl_mx_unexpected_handler( void *context, mx_endpoint_addr_t source,
uint64_t match_value, uint32_t length,
void * data_if_available )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)context;
mca_btl_active_message_callback_t* reg;
mca_btl_base_tag_t tag;
mca_btl_base_descriptor_t descriptor;
mca_btl_base_segment_t segment;
if( 0 == __counter ) {
return MX_RECV_CONTINUE;
}
/*opal_output( 0, "Get unexpected handler context %p source %lld match_value %lld\n"
"\tlength %d data %p\n", context, source.stuff[0], match_value, length,
data_if_available );*/
if( !(0x01 & match_value) ) {
return MX_RECV_CONTINUE;
}
tag = (match_value >> 8) & 0xff;
reg = mca_btl_base_active_message_trigger + tag;
segment.seg_addr.pval = data_if_available;
segment.seg_len = length;
descriptor.des_dst = &segment;
descriptor.des_dst_cnt = 1;
reg->cbfunc( &(mx_btl->super), tag, &descriptor, reg->cbdata );
return MX_RECV_FINISHED;
}
#endif /* HAVE_MX_REGISTER_UNEXP_HANDLER */
/*
* Create and initialize an MX BTL module, where each module
* represents a specific NIC or a specific bonded set of NICS.
*/
static mca_btl_mx_module_t* mca_btl_mx_create(uint32_t board_num)
{
mca_btl_mx_module_t* mx_btl;
mx_endpoint_t mx_endpoint;
mx_endpoint_addr_t mx_endpoint_addr;
mx_return_t status;
uint32_t endpoint_id, mx_unique_network_id = 0;
uint64_t nic_id;
/* open local endpoint */
status = mx_open_endpoint( board_num, MX_ANY_ENDPOINT,
mca_btl_mx_component.mx_filter,
NULL, 0, &mx_endpoint);
if(status != MX_SUCCESS) {
opal_output( 0, "mca_btl_mx_init: mx_open_endpoint() failed with status %d (%s)\n",
status, mx_strerror(status) );
return NULL;
}
/* query the endpoint address */
if((status = mx_get_endpoint_addr( mx_endpoint,
&mx_endpoint_addr)) != MX_SUCCESS) {
opal_output( 0, "mca_btl_mx_init: mx_get_endpoint_addr() failed with status %d (%s)\n",
status, mx_strerror(status) );
mx_close_endpoint(mx_endpoint);
return NULL;
}
status = mx_decompose_endpoint_addr(mx_endpoint_addr, &nic_id, &endpoint_id);
if( MX_SUCCESS != status ) {
opal_output( 0, "mca_btl_mx_init: mx_decompose_endpoint_addr() failed with status %d (%s)\n",
status, mx_strerror(status) );
mx_close_endpoint(mx_endpoint);
return NULL;
}
status = mx_nic_id_to_board_number(nic_id, &board_num);
if( MX_SUCCESS != status ) {
opal_output( 0, "mca_btl_mx_init: mx_nic_id_to_board_number() failed with status %d (%s)\n",
status, mx_strerror(status) );
mx_close_endpoint(mx_endpoint);
return NULL;
}
#if MX_HAVE_MAPPER_STATE
{
mx_endpt_handle_t endp_handle;
mx_mapper_state_t ms;
char mapper_mac[7], *where;
status = mx_open_board( board_num, &endp_handle );
if( MX_SUCCESS != status ) {
opal_output( 0, "Unable to open board %d: %s\n", board_num, mx_strerror(status) );
mx_close_endpoint(mx_endpoint);
return NULL;
}
ms.board_number = board_num;
ms.iport = 0;
status = mx__get_mapper_state( endp_handle, &ms );
if( MX_SUCCESS != status ) {
opal_output( 0, "get_mapper_state failed for board %d: %s\n",
board_num, mx_strerror(status) );
mx_close_endpoint(mx_endpoint);
return NULL;
}
/* Keep the first 4 bytes for the network speed */
mx_unique_network_id = ((ms.mapper_mac[3] << 16) +
(ms.mapper_mac[4] << 8) +
(ms.mapper_mac[5]));
/* Try to figure out if we are allowed to use this network */
snprintf( mapper_mac, 7, "%6x", mx_unique_network_id );
if( (NULL != mca_btl_mx_component.mx_if_exclude) &&
(NULL != (where = strstr(mca_btl_mx_component.mx_if_exclude, mapper_mac))) ) {
mx_close_endpoint(mx_endpoint);
return NULL;
}
else if( (NULL != mca_btl_mx_component.mx_if_include) &&
(NULL == (where = strstr(mca_btl_mx_component.mx_if_include, mapper_mac))) ) {
mx_close_endpoint(mx_endpoint);
return NULL;
}
}
#endif /* MX_HAVE_MAPPER_STATE */
mx_btl = malloc(sizeof(mca_btl_mx_module_t));
if( NULL == mx_btl ) {
opal_output( 0, "mca_btl_mx_init: unable to allocate %lu bytes of memory\n",
sizeof(mca_btl_mx_module_t) );
mx_close_endpoint(mx_endpoint);
return NULL;
}
/* copy over default settings */
memcpy( mx_btl, &mca_btl_mx_module, sizeof(mca_btl_mx_module_t) );
OBJ_CONSTRUCT( &mx_btl->mx_peers, opal_list_t );
OBJ_CONSTRUCT( &mx_btl->mx_lock, opal_mutex_t );
mx_btl->mx_endpoint = mx_endpoint;
mx_btl->mx_endpoint_addr = mx_endpoint_addr;
mx_btl->super.btl_bandwidth = 2000; /* whatever */
mx_btl->super.btl_latency = 10;
#if defined(MX_HAS_NET_TYPE)
{
int value, board = board_num;
if( (status = mx_get_info( mx_btl->mx_endpoint, MX_LINE_SPEED,
&board, sizeof(board),
&value, sizeof(int))) != MX_SUCCESS ) {
opal_output( 0, "mx_get_info(MX_LINE_SPEED) failed with status %d (%s)\n",
status, mx_strerror(status) );
} else {
if( MX_SPEED_2G == value ) {
mx_unique_network_id |= 0xaa000000;
mx_btl->super.btl_bandwidth = 2000;
mx_btl->super.btl_latency = 5;
} else if( MX_SPEED_10G == value ) {
mx_unique_network_id |= 0xbb000000;
mx_btl->super.btl_bandwidth = 10000;
mx_btl->super.btl_latency = 3;
} else {
mx_unique_network_id |= 0xcc000000;
}
}
}
#endif /* defined(MX_HAS_NET_TYPE) */
mx_btl->super.btl_bandwidth *= mca_btl_mx_component.mx_bonding;
mx_btl->mx_unique_network_id = mx_unique_network_id;
#ifdef HAVE_MX_REGISTER_UNEXP_HANDLER
if( mca_btl_mx_component.mx_use_unexpected ) {
status = mx_register_unexp_handler( mx_btl->mx_endpoint, mca_btl_mx_unexpected_handler,
(void*)mx_btl );
if( MX_SUCCESS != status ) {
opal_output( 0, "mca_btl_mx_init: mx_register_unexp_handler() failed with status %d (%s)\n",
status, mx_strerror(status) );
/* switch to a mode without the unexpected handler */
mca_btl_mx_component.mx_use_unexpected = 0;
}
}
#endif /* HAVE_MX_REGISTER_UNEXP_HANDLER */
return mx_btl;
}
/*
* MX component initialization:
* - check if MX can be initialized.
* - and construct all static objects.
*/
mca_btl_base_module_t** mca_btl_mx_component_init(int *num_btl_modules,
bool enable_progress_threads,
bool enable_mpi_threads)
{
mca_btl_base_module_t** btls;
mx_return_t status;
uint32_t count;
int32_t i;
mca_btl_mx_addr_t *mx_addrs;
*num_btl_modules = 0;
/* First check if MX is available ... */
if( OMPI_SUCCESS != ompi_common_mx_initialize() ) {
ompi_modex_send(&mca_btl_mx_component.super.btl_version, NULL, 0);
return NULL;
}
/* get the number of card available on the system */
if( MX_SUCCESS != (status = mx_get_info( NULL, MX_NIC_COUNT, NULL, 0,
&mca_btl_mx_component.mx_num_btls,
sizeof(uint32_t))) ) {
opal_output( 0, "mca_btl_mx_component_init: mx_get_info(MX_NIC_COUNT) failed with status %d(%s)\n",
status, mx_strerror(status) );
return NULL;
}
/* Don't forget the bonding rules ... */
assert( mca_btl_mx_component.mx_bonding >= 1 );
mca_btl_mx_component.mx_num_btls /= mca_btl_mx_component.mx_bonding;
if (0 == mca_btl_mx_component.mx_num_btls) {
mca_btl_base_error_no_nics("Myrinet/MX", "NIC");
return NULL;
}
/* Limit ourselves to the number of devices requested by the users. */
if( mca_btl_mx_component.mx_num_btls > mca_btl_mx_component.mx_max_btls ) {
mca_btl_mx_component.mx_num_btls = mca_btl_mx_component.mx_max_btls;
}
/* initialize objects */
OBJ_CONSTRUCT(&mca_btl_mx_component.mx_send_eager_frags, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_mx_component.mx_send_user_frags, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_mx_component.mx_procs, opal_list_t);
OBJ_CONSTRUCT(&mca_btl_mx_component.mx_lock, opal_mutex_t);
ompi_free_list_init_new( &mca_btl_mx_component.mx_send_eager_frags,
sizeof(mca_btl_mx_frag_t) + mca_btl_mx_module.super.btl_eager_limit,
opal_cache_line_size,
OBJ_CLASS(mca_btl_mx_frag_t),
0,opal_cache_line_size,
mca_btl_mx_component.mx_free_list_num,
mca_btl_mx_component.mx_free_list_max,
mca_btl_mx_component.mx_free_list_inc,
NULL ); /* use default allocator */
ompi_free_list_init_new( &mca_btl_mx_component.mx_send_user_frags,
sizeof(mca_btl_mx_frag_t),
opal_cache_line_size,
OBJ_CLASS(mca_btl_mx_frag_t),
0,opal_cache_line_size,
mca_btl_mx_component.mx_free_list_num,
mca_btl_mx_component.mx_free_list_max,
mca_btl_mx_component.mx_free_list_inc,
NULL ); /* use default allocator */
/* intialize process hash table */
OBJ_CONSTRUCT( &mca_btl_mx_component.mx_procs, opal_list_t );
/* Now we know how many NIC are available on the system. We will create a BTL
* for each one and then give a pointer to the BTL to the upper level.
*/
mca_btl_mx_component.mx_btls = malloc( mca_btl_mx_component.mx_num_btls *
sizeof(mca_btl_base_module_t*) );
if( NULL == mca_btl_mx_component.mx_btls ) {
opal_output( 0, "MX BTL unable to allocate memory\n" );
return NULL;
}
mx_addrs = (mca_btl_mx_addr_t*)calloc( mca_btl_mx_component.mx_num_btls,
sizeof(mca_btl_mx_addr_t) );
if( NULL == mx_addrs ) {
opal_output( 0, "MX BTL unable to allocate memory\n" );
free(mca_btl_mx_component.mx_btls);
mca_btl_mx_component.mx_btls = NULL;
return NULL;
}
/* create a btl for each NIC */
for( i = count = 0; i < mca_btl_mx_component.mx_num_btls; i++ ) {
mca_btl_mx_module_t* mx_btl = mca_btl_mx_create(MX_ANY_NIC);
if( NULL == mx_btl ) {
continue;
}
status = mx_decompose_endpoint_addr( mx_btl->mx_endpoint_addr,
&(mx_addrs[count].nic_id),
&(mx_addrs[count].endpoint_id) );
if( MX_SUCCESS != status ) {
mca_btl_mx_finalize( &mx_btl->super );
continue;
}
mx_addrs[count].unique_network_id = mx_btl->mx_unique_network_id;
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
BTL_MX_ADDR_HTON(mx_addrs[count]);
#endif
mca_btl_mx_component.mx_btls[count] = mx_btl;
count++; /* one more succesfully initialized MX interface */
}
mca_btl_mx_component.mx_num_btls = count;
*num_btl_modules = count;
if( 0 == count ) {
/* No active BTL module */
free(mx_addrs);
free(mca_btl_mx_component.mx_btls);
mca_btl_mx_component.mx_btls = NULL;
return NULL;
}
/* publish the MX addresses via the MCA framework */
ompi_modex_send(&mca_btl_mx_component.super.btl_version, mx_addrs,
sizeof(mca_btl_mx_addr_t) * mca_btl_mx_component.mx_num_btls);
free( mx_addrs );
btls = malloc( mca_btl_mx_component.mx_num_btls * sizeof(mca_btl_base_module_t*) );
if( NULL == btls ) {
free( mca_btl_mx_component.mx_btls );
mca_btl_mx_component.mx_num_btls = 0; /* no active BTL modules */
return NULL;
}
memcpy( btls, mca_btl_mx_component.mx_btls,
mca_btl_mx_component.mx_num_btls*sizeof(mca_btl_mx_module_t*) );
return btls;
}
/*
* MX component progress.
*/
int mca_btl_mx_component_progress(void)
{
int32_t num_progressed = 0, i;
mx_status_t mx_status;
mx_return_t mx_return;
mx_request_t mx_request;
mca_btl_mx_frag_t* frag;
++__counter;
for( i = 0; i < mca_btl_mx_component.mx_num_btls; i++ ) {
mca_btl_mx_module_t* mx_btl = mca_btl_mx_component.mx_btls[i];
uint32_t mx_result = 0;
recheck_device:
mx_return = mx_ipeek( mx_btl->mx_endpoint, &mx_request, &mx_result );
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output( 0, "mca_btl_mx_component_progress: mx_ipeek() failed with status %d (%s)\n",
mx_return, mx_strerror(mx_return) );
continue;
}
if( OPAL_LIKELY(mx_result == 0) ) {
continue;
}
mx_return = mx_test( mx_btl->mx_endpoint, &mx_request, &mx_status, &mx_result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(0, "mca_btl_mx_progress: mx_test() failed with status %d (%s)\n",
mx_return, mx_strerror(mx_return));
continue;
}
/* on the mx_status we have now the pointer attached to the request.
* This pointer indicate which fragment we are working on. On the
* status we have the status of the operation, so we know what we
* are supposed to do next.
*/
frag = mx_status.context;
num_progressed++;
/* If the context is NULL then we are facing a send immediate request. Therefore,
* nothing special should be done, just keep going.
*/
if( NULL == frag ) goto recheck_device;
if( MCA_BTL_MX_SEND == frag->type ) { /* it's a send */
int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
/* call the completion callback */
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS );
}
if( btl_ownership ) {
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
}
goto recheck_device;
} else {
mca_btl_active_message_callback_t* reg;
mx_segment_t mx_segment;
uint8_t tag = (mx_status.match_info >> 8) & 0xff;
reg = mca_btl_base_active_message_trigger + tag;
frag->base.des_dst->seg_len = mx_status.msg_length;
reg->cbfunc( &(mx_btl->super), tag, &(frag->base), reg->cbdata );
/**
* The upper level extract the data from the fragment.
* Now we can register the fragment
* again with the MX BTL.
*/
mx_segment.segment_ptr = frag->base.des_dst->seg_addr.pval;
mx_segment.segment_length = mca_btl_mx_module.super.btl_eager_limit;
mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1,
0x01ULL, BTL_MX_RECV_MASK,
frag, &(frag->mx_request) );
if( MX_SUCCESS != mx_return ) {
opal_output( 0, "Fail to re-register a fragment with the MX NIC ... (%s)\n",
mx_strerror(mx_return) );
}
}
}
__counter--;
return num_progressed;
}
#if 0
{
int counters, board, i, value, *counters_value;
char text[MX_MAX_STR_LEN];
char *counters_name;
if( (status = mx_get_info( mx_btl->mx_endpoint, MX_PIO_SEND_MAX, NULL, 0,
&value, sizeof(int))) != MX_SUCCESS ) {
opal_output( 0, "mx_get_info(MX_PIO_SEND_MAX) failed with status %d (%s)\n",
status, mx_strerror(status) );
}
printf( "MX_PIO_SEND_MAX = %d\n", value );
if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COPY_SEND_MAX, NULL, 0,
&value, sizeof(int))) != MX_SUCCESS ) {
opal_output( 0, "mx_get_info(MX_COPY_SEND_MAX) failed with status %d (%s)\n",
status, mx_strerror(status) );
}
printf( "MX_COPY_SEND_MAX = %d\n", value );
board = 0;
if( (status = mx_get_info( mx_btl->mx_endpoint, MX_PRODUCT_CODE, &board, sizeof(int),
text, MX_MAX_STR_LEN)) != MX_SUCCESS ) {
opal_output( 0, "mx_get_info(MX_PRODUCT_CODE) failed with status %d (%s)\n",
status, mx_strerror(status) );
}
printf( "product code %s\n", text );
if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COUNTERS_COUNT, &board, sizeof(int),
&counters, sizeof(int))) != MX_SUCCESS ) {
opal_output( 0, "mx_get_info(MX_COUNTERS_COUNT) failed with status %d (%s)\n",
status, mx_strerror(status) );
}
printf( "counters = %d\n", counters );
counters_name = (char*)malloc( counters * MX_MAX_STR_LEN );
if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COUNTERS_LABELS, &board, sizeof(int),
counters_name, counters * MX_MAX_STR_LEN)) != MX_SUCCESS ) {
opal_output( 0, "mx_get_info(MX_COUNTERS_LABELS) failed with status %d (%s)\n",
status, mx_strerror(status) );
}
counters_value = (int*)malloc( counters * sizeof(int) );
if( (status = mx_get_info( mx_btl->mx_endpoint, MX_COUNTERS_VALUES, &board, sizeof(int),
counters_value, counters * sizeof(int))) != MX_SUCCESS ) {
opal_output( 0, "mx_get_info(MX_COUNTERS_VALUES) failed with status %d (%s)\n",
status, mx_strerror(status) );
}
for( i = 0; i < counters; i++ )
printf( "%d -> %s = %d\n", i, counters_name + i * MX_MAX_STR_LEN,
counters_value[i] );
free( counters_name );
free( counters_value );
}
#endif

55
ompi/mca/btl/mx/btl_mx_endpoint.c
Просмотреть файл

@ -1,55 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/time.h>
#include <time.h>
#include "ompi/types.h"
#include "btl_mx.h"
#include "btl_mx_endpoint.h"
#include "btl_mx_proc.h"
#include "btl_mx_frag.h"
/*
* Initialize state of the endpoint instance.
*
*/
static void mca_btl_mx_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
{
endpoint->endpoint_btl = NULL;
endpoint->endpoint_proc = NULL;
endpoint->status = MCA_BTL_MX_NOT_CONNECTED;
}
/*
* Destroy a endpoint
*
*/
static void mca_btl_mx_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
{
}
OBJ_CLASS_INSTANCE(
mca_btl_mx_endpoint_t,
opal_list_item_t,
mca_btl_mx_endpoint_construct,
mca_btl_mx_endpoint_destruct);

92
ompi/mca/btl/mx/btl_mx_endpoint.h
Просмотреть файл

@ -1,92 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_MX_ENDPOINT_H
#define MCA_BTL_MX_ENDPOINT_H
#include "opal/class/opal_list.h"
#include "opal/mca/event/event.h"
#include "btl_mx_frag.h"
#include "btl_mx.h"
#include <myriexpress.h>
BEGIN_C_DECLS
#define MCA_BTL_MX_NOT_CONNECTED 0x0000
#define MCA_BTL_MX_NOT_REACHEABLE 0x0001
#define MCA_BTL_MX_CONNECTION_PENDING 0x0002
#define MCA_BTL_MX_CONNECTED 0x0004
/**
* Structure used to publish MX information to peers
*/
struct mca_btl_mx_addr_t {
uint64_t nic_id;
uint32_t endpoint_id;
uint32_t unique_network_id; /* Unique identifier for each MX network */
};
typedef struct mca_btl_mx_addr_t mca_btl_mx_addr_t;
#define BTL_MX_ADDR_HTON(h) \
do { \
h.nic_id = hton64(h.nic_id); \
h.endpoint_id = htonl(h.endpoint_id); \
h.unique_network_id = htonl(h.unique_network_id); \
} while (0)
#define BTL_MX_ADDR_NTOH(h) \
do { \
h.nic_id = ntoh64(h.nic_id); \
h.endpoint_id = ntohl(h.endpoint_id); \
h.unique_network_id = ntohl(h.unique_network_id); \
} while (0)
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_btl_base_endpoint_t {
opal_list_item_t super;
struct mca_btl_mx_module_t* endpoint_btl;
/**< BTL instance that created this connection */
struct mca_btl_mx_proc_t* endpoint_proc;
/**< proc structure corresponding to endpoint */
struct mca_btl_mx_addr_t* mx_peer;
/** the address as reported by the peer */
mx_endpoint_addr_t mx_peer_addr;
/** the remote MX endpoint address */
int status; /**< status of the endpoint */
};
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
typedef mca_btl_base_endpoint_t mca_btl_mx_endpoint_t;
OBJ_CLASS_DECLARATION(mca_btl_mx_endpoint_t);
END_C_DECLS
#endif

71
ompi/mca/btl/mx/btl_mx_frag.c
Просмотреть файл

@ -1,71 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "btl_mx_frag.h"
static void mca_btl_mx_frag_common_constructor(mca_btl_mx_frag_t* frag)
{
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
}
static void mca_btl_mx_frag_eager_constructor(mca_btl_mx_frag_t* frag)
{
frag->size = mca_btl_mx_module.super.btl_eager_limit;
mca_btl_mx_frag_common_constructor(frag);
}
static void mca_btl_mx_frag_max_constructor(mca_btl_mx_frag_t* frag)
{
frag->size = mca_btl_mx_module.super.btl_max_send_size;
mca_btl_mx_frag_common_constructor(frag);
}
static void mca_btl_mx_frag_user_constructor(mca_btl_mx_frag_t* frag)
{
frag->size = 0;
mca_btl_mx_frag_common_constructor(frag);
}
OBJ_CLASS_INSTANCE(
mca_btl_mx_frag_t,
mca_btl_base_descriptor_t,
NULL,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_mx_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_mx_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_mx_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_mx_frag_max_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_mx_frag_user_t,
mca_btl_base_descriptor_t,
mca_btl_mx_frag_user_constructor,
NULL);

99
ompi/mca/btl/mx/btl_mx_frag.h
Просмотреть файл

@ -1,99 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2013 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_MX_FRAG_H
#define MCA_BTL_MX_FRAG_H
#include "ompi_config.h"
#include "ompi/class/ompi_free_list.h"
#include "btl_mx.h"
#define MCA_BTL_MX_SEND 0x01
#define MCA_BTL_MX_RECV 0x02
BEGIN_C_DECLS
struct mca_btl_mx_segment_t {
mca_btl_base_segment_t base;
uint64_t key;
};
typedef struct mca_btl_mx_segment_t mca_btl_mx_segment_t;
/**
* MX send framxent derived type.
*/
struct mca_btl_mx_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_mx_segment_t segment[2];
struct mca_btl_base_endpoint_t* endpoint;
uint8_t type;
mx_request_t mx_request;
size_t size;
ompi_free_list_t* mx_frag_list;
};
typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_mx_frag_t);
typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_btl_mx_frag_eager_t);
typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_max_t;
OBJ_CLASS_DECLARATION(mca_btl_mx_frag_max_t);
typedef struct mca_btl_mx_frag_t mca_btl_mx_frag_user_t;
OBJ_CLASS_DECLARATION(mca_btl_mx_frag_user_t);
/*
* Macros to allocate/return descriptors from module specific
* free list(s).
*/
#define MCA_BTL_MX_FRAG_ALLOC_EAGER(btl, frag) \
do { \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT( &mca_btl_mx_component.mx_send_eager_frags, item); \
if( OPAL_LIKELY(NULL != item) ) { \
frag = (mca_btl_mx_frag_t*) item; \
frag->mx_frag_list = &(mca_btl_mx_component.mx_send_eager_frags); \
frag->segment[0].base.seg_addr.pval = (void*)(frag+1); \
} \
} while(0)
#define MCA_BTL_MX_FRAG_ALLOC_USER(btl, frag) \
do { \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET_MT( &mca_btl_mx_component.mx_send_user_frags, item); \
if( OPAL_LIKELY(NULL != item) ) { \
frag = (mca_btl_mx_frag_t*) item; \
frag->mx_frag_list = &(mca_btl_mx_component.mx_send_user_frags); \
} \
} while(0)
#define MCA_BTL_MX_FRAG_RETURN(btl, frag) \
do { \
OMPI_FREE_LIST_RETURN_MT( frag->mx_frag_list, \
(ompi_free_list_item_t*)(frag)); \
} while(0)
END_C_DECLS
#endif

243
ompi/mca/btl/mx/btl_mx_proc.c
Просмотреть файл

@ -1,243 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "btl_mx.h"
#include "btl_mx_proc.h"
static void mca_btl_mx_proc_construct(mca_btl_mx_proc_t* proc);
static void mca_btl_mx_proc_destruct(mca_btl_mx_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_btl_mx_proc_t,
opal_list_item_t, mca_btl_mx_proc_construct,
mca_btl_mx_proc_destruct);
void mca_btl_mx_proc_construct(mca_btl_mx_proc_t* mx_proc)
{
mx_proc->proc_ompi = 0;
mx_proc->mx_peers_count = 0;
mx_proc->mx_peers = NULL;
mx_proc->mx_routing = NULL;
OBJ_CONSTRUCT(&mx_proc->proc_lock, opal_mutex_t);
/* add to list of all proc instance */
OPAL_THREAD_LOCK(&mca_btl_mx_component.mx_lock);
opal_list_append(&mca_btl_mx_component.mx_procs, &mx_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock);
}
/*
* Cleanup MX proc instance
*/
void mca_btl_mx_proc_destruct(mca_btl_mx_proc_t* mx_proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK(&mca_btl_mx_component.mx_lock);
opal_list_remove_item(&mca_btl_mx_component.mx_procs, &mx_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock);
/* release resources */
if( NULL != mx_proc->mx_peers ) {
free(mx_proc->mx_peers);
mx_proc->mx_peers = NULL;
}
if( NULL != mx_proc->mx_routing ) {
free(mx_proc->mx_routing);
mx_proc->mx_routing = NULL;
}
OBJ_DESTRUCT(&mx_proc->proc_lock);
}
/*
* Look for an existing MX process instances based on the associated
* ompi_proc_t instance.
*/
static mca_btl_mx_proc_t* mca_btl_mx_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_btl_mx_proc_t* mx_proc;
OPAL_THREAD_LOCK(&mca_btl_mx_component.mx_lock);
for( mx_proc = (mca_btl_mx_proc_t*)opal_list_get_first(&mca_btl_mx_component.mx_procs);
mx_proc != (mca_btl_mx_proc_t*)opal_list_get_end(&mca_btl_mx_component.mx_procs);
mx_proc = (mca_btl_mx_proc_t*)opal_list_get_next(mx_proc) ) {
if(mx_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock);
return mx_proc;
}
}
OPAL_THREAD_UNLOCK(&mca_btl_mx_component.mx_lock);
return NULL;
}
/**
* Create a MX process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_btl_mx_proc_t instance. We cache
* additional data (specifically the list of mca_btl_mx_endpoint_t instances,
* and published addresses) associated w/ a given destination on this
* datastructure.
*/
mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc)
{
mca_btl_mx_proc_t* module_proc = NULL;
mca_btl_mx_addr_t *mx_peers;
int i, j, rc, mx_peers_count, *mx_routing;
bool at_least_one_route = false;
size_t size;
/* Check if we have already created a MX proc
* structure for this ompi process */
module_proc = mca_btl_mx_proc_lookup_ompi(ompi_proc);
if( module_proc != NULL ) {
return module_proc; /* Gotcha! */
}
/* query for the peer address info */
rc = ompi_modex_recv( &mca_btl_mx_component.super.btl_version,
ompi_proc, (void*)&mx_peers, &size );
if( OMPI_SUCCESS != rc ) {
opal_output( 0, "mca_pml_base_modex_recv failed for peer %s",
OMPI_NAME_PRINT(&ompi_proc->proc_name) );
return NULL;
}
if( size < sizeof(mca_btl_mx_addr_t) ) { /* no available connection */
return NULL;
}
if( (size % sizeof(mca_btl_mx_addr_t)) != 0 ) {
opal_output( 0, "invalid mx address for peer %s",
OMPI_NAME_PRINT(&ompi_proc->proc_name) );
return NULL;
}
/* Let's see if we have a way to connect to the remote proc using MX.
* Without the routing information from the mapper, it is pretty
* to do this. Right now, we base this connection detection on the last
* 6 digits of the mapper MAC.
*/
mx_peers_count = size / sizeof(mca_btl_mx_addr_t);
mx_routing = (int*)malloc( mx_peers_count * sizeof(int) );
for( i = 0; i < mx_peers_count; mx_routing[i++] = -1 );
for( i = 0; i < mx_peers_count; i++ ) {
mca_btl_mx_module_t* mx_btl;
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
BTL_MX_ADDR_NTOH(mx_peers[rc]);
#endif
for( j = 0; j < mca_btl_mx_component.mx_num_btls; j++ ) {
mx_btl = mca_btl_mx_component.mx_btls[j];
if( mx_btl->mx_unique_network_id == mx_peers[j].unique_network_id ) {
/* There is at least one connection between these two nodes */
if( -1 == mx_routing[j] ) {
/* First connection */
mx_routing[j] = i;
at_least_one_route = true;
break;
}
/* If multiple remote endpoints match mine, we keep going. As a
* result we will match them in order, i.e. remote endpoint 0
* will be connected to local endpoint 0.
*/
}
}
}
if( false == at_least_one_route ) {
free(mx_routing);
return NULL;
}
module_proc = OBJ_NEW(mca_btl_mx_proc_t);
module_proc->proc_ompi = ompi_proc;
module_proc->mx_peers_count = mx_peers_count;
module_proc->mx_peers = mx_peers;
module_proc->mx_routing = mx_routing;
return module_proc;
}
/**
* Note that this routine must be called with the lock on the process
* already held. Insert a btl instance into the proc array and assign
* it an address.
*/
int mca_btl_mx_proc_insert( mca_btl_mx_proc_t* module_proc,
mca_btl_mx_endpoint_t* module_endpoint )
{
mca_btl_mx_module_t* mx_btl;
int btl_index, peer_endpoint_index;
for( btl_index = 0; btl_index < mca_btl_mx_component.mx_num_btls; btl_index++ ) {
mx_btl = mca_btl_mx_component.mx_btls[btl_index];
peer_endpoint_index = module_proc->mx_routing[btl_index];
if( (-1 != peer_endpoint_index) && (mx_btl == module_endpoint->endpoint_btl) ) {
module_endpoint->mx_peer = module_proc->mx_peers + peer_endpoint_index;
module_endpoint->endpoint_proc = module_proc;
return OMPI_SUCCESS;
}
}
module_proc->mx_peers_count = 0;
/**
* No Myrinet connectivity. Let the PML layer figure out another
* way to communicate with the peer.
*/
return OMPI_ERROR;
}
int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint )
{
int num_retry = 0;
mx_return_t mx_status;
mx_endpoint_addr_t mx_remote_addr;
module_endpoint->status = MCA_BTL_MX_CONNECTION_PENDING;
retry_connect:
mx_status = mx_connect( module_endpoint->endpoint_btl->mx_endpoint,
module_endpoint->mx_peer->nic_id, module_endpoint->mx_peer->endpoint_id,
mca_btl_mx_component.mx_filter, mca_btl_mx_component.mx_timeout, &mx_remote_addr );
if( MX_SUCCESS != mx_status ) {
if( MX_TIMEOUT == mx_status )
if( num_retry++ < mca_btl_mx_component.mx_connection_retries )
goto retry_connect;
{
char peer_name[MX_MAX_HOSTNAME_LEN];
if( MX_SUCCESS != mx_nic_id_to_hostname( module_endpoint->mx_peer->nic_id, peer_name ) )
sprintf( peer_name, "unknown %lx nic_id", (long)module_endpoint->mx_peer->nic_id );
opal_output( 0, "mx_connect fail for %s with key %x (error %s)\n\tUnique ID (local %x remote %x)\n",
peer_name, mca_btl_mx_component.mx_filter, mx_strerror(mx_status),
module_endpoint->endpoint_btl->mx_unique_network_id,
module_endpoint->mx_peer->unique_network_id );
}
module_endpoint->status = MCA_BTL_MX_NOT_REACHEABLE;
return OMPI_ERROR;
}
module_endpoint->mx_peer_addr = mx_remote_addr;
module_endpoint->status = MCA_BTL_MX_CONNECTED;
return OMPI_SUCCESS;
}

58
ompi/mca/btl/mx/btl_mx_proc.h
Просмотреть файл

@ -1,58 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_MX_PROC_H
#define MCA_BTL_MX_PROC_H
#include "opal/class/opal_object.h"
#include "ompi/proc/proc.h"
#include "btl_mx.h"
#include "btl_mx_endpoint.h"
BEGIN_C_DECLS
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each BTL instance that attempts to open a connection to the process.
*/
struct mca_btl_mx_proc_t {
opal_list_item_t super;
/**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
mca_btl_mx_addr_t *mx_peers; /**< peers addresses */
int mx_peers_count;
int* mx_routing; /**< peer routing information */
opal_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_btl_mx_proc_t mca_btl_mx_proc_t;
OBJ_CLASS_DECLARATION(mca_btl_mx_proc_t);
mca_btl_mx_proc_t* mca_btl_mx_proc_create(ompi_proc_t* ompi_proc);
int mca_btl_mx_proc_insert(mca_btl_mx_proc_t*, mca_btl_base_endpoint_t*);
int mca_btl_mx_proc_connect( mca_btl_mx_endpoint_t* module_endpoint );
END_C_DECLS
#endif /* MCA_BTL_MX_PROC_H */

42
ompi/mca/btl/mx/configure.m4
Просмотреть файл

@ -1,42 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2007 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_btl_mx_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_btl_mx_CONFIG],[
AC_CONFIG_FILES([ompi/mca/btl/mx/Makefile])
OMPI_CHECK_MX([btl_mx],
[btl_mx_happy="yes"],
[btl_mx_happy="no"])
AS_IF([test "$btl_mx_happy" = "yes"],
[$1],
[$2])
# substitute in the things needed to build mx
AC_SUBST([btl_mx_CFLAGS])
AC_SUBST([btl_mx_CPPFLAGS])
AC_SUBST([btl_mx_LDFLAGS])
AC_SUBST([btl_mx_LIBS])
])dnl

94
ompi/mca/common/mx/Makefile.am
Просмотреть файл

@ -1,94 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# A word of explanation...
#
# This library is linked against various MCA components because all
# shared-memory based components (e.g., mpool, ptl, etc.) need to
# share some common code and data. There's two cases:
#
# 1. libmca_common_portals.la is a shared library. By linking that shared
# library to all components that need it, the OS linker will
# automatically load it into the process as necessary, and there will
# only be one copy (i.e., all the components will share *one* copy of
# the code and data).
# 2. libmca_common_portals.la is a static library. In this case, it
# will be rolled up into the top-level libmpi.la. It will also be
# rolled into each component, but then the component will also be
# rolled up into the upper-level libmpi.la. Libtool sorts this all
# out and it all works out in the end.
#
# Note that building this common component statically and linking
# against other dynamic components is *not* supported!
AM_CPPFLAGS = $(common_mx_CPPFLAGS)
# Header files
headers = \
common_mx.h
# Source files
sources = \
common_mx.c
lib_LTLIBRARIES =
noinst_LTLIBRARIES =
comp_inst = libmca_common_mx.la
comp_noinst = libmca_common_mx_noinst.la
if MCA_BUILD_ompi_common_mx_DSO
lib_LTLIBRARIES += $(comp_inst)
else
noinst_LTLIBRARIES += $(comp_noinst)
endif
libmca_common_mx_la_SOURCES = $(headers) $(sources)
libmca_common_mx_la_LDFLAGS = -version-info $(libmca_common_mx_so_version) $(common_mx_LDFLAGS)
libmca_common_mx_la_LIBADD = $(common_mx_LIBS)
libmca_common_mx_noinst_la_SOURCES = $(libmca_common_mx_la_SOURCES)
libmca_common_mx_noinst_la_LDFLAGS = $(common_mx_LDFLAGS)
libmca_common_mx_noinst_la_LIBADD = $(common_mx_LIBS)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
ompidir = $(includedir)/openmpi/$(subdir)
ompi_HEADERS = $(headers)
endif
# These two rules will sym link the "noinst" libtool library filename
# to the installable libtool library filename in the case where we are
# compiling this component statically (case 2), described above).
all-local:
if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
$(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
fi
clean-local:
if test -z "$(lib_LTLIBRARIES)"; then \
rm -f "$(comp_inst)"; \
fi

115
ompi/mca/common/mx/common_mx.c
Просмотреть файл

@ -1,115 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 Myricom. All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "common_mx.h"
#include <errno.h>
#include "opal/memoryhooks/memory.h"
#include "opal/util/output.h"
#include "ompi/runtime/params.h"
int mx__regcache_clean(void *ptr, size_t size);
static int ompi_common_mx_initialize_ref_cnt = 0;
static int ompi_common_mx_available = 0;
static int ompi_common_mx_memory_cb_registered = 0;
static void
memory_release_cb(void *buf, size_t length, void *cbdata, bool from_alloc)
{
mx__regcache_clean(buf, length);
}
int
ompi_common_mx_initialize(void)
{
mx_return_t mx_return;
int ret = OMPI_SUCCESS;
ompi_common_mx_initialize_ref_cnt++;
if(ompi_common_mx_initialize_ref_cnt == 1) {
/* set the MX error handle to always return. This function is the
* only MX function allowed to be called before mx_init in order
* to make sure that if the MX is not up and running the MX
* library does not exit the application.
*/
mx_set_error_handler(MX_ERRORS_RETURN);
/* If we have a memory manager available, then tell MX to use
the rcache */
if (0 != (OPAL_MEMORY_FREE_SUPPORT & opal_mem_hooks_support_level())) {
if (OMPI_SUCCESS ==
opal_mem_hooks_register_release(memory_release_cb, NULL)) {
ompi_common_mx_memory_cb_registered = 1;
setenv("MX_RCACHE", "2", 1);
}
}
/* initialize the mx library */
mx_return = mx_init();
if(MX_SUCCESS != mx_return) {
ompi_common_mx_available = -1;
if (0 != ompi_common_mx_memory_cb_registered) {
opal_mem_hooks_unregister_release(memory_release_cb);
ompi_common_mx_memory_cb_registered = 0;
}
opal_output(0,
"Error in mx_init (error %s)\n",
mx_strerror(mx_return));
/* We did not succeed to initialize the MX device */
ompi_common_mx_initialize_ref_cnt = 0;
return OMPI_ERR_NOT_AVAILABLE;
}
ompi_common_mx_available = 1;
} else if (ompi_common_mx_available < 0) {
ret = OMPI_ERR_NOT_AVAILABLE;
}
return ret;
}
int
ompi_common_mx_finalize(void)
{
ompi_common_mx_initialize_ref_cnt--;
if( 0 == ompi_common_mx_initialize_ref_cnt ) {
mx_return_t mx_return;
if (0 != ompi_common_mx_memory_cb_registered) {
opal_mem_hooks_unregister_release(memory_release_cb);
ompi_common_mx_memory_cb_registered = 0;
}
mx_return = mx_finalize();
if(mx_return != MX_SUCCESS){
opal_output(0, "Error in mx_finalize (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
}
return OMPI_SUCCESS;
}

43
ompi/mca/common/mx/common_mx.h
Просмотреть файл

@ -1,43 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MCA_COMMON_MX_H
#define OMPI_MCA_COMMON_MX_H
#include "ompi_config.h"
#include "myriexpress.h"
/**
* Initialize mx library
* *
* @retval OMPI_SUCCESS MX successfully initialized
* @retval OMPI_ERR_NOT_AVAILABLE MX could not be initialized
*/
OMPI_DECLSPEC int ompi_common_mx_initialize(void);
/**
* Shut down mx library
*
*/
OMPI_DECLSPEC int ompi_common_mx_finalize(void);
#endif /* OMPI_MCA_COMMON_PORTALS_H */

42
ompi/mca/common/mx/configure.m4
Просмотреть файл

@ -1,42 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_mtl_mx_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_common_mx_CONFIG],[
AC_CONFIG_FILES([ompi/mca/common/mx/Makefile])
OMPI_CHECK_MX([common_mx],
[common_mx_happy="yes"],
[common_mx_happy="no"])
AS_IF([test "$common_mx_happy" = "yes"],
[$1],
[$2])
# substitute in the things needed to build mx
AC_SUBST([common_mx_CFLAGS])
AC_SUBST([common_mx_CPPFLAGS])
AC_SUBST([common_mx_LDFLAGS])
AC_SUBST([common_mx_LIBS])
])dnl

60
ompi/mca/mtl/mx/Makefile.am
Просмотреть файл

@ -1,60 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2006 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
EXTRA_DIST = post_configure.sh
AM_CPPFLAGS = $(mtl_mx_CPPFLAGS)
mtl_mx_sources = \
mtl_mx.c \
mtl_mx.h \
mtl_mx_cancel.c \
mtl_mx_component.c \
mtl_mx_endpoint.c \
mtl_mx_endpoint.h \
mtl_mx_probe.c \
mtl_mx_recv.c \
mtl_mx_request.h \
mtl_mx_send.c \
mtl_mx_types.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_ompi_mtl_mx_DSO
component_noinst =
component_install = mca_mtl_mx.la
else
component_noinst = libmca_mtl_mx.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_mtl_mx_la_SOURCES = $(mtl_mx_sources)
mca_mtl_mx_la_LIBADD = \
$(mtl_mx_LIBS) \
$(top_ompi_builddir)/ompi/mca/common/mx/libmca_common_mx.la
mca_mtl_mx_la_LDFLAGS = -module -avoid-version $(mtl_mx_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_mtl_mx_la_SOURCES = $(mtl_mx_sources)
libmca_mtl_mx_la_LIBADD = $(mtl_mx_LIBS)
libmca_mtl_mx_la_LDFLAGS = -module -avoid-version $(mtl_mx_LDFLAGS)

49
ompi/mca/mtl/mx/configure.m4
Просмотреть файл

@ -1,49 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2013 Sandia National Laboratories. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# MCA_ompi_mtl_mx_POST_CONFIG(will_build)
# ----------------------------------------
# Only require the tag if we're actually going to be built
AC_DEFUN([MCA_ompi_mtl_mx_POST_CONFIG], [
AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([MTL])])
])dnl
# MCA_mtl_mx_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_ompi_mtl_mx_CONFIG],[
AC_CONFIG_FILES([ompi/mca/mtl/mx/Makefile])
OMPI_CHECK_MX([mtl_mx],
[mtl_mx_happy="yes"],
[mtl_mx_happy="no"])
AS_IF([test "$mtl_mx_happy" = "yes"],
[$1],
[$2])
# substitute in the things needed to build mx
AC_SUBST([mtl_mx_CFLAGS])
AC_SUBST([mtl_mx_CPPFLAGS])
AC_SUBST([mtl_mx_LDFLAGS])
AC_SUBST([mtl_mx_LIBS])
])dnl

278
ompi/mca/mtl/mx/mtl_mx.c
Просмотреть файл

@ -1,278 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/prefetch.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "ompi/mca/common/mx/common_mx.h"
#include "mtl_mx.h"
#include "mtl_mx_types.h"
#include "mtl_mx_endpoint.h"
#include "mtl_mx_request.h"
static int
ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs);
static int
ompi_mtl_mx_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs);
static int
ompi_mtl_mx_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
static int
ompi_mtl_mx_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm);
mca_mtl_mx_module_t ompi_mtl_mx = {
{
8191, /* max cid - 2^13 - 1 */
(1UL << 30), /* max tag value - must allow negatives */
0, /* request reserve space */
0, /* flags */
ompi_mtl_mx_add_procs,
ompi_mtl_mx_del_procs,
ompi_mtl_mx_finalize,
ompi_mtl_mx_send,
ompi_mtl_mx_isend,
ompi_mtl_mx_irecv,
ompi_mtl_mx_iprobe,
ompi_mtl_mx_imrecv,
ompi_mtl_mx_improbe,
ompi_mtl_mx_cancel,
ompi_mtl_mx_add_comm,
ompi_mtl_mx_del_comm
}
};
int ompi_mtl_mx_progress( void );
int ompi_mtl_mx_module_init(){
mx_param_t mx_param;
mx_return_t mx_return;
int32_t nic, ep;
/* setup params */
mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX;
mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max;
/* get a local endpoint */
nic = ompi_mtl_mx.mx_board_num;
if (nic < 0) {
nic = MX_ANY_NIC;
}
ep = ompi_mtl_mx.mx_endpoint_num;
if (ep < 0) {
ep = MX_ANY_ENDPOINT;
}
mx_return = mx_open_endpoint(nic,
ep,
ompi_mtl_mx.mx_filter,
NULL,
0,
&ompi_mtl_mx.mx_endpoint);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
/* get the endpoint address */
mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint,
&ompi_mtl_mx.mx_endpoint_addr);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id),
&(ompi_mtl_mx.mx_addr.endpoint_id) );
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
opal_output_verbose(10, ompi_mtl_base_framework.framework_output,
"mtl:mx: local nic %d, endpoint %d, got nic %d, ep %d\n", nic, ep,
(int)ompi_mtl_mx.mx_addr.nic_id,
ompi_mtl_mx.mx_addr.endpoint_id);
ompi_modex_send( &mca_mtl_mx_component.super.mtl_version,
&ompi_mtl_mx.mx_addr,
sizeof(mca_mtl_mx_addr_t));
/* register the mtl mx progress function */
opal_progress_register(ompi_mtl_mx_progress);
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_finalize(struct mca_mtl_base_module_t* mtl) {
mx_return_t mx_return;
opal_progress_unregister(ompi_mtl_mx_progress);
/* free resources */
mx_return = mx_close_endpoint(ompi_mtl_mx.mx_endpoint);
if(mx_return != MX_SUCCESS){
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_close_endpoint (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
return ompi_common_mx_finalize();
}
int
ompi_mtl_mx_add_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs)
{
int i;
assert(mtl == &ompi_mtl_mx.super);
for( i = 0; i < (int) nprocs; i++ ){
mca_mtl_mx_endpoint_t* mtl_mx_endpoint =
mca_mtl_mx_endpoint_create(procs[i]);
if(NULL == mtl_mx_endpoint) {
return OMPI_ERROR;
}
procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL] =
mtl_mx_endpoint;
}
/* because mx_connect isn't an interupting function, need to
progress MX as often as possible during the stage gate 2. This
would have happened after the stage gate anyway, so we're just
speeding things up a bit. */
#if OMPI_ENABLE_PROGRESS_THREADS == 0
/* switch from letting us sit in the event library for a bit each
time through opal_progress() to completely non-blocking */
opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
#endif
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_del_procs(struct mca_mtl_base_module_t *mtl,
size_t nprocs,
struct ompi_proc_t** procs)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_add_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_del_comm(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm)
{
return OMPI_SUCCESS;
}
int ompi_mtl_mx_progress( void ) {
mx_return_t mx_return;
mx_request_t mx_request;
mx_status_t mx_status;
uint32_t result;
mca_mtl_mx_request_t* mtl_mx_request;
int completed = 0;
while(1){
mx_return = mx_ipeek(ompi_mtl_mx.mx_endpoint,
&mx_request,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_ipeek (error %s)\n", mx_strerror(mx_return));
}
if(result) {
completed++;
mx_return = mx_test(ompi_mtl_mx.mx_endpoint,
&mx_request,
&mx_status,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_test (error %s)\n", mx_strerror(mx_return));
abort();
}
if( OPAL_UNLIKELY(0 == result) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in ompi_mtl_mx_progress, mx_ipeek returned a request, mx_test on the request resulted failure.\n");
abort();
}
mtl_mx_request = (mca_mtl_mx_request_t*) mx_status.context;
if(OMPI_MTL_MX_ISEND == mtl_mx_request->type) {
if(mtl_mx_request->free_after) {
free(mtl_mx_request->mx_segment[0].segment_ptr);
}
} else {
assert( OMPI_MTL_MX_IRECV == mtl_mx_request->type );
ompi_mtl_datatype_unpack(mtl_mx_request->convertor,
mtl_mx_request->mx_segment[0].segment_ptr,
mx_status.xfer_length);
/* set the status */
MX_GET_SRC(mx_status.match_info,
mtl_mx_request->super.ompi_req->req_status.MPI_SOURCE);
MX_GET_TAG(mx_status.match_info,
mtl_mx_request->super.ompi_req->req_status.MPI_TAG);
mtl_mx_request->super.ompi_req->req_status._ucount =
mx_status.xfer_length;
}
/* suppose everything went just fine ... */
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
if( OPAL_UNLIKELY(MX_STATUS_SUCCESS != mx_status.code) ) {
if( MX_STATUS_TRUNCATED == mx_status.code ) {
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
} else {
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_INTERN;
}
return completed;
}
mtl_mx_request->super.completion_callback(&mtl_mx_request->super);
return completed;
} else {
return completed;
}
}
}

87
ompi/mca/mtl/mx/mtl_mx.h
Просмотреть файл

@ -1,87 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MTL_MX_H_HAS_BEEN_INCLUDED
#define MTL_MX_H_HAS_BEEN_INCLUDED
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "opal/datatype/opal_convertor.h"
BEGIN_C_DECLS
int
ompi_mtl_mx_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode);
extern int ompi_mtl_mx_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t * mtl_request);
extern int ompi_mtl_mx_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
struct mca_mtl_request_t *mtl_request);
extern int ompi_mtl_mx_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status);
extern int ompi_mtl_mx_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request);
extern int ompi_mtl_mx_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status);
extern int ompi_mtl_mx_cancel(struct mca_mtl_base_module_t* mtl,
struct mca_mtl_request_t *mtl_request,
int flag);
extern int ompi_mtl_mx_finalize(struct mca_mtl_base_module_t* mtl);
int ompi_mtl_mx_module_init(void);
END_C_DECLS
#endif /* MTL_MX_H_HAS_BEEN_INCLUDED */

47
ompi/mca/mtl/mx/mtl_mx_cancel.c
Просмотреть файл

@ -1,47 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "mtl_mx.h"
#include "mtl_mx_types.h"
#include "mtl_mx_request.h"
int
ompi_mtl_mx_cancel(struct mca_mtl_base_module_t* mtl,
struct mca_mtl_request_t *mtl_request,
int flag)
{
mca_mtl_mx_request_t *mtl_mx_request =
(mca_mtl_mx_request_t*) mtl_request;
uint32_t result;
mx_cancel(ompi_mtl_mx.mx_endpoint,
&mtl_mx_request->mx_request,
&result);
/* If MX canceled the request, mark it as canceled and complete
it. Otherwise, keep on going */
if (result) {
mtl_request->ompi_req->req_status._cancelled = true;
mtl_mx_request->super.completion_callback(&mtl_mx_request->super);
}
return OMPI_SUCCESS;
}

168
ompi/mca/mtl/mx/mtl_mx_component.c
Просмотреть файл

@ -1,168 +0,0 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/mca/event/event.h"
#include "ompi/mca/mtl/base/base.h"
#include "ompi/mca/common/mx/common_mx.h"
#include "mtl_mx.h"
#include "mtl_mx_types.h"
#include "mtl_mx_request.h"
#include "myriexpress.h"
#define MCA_MTL_MX_QUEUE_LENGTH_MAX 2*1024*1024
static int ompi_mtl_mx_component_register(void);
static int ompi_mtl_mx_component_open(void);
static int ompi_mtl_mx_component_close(void);
static int ompi_mtl_mx_component_initialized = 0;
static mca_mtl_base_module_t* ompi_mtl_mx_component_init( bool enable_progress_threads,
bool enable_mpi_threads );
mca_mtl_mx_component_t mca_mtl_mx_component = {
{
/* First, the mca_base_component_t struct containing meta
* information about the component itself */
{
MCA_MTL_BASE_VERSION_2_0_0,
"mx", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
ompi_mtl_mx_component_open, /* component open */
ompi_mtl_mx_component_close, /* component close */
NULL,
ompi_mtl_mx_component_register
},
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
ompi_mtl_mx_component_init /* component init */
}
};
static int
ompi_mtl_mx_component_register(void)
{
ompi_mtl_mx.mx_filter = 0xaaaaffff;
(void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "filter",
"user assigned value used to filter incomming messages",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_filter);
ompi_mtl_mx.mx_timeout = MX_INFINITE;
(void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "timeout",
"Timeout for connections", MCA_BASE_VAR_TYPE_INT,
NULL, 0, 0, OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_timeout);
ompi_mtl_mx.mx_retries = 20;
(void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "retries",
"Number of retries for each new connection before considering the peer as unreacheable",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_retries);
ompi_mtl_mx.mx_support_sharedmem = 1;
(void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "shared_mem",
"Enable the MX support for shared memory", MCA_BASE_VAR_TYPE_INT,
NULL, 0, 0, OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_support_sharedmem);
ompi_mtl_mx.mx_unexp_queue_max = MCA_MTL_MX_QUEUE_LENGTH_MAX;
(void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "unexpected_queue_length",
"Length of MX unexpected message queue", MCA_BASE_VAR_TYPE_INT,
NULL, 0, 0, OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_unexp_queue_max);
ompi_mtl_mx.mx_board_num = -1;
(void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "board",
"Which MX board number to use (<0 = any)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_board_num);
ompi_mtl_mx.mx_endpoint_num = -1;
(void) mca_base_component_var_register(&mca_mtl_mx_component.super.mtl_version, "endpoint",
"Which MX endpoint number to use (<0 = any)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mx.mx_endpoint_num);
return OMPI_SUCCESS;
}
static int
ompi_mtl_mx_component_open(void)
{
if(ompi_mtl_mx.mx_unexp_queue_max > MCA_MTL_MX_QUEUE_LENGTH_MAX) {
ompi_mtl_mx.mx_unexp_queue_max = MCA_MTL_MX_QUEUE_LENGTH_MAX;
}
return OMPI_SUCCESS;
}
static int
ompi_mtl_mx_component_close(void)
{
--ompi_mtl_mx_component_initialized;
if( 0 == ompi_mtl_mx_component_initialized ) {
int ret = ompi_common_mx_finalize();
if(OMPI_SUCCESS != ret) {
return OMPI_ERROR;
}
}
return OMPI_SUCCESS;
}
static mca_mtl_base_module_t*
ompi_mtl_mx_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
int ret;
ret = ompi_common_mx_initialize();
if(OMPI_SUCCESS != ret) {
return NULL;
}
ompi_mtl_mx_component_initialized++;
ret = ompi_mtl_mx_module_init();
if (OMPI_SUCCESS != ret) {
return NULL;
}
ompi_mtl_mx.super.mtl_request_size =
sizeof(mca_mtl_mx_request_t) -
sizeof(struct mca_mtl_request_t);
return &ompi_mtl_mx.super;
}

105
ompi/mca/mtl/mx/mtl_mx_endpoint.c
Просмотреть файл

@ -1,105 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/util/output.h"
#include <sys/time.h>
#include <time.h>
#include "ompi/types.h"
#include "mtl_mx.h"
#include "mtl_mx_types.h"
#include "mtl_mx_endpoint.h"
#include "ompi/runtime/ompi_module_exchange.h"
/*
* Initialize state of the endpoint instance.
*
*/
static void mca_mtl_mx_endpoint_construct(mca_mtl_mx_endpoint_t* endpoint)
{
endpoint->mtl_mx_module = NULL;
endpoint->mx_peer = NULL;
}
/*
* Destroy a endpoint
*
*/
static void mca_mtl_mx_endpoint_destruct(mca_mtl_mx_endpoint_t* endpoint)
{
}
OBJ_CLASS_INSTANCE(
mca_mtl_mx_endpoint_t,
opal_list_item_t,
mca_mtl_mx_endpoint_construct,
mca_mtl_mx_endpoint_destruct);
mca_mtl_mx_endpoint_t* mca_mtl_mx_endpoint_create(ompi_proc_t* ompi_proc) {
mca_mtl_mx_endpoint_t* mtl_mx_endpoint = NULL;
int rc;
mca_mtl_mx_addr_t *mx_peer;
size_t size;
mx_return_t mx_return;
int num_retry = 0;
/* get the remote proc's address (only one) */
rc = ompi_modex_recv(&mca_mtl_mx_component.super.mtl_version,
ompi_proc, (void**)&mx_peer, &size);
if( rc != OMPI_SUCCESS || size != sizeof(mca_mtl_mx_addr_t)) {
return NULL;
}
mtl_mx_endpoint = (mca_mtl_mx_endpoint_t*) OBJ_NEW(mca_mtl_mx_endpoint_t);
mtl_mx_endpoint->mx_peer = mx_peer;
retry_connect:
mx_return = mx_connect(ompi_mtl_mx.mx_endpoint,
mx_peer->nic_id,
mx_peer->endpoint_id,
ompi_mtl_mx.mx_filter,
ompi_mtl_mx.mx_timeout,
&mtl_mx_endpoint->mx_peer_addr);
if(MX_SUCCESS != mx_return) {
char peer_name[MX_MAX_HOSTNAME_LEN];
if(MX_TIMEOUT == mx_return) {
if( num_retry++ < ompi_mtl_mx.mx_retries ) {
goto retry_connect;
}
}
if(MX_SUCCESS != mx_nic_id_to_hostname( mx_peer->nic_id, peer_name)) {
sprintf( peer_name, "unknown %lx nic_id", (long)mx_peer->nic_id );
}
opal_output(ompi_mtl_base_framework.framework_output,
"mx_connect fail for %s with key %x (error %s)\n",
peer_name, ompi_mtl_mx.mx_filter, mx_strerror(mx_return) );
return NULL;
}
return mtl_mx_endpoint;
}

71
ompi/mca/mtl/mx/mtl_mx_endpoint.h
Просмотреть файл

@ -1,71 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_MTL_MX_ENDPOINT_H
#define MCA_MTL_MX_ENDPOINT_H
#include "opal/class/opal_list.h"
#include "opal/mca/event/event.h"
#include "ompi/proc/proc.h"
#include "ompi/mca/mtl/mtl.h"
#include "mtl_mx.h"
#include "myriexpress.h"
BEGIN_C_DECLS
OBJ_CLASS_DECLARATION(mca_mtl_mx_endpoint_t);
/**
* Structure used to publish MX information to peers
*/
struct mca_mtl_mx_addr_t {
uint64_t nic_id;
uint32_t endpoint_id;
};
typedef struct mca_mtl_mx_addr_t mca_mtl_mx_addr_t;
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_mtl_mx_endpoint_t is associated w/ each process
* and MTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_mtl_mx_endpoint_t {
opal_list_item_t super;
struct mca_mtl_mx_module_t* mtl_mx_module;
/**< MTL instance that created this connection */
struct mca_mtl_mx_addr_t* mx_peer;
/** the address as reported by the peer */
mx_endpoint_addr_t mx_peer_addr;
/** the remote MX endpoint address */
};
typedef struct mca_mtl_mx_endpoint_t mca_mtl_mx_endpoint_t;
OBJ_CLASS_DECLARATION(mca_mtl_mx_endpoint);
mca_mtl_mx_endpoint_t* mca_mtl_mx_endpoint_create(ompi_proc_t*);
END_C_DECLS
#endif

86
ompi/mca/mtl/mx/mtl_mx_probe.c
Просмотреть файл

@ -1,86 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved.
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/mtl/base/base.h"
#include "mtl_mx.h"
#include "mtl_mx_types.h"
#include "mtl_mx_request.h"
int
ompi_mtl_mx_iprobe(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *flag,
struct ompi_status_public_t *status)
{
uint32_t result;
mx_return_t ret;
mx_status_t mx_status;
uint64_t match_bits;
uint64_t mask_bits;
MX_SET_RECV_BITS(match_bits,
mask_bits,
comm->c_contextid,
src,
tag);
ret = mx_iprobe(ompi_mtl_mx.mx_endpoint,
match_bits,
mask_bits,
&mx_status,
&result);
if (MX_SUCCESS != ret) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_iprobe (error %s)\n", mx_strerror(ret));
return OMPI_ERROR;
}
if (result) {
if(MPI_STATUS_IGNORE != status) {
MX_GET_SRC(mx_status.match_info, status->MPI_SOURCE);
MX_GET_TAG(mx_status.match_info, status->MPI_TAG);
status->_ucount = mx_status.msg_length;
}
*flag = 1;
} else {
*flag = 0;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_improbe(struct mca_mtl_base_module_t *mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
int *matched,
struct ompi_message_t **message,
struct ompi_status_public_t *status)
{
return OMPI_ERR_NOT_IMPLEMENTED;
}

89
ompi/mca/mtl/mx/mtl_mx_recv.c
Просмотреть файл

@ -1,89 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/communicator/communicator.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
#include "mtl_mx.h"
#include "mtl_mx_types.h"
#include "mtl_mx_request.h"
int
ompi_mtl_mx_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm,
int src,
int tag,
struct opal_convertor_t *convertor,
struct mca_mtl_request_t *mtl_request)
{
int ret;
mx_return_t mx_return;
mca_mtl_mx_request_t * mtl_mx_request = (mca_mtl_mx_request_t*) mtl_request;
uint64_t match_bits;
uint64_t mask_bits;
size_t length;
ret = ompi_mtl_datatype_recv_buf(convertor,
&mtl_mx_request->mx_segment[0].segment_ptr,
&length,
&mtl_mx_request->free_after);
mtl_mx_request->mx_segment[0].segment_length = length;
mtl_mx_request->convertor = convertor;
mtl_mx_request->type = OMPI_MTL_MX_IRECV;
if(OMPI_SUCCESS != ret) return ret;
MX_SET_RECV_BITS(match_bits,
mask_bits,
comm->c_contextid,
src,
tag);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"recv bits: 0x%016" PRIu64 " 0x%016" PRIu64 "\n",
match_bits, mask_bits));
mx_return = mx_irecv( ompi_mtl_mx.mx_endpoint,
mtl_mx_request->mx_segment,
1,
match_bits,
mask_bits,
mtl_mx_request,
&mtl_mx_request->mx_request);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_irecv (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_imrecv(struct mca_mtl_base_module_t* mtl,
struct opal_convertor_t *convertor,
struct ompi_message_t **message,
struct mca_mtl_request_t *mtl_request)
{
return OMPI_ERR_NOT_IMPLEMENTED;
}

40
ompi/mca/mtl/mx/mtl_mx_request.h
Просмотреть файл

@ -1,40 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OMPI_MTL_MX_REQUEST_H
#define OMPI_MTL_MX_REQUEST_H
#include "opal/datatype/opal_convertor.h"
typedef enum {
OMPI_MTL_MX_ISEND,
OMPI_MTL_MX_IRECV
} mca_mtl_mx_request_type_t;
struct mca_mtl_mx_request_t {
struct mca_mtl_request_t super;
mx_request_t mx_request;
mx_segment_t mx_segment[1];
struct opal_convertor_t *convertor;
bool free_after;
mca_mtl_mx_request_type_t type;
};
typedef struct mca_mtl_mx_request_t mca_mtl_mx_request_t;
#endif

191
ompi/mca/mtl/mx/mtl_mx_send.c
Просмотреть файл

@ -1,191 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/prefetch.h"
#include "ompi/communicator/communicator.h"
#include "opal/datatype/opal_convertor.h"
#include "mtl_mx.h"
#include "mtl_mx_types.h"
#include "mtl_mx_request.h"
#include "ompi/mca/mtl/base/mtl_base_datatype.h"
int
ompi_mtl_mx_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
{
mx_return_t mx_return;
uint64_t match_bits;
mca_mtl_mx_request_t mtl_mx_request;
size_t length;
mx_status_t mx_status;
uint32_t result;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
char* where;
assert(mtl == &ompi_mtl_mx.super);
MX_SET_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag);
ompi_mtl_datatype_pack(convertor,
&mtl_mx_request.mx_segment[0].segment_ptr,
&length,
&mtl_mx_request.free_after);
mtl_mx_request.mx_segment[0].segment_length = length;
mtl_mx_request.convertor = convertor;
mtl_mx_request.type = OMPI_MTL_MX_ISEND;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"issend bits: 0x%016" PRIu64 "\n",
match_bits));
if(mode == MCA_PML_BASE_SEND_SYNCHRONOUS) {
mx_return = mx_issend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request.mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
&mtl_mx_request,
&mtl_mx_request.mx_request
);
where = "mx_issend";
} else {
mx_return = mx_isend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request.mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
&mtl_mx_request,
&mtl_mx_request.mx_request
);
where = "mx_isend";
}
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
char peer_name[MX_MAX_HOSTNAME_LEN];
if(MX_SUCCESS != mx_nic_id_to_hostname( mx_endpoint->mx_peer->nic_id, peer_name)) {
sprintf( peer_name, "unknown %lx nic_id", (long)mx_endpoint->mx_peer->nic_id );
}
opal_output(ompi_mtl_base_framework.framework_output, "Error in %s (error %s) sending to %s\n",
where, mx_strerror(mx_return), peer_name);
/* Free buffer if needed */
if(mtl_mx_request.free_after) {
free(mtl_mx_request.mx_segment[0].segment_ptr);
}
return OMPI_ERROR;
}
do {
mx_return = mx_test(ompi_mtl_mx.mx_endpoint,
&mtl_mx_request.mx_request,
&mx_status,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_wait (error %s)\n", mx_strerror(mx_return));
abort();
}
if( OPAL_UNLIKELY(result && mx_status.code != MX_STATUS_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output,
"Error in ompi_mtl_mx_send, mx_wait returned something other than MX_STATUS_SUCCESS: mx_status.code = %d.\n",
mx_status.code);
abort();
}
} while(!result);
/* Free buffer if needed */
if(mtl_mx_request.free_after) {
free(mtl_mx_request.mx_segment[0].segment_ptr);
}
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t * mtl_request)
{
mx_return_t mx_return;
uint64_t match_bits;
mca_mtl_mx_request_t * mtl_mx_request = (mca_mtl_mx_request_t*) mtl_request;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
char* where;
assert(mtl == &ompi_mtl_mx.super);
MX_SET_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag);
ompi_mtl_datatype_pack(convertor,
&mtl_mx_request->mx_segment[0].segment_ptr,
&length,
&mtl_mx_request->free_after);
mtl_mx_request->mx_segment[0].segment_length = length;
mtl_mx_request->convertor = convertor;
mtl_mx_request->type = OMPI_MTL_MX_ISEND;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"issend bits: 0x%016" PRIu64 "\n", match_bits));
if(mode == MCA_PML_BASE_SEND_SYNCHRONOUS) {
mx_return = mx_issend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request->mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
mtl_mx_request,
&mtl_mx_request->mx_request
);
where = "mx_issend";
} else {
mx_return = mx_isend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request->mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
mtl_mx_request,
&mtl_mx_request->mx_request
);
where = "mx_isend";
}
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
char peer_name[MX_MAX_HOSTNAME_LEN];
if(MX_SUCCESS != mx_nic_id_to_hostname( mx_endpoint->mx_peer->nic_id, peer_name)) {
sprintf( peer_name, "unknown %lx nic_id", (long)mx_endpoint->mx_peer->nic_id );
}
opal_output(ompi_mtl_base_framework.framework_output, "Error in %s (error %s) sending to %s\n",
where, mx_strerror(mx_return), peer_name);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}

128
ompi/mca/mtl/mx/mtl_mx_types.h
Просмотреть файл

@ -1,128 +0,0 @@
/*
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2006 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MTL_MX_TYPES_H_HAS_BEEN_INCLUDED
#define MTL_MX_TYPES_H_HAS_BEEN_INCLUDED
#include "ompi_config.h"
#include "mtl_mx.h"
#include "ompi/mca/mtl/mtl.h"
#include "ompi/mca/mtl/base/base.h"
#include "mtl_mx_endpoint.h"
#include "myriexpress.h"
BEGIN_C_DECLS
/**
* MTL Module Interface
*/
struct mca_mtl_mx_module_t {
mca_mtl_base_module_t super; /**< base MTL interface */
int32_t mx_unexp_queue_max; /**< maximium size of the MX unexpected message queue */
int32_t mx_filter; /**< user assigned value used to filter incomming messages */
int32_t mx_timeout;
int32_t mx_retries;
int32_t mx_support_sharedmem;
int mx_board_num;
int mx_endpoint_num;
mx_endpoint_t mx_endpoint; /**< mx data structure for local endpoint */
mx_endpoint_addr_t mx_endpoint_addr; /**< mx data structure for local endpoint address */
mca_mtl_mx_addr_t mx_addr;
};
typedef struct mca_mtl_mx_module_t mca_mtl_mx_module_t;
extern mca_mtl_mx_module_t ompi_mtl_mx;
struct mca_mtl_mx_component_t{
mca_mtl_base_component_2_0_0_t super; /**< base MTL component */
};
typedef struct mca_mtl_mx_component_t mca_mtl_mx_component_t;
OMPI_MODULE_DECLSPEC extern mca_mtl_mx_component_t mca_mtl_mx_component;
/* match/ignore bit manipulation
*
* 01234567 01234567 01234567 01234567 01234567 01234567 01234567 01234567
* | |
* context id | source | message tag
* | |
*/
#define MX_SOURCE_MASK 0x0000FFFF00000000ULL
#define MX_TAG_MASK 0x00000000FFFFFFFFULL
#define MX_SOURCE_IGNR ~MX_SOURCE_MASK
/* we need to keep top bit (sign bit) of the tag
collectives use this to distinguish the message */
#define MX_TAG_IGNR 0xFFFFFFFF80000000ULL
/* get the tag from the bits */
#define MX_GET_TAG(match_bits, tag) \
{ \
tag = (int) (match_bits & MX_TAG_MASK); \
}
/* get the tag from the bits */
#define MX_GET_SRC(match_bits, src) \
{ \
src = (int) ((match_bits & MX_SOURCE_MASK) >> 32); \
}
/* send posting */
#define MX_SET_SEND_BITS(match_bits, contextid, source, tag) \
{ \
match_bits = contextid; \
match_bits = (match_bits << 16); \
match_bits |= source; \
match_bits = (match_bits << 32); \
match_bits |= (MX_TAG_MASK & tag); \
}
/* receive posting */
#define MX_SET_RECV_BITS(match_bits, mask_bits, contextid, source, tag) \
{ \
match_bits = contextid; \
match_bits = (match_bits << 16); \
\
if (MPI_ANY_SOURCE == source) { \
mask_bits = MX_SOURCE_IGNR; \
} else { \
mask_bits = ~0; \
match_bits |= source; \
} \
match_bits = (match_bits << 32); \
\
if (MPI_ANY_TAG == tag) { \
mask_bits &= MX_TAG_IGNR; \
} else { \
match_bits |= (MX_TAG_MASK & tag); \
} \
}
END_C_DECLS
#endif /* MTL_MX_TYPES_H_HAS_BEEN_INCLUDED */

1
ompi/mca/mtl/mx/post_configure.sh
Просмотреть файл

@ -1 +0,0 @@
DIRECT_CALL_HEADER="ompi/mca/mtl/mx/mtl_mx.h"