ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

A copy for posterity of the Open MPI Sicortex BTL.

Просмотр исходного кода 
This commit was SVN r27938.
Этот коммит содержится в:
George Bosilca 2013-01-28 03:42:52 +00:00
родитель e5d99a1877
Коммит 1b7dff3f2f
14 изменённых файлов: 2046 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
ompi/mca/btl/sicortex/.ompi_ignore Исполняемый файл
Просмотреть файл

2
ompi/mca/btl/sicortex/.ompi_unignore Исполняемый файл
Просмотреть файл

@ -0,0 +1,2 @@
tma
bosilca

40
ompi/mca/btl/sicortex/Makefile.am Исполняемый файл
Просмотреть файл

@ -0,0 +1,40 @@
#
# Copyright (c) 2008-2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CPPFLAGS = $(btl_sicortex_CPPFLAGS)
btl_sicortex_sources = btl_sicortex.c btl_sicortex.h btl_sicortex_component.c btl_sicortex_endpoint.c \
btl_sicortex_endpoint.h btl_sicortex_frag.c btl_sicortex_frag.h btl_sicortex_proc.c btl_sicortex_proc.h
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_btl_sicortex_DSO
component_noinst =
component_install = mca_btl_sicortex.la
else
component_noinst = libmca_btl_sicortex.la
component_install =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component_install)
mca_btl_sicortex_la_SOURCES = $(btl_sicortex_sources)
mca_btl_sicortex_la_CFLAGS = -DBCAST
mca_btl_sicortex_la_LIBADD = -lscmpi_debug $(btl_sicortex_LIBS)
mca_btl_sicortex_la_LDFLAGS = -module -avoid-version $(btl_sicortex_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_btl_sicortex_la_SOURCES = $(btl_sicortex_sources)
libmca_btl_sicortex_la_CFLAGS = -DBCAST
libmca_btl_sicortex_la_LIBADD = -lscmpi_debug $(btl_sicortex_LIBS)
libmca_btl_sicortex_la_LDFLAGS = -module -avoid-version $(btl_sicortex_LDFLAGS)

843
ompi/mca/btl/sicortex/btl_sicortex.c Исполняемый файл
Просмотреть файл

@ -0,0 +1,843 @@
/*
* Copyright (c) 2004-2009 The University of Tennessee and The Univer
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "orte/util/show_help.h"
#include "opal/util/if.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "btl_sicortex.h"
#include "btl_sicortex_frag.h"
#include "btl_sicortex_proc.h"
#include "btl_sicortex_endpoint.h"
#include "ompi/datatype/convertor.h"
#include "ompi/mca/mpool/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include <unistd.h>
#include <linux/sicortex/scdma_hw.h>
#include <linux/sicortex/scdma.h>
#include <assert.h>
static int mca_btl_sicortex_add_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **ompi_procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable )
{
mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*)btl;
int i,rc,index = 0;
sicortex_btl->peers = (mca_btl_base_endpoint_t**)malloc(sizeof(mca_btl_base_endpoint_t*)*(nprocs-1));
for(i = 0; i < (int) nprocs; i++) {
struct ompi_proc_t* ompi_proc = ompi_procs[i];
mca_btl_sicortex_proc_t* sicortex_proc;
mca_btl_base_endpoint_t* sicortex_endpoint;
if(ompi_proc_local_proc == ompi_proc)
continue;
if(NULL == (sicortex_proc = mca_btl_sicortex_proc_create(ompi_proc))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*
* Check to make sure that the peer has at least as many interface
* addresses exported as we are trying to use. If not, then
* don't bind this PTL instance to the proc.
*/
OPAL_THREAD_LOCK(&sicortex_proc->proc_lock);
/* The btl_proc datastructure is shared by all SICORTEX PTL
* instances that are trying to reach this destination.
* Cache the peer instance on the btl_proc.
*/
sicortex_endpoint = OBJ_NEW(mca_btl_sicortex_endpoint_t);
if(NULL == sicortex_endpoint) {
OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
sicortex_endpoint->route = (peer_struct_t*)malloc(sizeof(peer_struct_t));
sicortex_endpoint->endpoint_btl = sicortex_btl;
rc = mca_btl_sicortex_proc_insert(sicortex_proc, sicortex_endpoint);
if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(sicortex_endpoint);
OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock);
continue;
}
rc = scdma_set_route(sicortex_btl->ctx,
sicortex_endpoint->route->route_handles[0],
sicortex_endpoint->route->route_handles[1],
sicortex_endpoint->route->route_handles[2],
sicortex_endpoint->route->context_id,
&(sicortex_endpoint->route->ports[0]),
&(sicortex_endpoint->route->ports[1]),
&(sicortex_endpoint->route->ports[2]));
if(rc != 0) {
OBJ_RELEASE(sicortex_endpoint);
OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock);
continue;
}
opal_bitmap_set_bit(reachable, i);
OPAL_THREAD_UNLOCK(&sicortex_proc->proc_lock);
sicortex_btl->peers[index++] = peers[i] = sicortex_endpoint;
}
sicortex_btl->bds_size = sicortex_btl->ctx->bdt_size;
sicortex_btl->bds_limit = sicortex_btl->ctx->num_bdt_entries;
sicortex_btl->bds_head = 0;
return OMPI_SUCCESS;
}
static int
mca_btl_sicortex_del_procs(struct mca_btl_base_module_t* btl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_btl_base_endpoint_t ** peers)
{
/* TODO */
return OMPI_SUCCESS;
}
/**
* Allocate a segment.
*
* @param btl (IN) BTL module
* @param size (IN) Request segment size.
*/
static mca_btl_base_descriptor_t*
mca_btl_sicortex_alloc( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
uint8_t order,
size_t size,
uint32_t flags )
{
mca_btl_sicortex_frag_t* frag = NULL;
int rc;
if(size <= btl->btl_eager_limit) {
MCA_BTL_SICORTEX_FRAG_ALLOC_EAGER(sicortex_btl, frag, rc);
} else if( size <= btl->btl_max_send_size ) {
MCA_BTL_SICORTEX_FRAG_ALLOC_MAX(sicortex_btl, frag, rc);
}
if(OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
frag->segment.seg_len = size;
frag->segment.seg_addr.pval = (void*)((char*)(frag + 1));
frag->segment.seg_len = size;
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_flags = flags;
frag->base.order = MCA_BTL_NO_ORDER;
return (mca_btl_base_descriptor_t*)frag;
}
/**
* Return a segment
*/
static int
mca_btl_sicortex_free( struct mca_btl_base_module_t* btl,
mca_btl_base_descriptor_t* des )
{
mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*)des;
MCA_BTL_SICORTEX_FRAG_RETURN(btl, frag);
return OMPI_SUCCESS;
}
/**
* Initiate an immediate blocking send.
* Completion Semantics: the BTL will make a best effort
* to send the header and "size" bytes from the datatype using the convertor.
* The header is guaranteed to be delivered entirely in the first segment.
* Should the BTL be unable to deliver the data due to resource constraints
* the BTL will return a descriptor (via the OUT param)
* of size "payload_size + header_size".
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param convertor (IN) Data type convertor
* @param header (IN) Pointer to header.
* @param header_size (IN) Size of header. * @param payload_size (IN) Size of payload (from convertor).
* @param order (IN) The ordering tag (may be MCA_BTL_NO_ORDER)
* @param flags (IN) Flags.
* @param tag (IN) The tag value used to notify the peer.
* @param descriptor (OUT) The descriptor to be returned unable to be sent immediately
*/
static int mca_btl_sicortex_sendi( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct ompi_convertor_t* convertor,
void* header,
size_t header_size,
size_t payload_size,
uint8_t order,
uint32_t flags,
mca_btl_base_tag_t tag,
mca_btl_base_descriptor_t** descriptor )
{
mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl;
size_t max_data, length = header_size + payload_size;
struct iovec iov;
uint32_t iov_count = 1, length_on_wire;
scdma_hw_cmd_t* cmd;
char* payload;
if( length > SCDMA_HW_CMD_PAYLOAD_MAX_SIZE ) {
*descriptor = mca_btl_sicortex_alloc( btl, endpoint, order, length, flags );
return OMPI_ERR_RESOURCE_BUSY;
}
/* Adjust the length on the wire to allow the DMA engine to
* know exactly how many cache lines have to be transfered.
*/
length_on_wire = scdma_send_event_len(length + 2 * sizeof(uint64_t));
cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx);
cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT,
length_on_wire,
endpoint->route->route_handles[0/*sicortex_btl->handle_count*/],
endpoint->route->ports[0/*sicortex_btl->handle_count*/]);
/*sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM;*/
cmd->payload[0] = (((uint64_t)tag << 16) | (EVENT_SEND_IM << 8)
| (length + 2 * sizeof(uint64_t)) );
payload = (char*)&cmd->payload[1];
memcpy( payload, header, header_size );
iov.iov_base = payload + header_size;
iov.iov_len = payload_size;
max_data = payload_size;
if( 0 != payload_size ) {
ompi_convertor_pack( convertor, &iov, &iov_count, &max_data );
assert( max_data == payload_size );
}
if( length > (SCDMA_HW_CMD_PAYLOAD_MAX_SIZE-64) )
cmd->payload[15] = 0;
scdma_cq_post_fastpath(sicortex_btl->ctx,cmd->header);
return OMPI_SUCCESS;
}
/**
* Initiate an asynchronous send.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transfered
* @param tag (IN) The tag value used to notify the peer.
*/
static int
mca_btl_sicortex_send( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag )
{
mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl;
mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*)descriptor;
int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP );
size_t length = frag->segment.seg_len;
int offset, len, length_on_wire;
scdma_hw_cmd_t *cmd;
scdma_hw_put_im_heap_cmd_t *pb;
length_on_wire = length + 2 * sizeof(uint64_t);
frag->endpoint = endpoint;
if( SCDMA_HW_CMD_PAYLOAD_MAX_SIZE < length_on_wire ) {
offset = 0;
do {
if( length_on_wire > 16*8 )
len = 8*8;/* SCDMA_HW_CMD_PAYLOAD_MAX_SIZE;*/
else
len = length_on_wire - 2 * 8;
cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx);
pb = (scdma_hw_put_im_heap_cmd_t *)cmd;
pb->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_PUT_IM_HP,
(len + 0x17) & ~0x7, /* 2 * 64 bits + len */
endpoint->route->route_handles[0/*sicortex_btl->handle_count*/],
endpoint->route->ports[0/*sicortex_btl->handle_count*/]);
pb->count_offset = (SCDMA_USABLE_HEAP_OFFSET + offset);
memcpy( scdma_cmd_payload(cmd), (char*)frag->base.des_src->seg_addr.pval + offset, len);
scdma_cq_post_fastpath(sicortex_btl->ctx, pb->header);
offset += len;
length_on_wire -= len;
} while(length_on_wire > 2*8);
cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx);
cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT,
0x10,
endpoint->route->route_handles[0/*sicortex_btl->handle_count*/],
endpoint->route->ports[0/*sicortex_btl->handle_count*/]);
cmd->payload[0] = (((uint64_t)length << 32) | ((uint64_t)tag << 16) |
(EVENT_SEND_IM_PUT_POST << 8) | 0x10);
cmd->payload[1] = 0;
/* Next operation will take another path. */
/*sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM;*/
} else {
/* Adjust the length on the wire to allow the DMA engine to
* know exactly how many cache lines have to be transfered.
*/
length_on_wire = scdma_send_event_len(length + 2 * sizeof(uint64_t));
cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx);
cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT,
length_on_wire,
endpoint->route->route_handles[0/*sicortex_btl->handle_count*/],
endpoint->route->ports[0/*sicortex_btl->handle_count*/]);
cmd->payload[0] = (((uint64_t)tag << 16) | (EVENT_SEND_IM << 8)
| (length + 2 * sizeof(uint64_t)) );
memcpy( &cmd->payload[1], frag->base.des_src->seg_addr.pval, length );
cmd->payload[15] = 0;
}
scdma_cq_post_fastpath(sicortex_btl->ctx,cmd->header);
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(sicortex_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS);
}
if( btl_ownership ){
MCA_BTL_SICORTEX_FRAG_RETURN(sicortex_btl, frag);
}
return 1;
}
#define MCA_BTL_SICORTEX_PACK_BD(v64bits, offset, overhead, bd_index, num_entries) \
do { \
v64bits = (((uint64_t)(offset) << 48) | ((uint64_t)(overhead) << 32) | \
(((uint64_t)(bd_index) << 16) | (uint64_t)(num_entries))); \
} while (0)
#define MCA_BTL_SICORTEX_UNPACK_BD(v64bits, offset, overhead, bd_index, num_entries) \
do { \
(num_entries) = ((v64bits) & 0xFFFF); \
(bd_index) = (((v64bits) >> 16) & 0xFFFF); \
(overhead) = (((v64bits) >> 32) & 0xFFFF); \
(offset) = (((v64bits) >> 48) & 0xFFFF); \
} while (0)
static inline void
mca_btl_sicortex_map_bds( mca_btl_sicortex_module_t* sicortex_btl,
void* base_addr,
uint16_t offset,
size_t* size,
uint16_t* bd_start,
uint16_t* num_entries )
{
OPAL_THREAD_LOCK( &sicortex_btl->sicortex_lock);
*bd_start = sicortex_btl->bds_head;
*num_entries = 1 + (offset + *size) / sicortex_btl->bds_size;
sicortex_btl->bds_head += *num_entries;
if( sicortex_btl->bds_head >= sicortex_btl->bds_limit ) {
sicortex_btl->bds_head = 0; /* rollback at the begining */
*num_entries = sicortex_btl->bds_limit - *bd_start;
*size = (*size % sicortex_btl->bds_size) + (*num_entries - 1) * sicortex_btl->bds_size;
}
OPAL_THREAD_UNLOCK( &sicortex_btl->sicortex_lock);
/*opal_output(0, "map base_addr %p size %ld [bd_start %d: num_entries %d]",
base_addr, (unsigned long)*size, *bd_start, *num_entries );*/
}
/**
* Pack data and return a descriptor that can be
* used for send/put.
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
*/
static mca_btl_base_descriptor_t*
mca_btl_sicortex_prepare_src( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags )
{
mca_btl_sicortex_module_t *sicortex_btl = (mca_btl_sicortex_module_t *)btl;
mca_btl_sicortex_frag_t* frag;
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data = *size;
int rc;
if( 0 != reserve ) {
if( max_data + reserve <= btl->btl_eager_limit ) {
MCA_BTL_SICORTEX_FRAG_ALLOC_EAGER(sicortex_btl,frag, rc);
} else {
MCA_BTL_SICORTEX_FRAG_ALLOC_MAX(sicortex_btl,frag, rc);
if( (max_data + reserve) > btl->btl_max_send_size ) {
max_data = btl->btl_max_send_size - reserve;
}
}
if(OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
frag->segment.seg_addr.pval = (void*)((unsigned char*)(frag + 1));
iov.iov_len = max_data;
iov.iov_base = (unsigned char*)frag->segment.seg_addr.pval + reserve;
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data );
*size = max_data;
if( rc < 0 ) {
MCA_BTL_SICORTEX_FRAG_RETURN(sicortex_btl, frag);
return NULL;
}
frag->segment.seg_len = max_data + reserve;
} else { /* this is a real RDMA operation */
uint16_t bd_index, num_entries, offset;
void* base_addr;
MCA_BTL_SICORTEX_FRAG_ALLOC_USER(sicortex_btl,frag, rc);
if(NULL == frag) {
return NULL;
}
/**
* The best we can do is to decrease the number of bd we will have to map and
* to decrease the number of over 64K segments we have to transfer. Therefore,
* we will limit the amount of data to be sent to whatever is on the first 64K
* fragment plus a multiple of 64K.
* We know that the offset and the overhead will be 16 bits each. Similarly,
* the bd_index and num_entries are smaller than 16 bits each, so we can
* pack all required information in 64 bits.
*/
ompi_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
base_addr = (void*)(frag->segment.seg_addr.lval & ~((uintptr_t)0xFFFF));
offset = frag->segment.seg_addr.lval & (uintptr_t)0xffff;
mca_btl_sicortex_map_bds( sicortex_btl, base_addr, offset,
size, &bd_index, &num_entries);
assert( *size != 0 );
iov.iov_base = NULL;
iov.iov_len = *size;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data);
*size = max_data;
assert( 0 != iov_count );
frag->segment.seg_addr.pval = iov.iov_base;
frag->segment.seg_len = max_data;
assert( frag->segment.seg_len != 0 );
assert( frag->segment.seg_addr.pval != NULL );
rc = scdma_map_bds(sicortex_btl->ctx, bd_index,
base_addr, num_entries);
frag->base.des_src = &(frag->segment);
MCA_BTL_SICORTEX_PACK_BD( frag->base.des_src->seg_key.key64,
offset, 0, bd_index, num_entries );
/*opal_output(0, "prepare_src index %d offset %d (num_entries %d) length %d [base %p addr %p]",
bd_index, offset, num_entries, (int)max_data, base_addr, (void*)iov.iov_base );*/
}
frag->base.des_src = &(frag->segment);
frag->base.des_src_cnt = 1;
frag->base.order = MCA_BTL_NO_ORDER;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = flags;
return &frag->base;
}
/**
* Prepare a descriptor for send/rdma using the supplied
* convertor. If the convertor references data that is contigous,
* the descriptor may simply point to the user buffer. Otherwise,
* this routine is responsible for allocating buffer space and
* packing if required.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL peer addressing
* @param convertor (IN) Data type convertor
* @param reserve (IN) Additional bytes requested by upper layer to precede user data
* @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
*/
static mca_btl_base_descriptor_t*
mca_btl_sicortex_prepare_dst( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct ompi_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl;
mca_btl_sicortex_frag_t* frag;
uint16_t bd_index, offset, num_entries;
int rc, length;
void* base_addr;
MCA_BTL_SICORTEX_FRAG_ALLOC_USER(btl, frag, rc);
if(OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
/* Ignore all about alignments. We will cope with these on the _put function */
ompi_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
offset = (frag->segment.seg_addr.lval & (uintptr_t)0xffff);
base_addr = (void*)(frag->segment.seg_addr.lval & ~((uintptr_t)0xffff));
mca_btl_sicortex_map_bds( sicortex_btl, base_addr, offset,
size, &bd_index, &num_entries );
frag->segment.seg_len = *size;
length = *size;
rc = scdma_map_bds(sicortex_btl->ctx, bd_index,
base_addr, num_entries);
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = &frag->segment;
MCA_BTL_SICORTEX_PACK_BD(frag->base.des_dst->seg_key.key64,
offset, 0, bd_index, num_entries );
/*opal_output(0, "prepare_dst index %d offset %d (num_entries %d) length %d [base %p addr %p]",
bd_index, offset, num_entries, length, base_addr, (void*)frag->segment.seg_addr.pval);*/
frag->base.des_dst_cnt = 1;
frag->base.des_flags = flags;
return &frag->base;
}
#define MCA_BTL_SICORTEX_NEXT_BDS(val) \
do { \
if( ++(val) >= sicortex_btl->bds_limit ) \
(val) = 0; \
} while(0)
/**
* Initiate an asynchronous put.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
static int
mca_btl_sicortex_put( mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* des )
{
mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl;
mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*) des;
scdma_hw_put_bf_bf_cmd_t *pb;
uint32_t dst_offset, dst_overhead, src_offset, src_overhead;
uint32_t dst_bd_index, dst_num_entries, src_bd_index, src_num_entries;
uint32_t total_length = des->des_src->seg_len, seg_length, align;
uintptr_t dst_ptr, src_ptr;
frag->endpoint = endpoint;
frag->count = 0;
MCA_BTL_SICORTEX_UNPACK_BD( des->des_src->seg_key.key64, src_offset, src_overhead,
src_bd_index, src_num_entries );
MCA_BTL_SICORTEX_UNPACK_BD( des->des_dst->seg_key.key64, dst_offset, dst_overhead,
dst_bd_index, dst_num_entries );
src_ptr = des->des_src->seg_addr.lval;
dst_ptr = des->des_dst->seg_addr.lval;
/*opal_output(0, "src key %llx (src_ptr %llx) dst key %llx (dst_ptr %llx)",
(unsigned long long)des->des_src->seg_key.key64, (unsigned long long)src_ptr,
(unsigned long long)des->des_dst->seg_key.key64, (unsigned long long)dst_ptr );*/
/* Let's check for the conditions imposed by the hardware:
* - If the destination buffer does not start and end at cache block (32-byte)
* boundaries, software must use another mechanism (probably built upon
* Send Event) to deliver the data which belongs in partial blocks.
* - If the destination buffer does not start at a 64-byte boundary, the
* transfer will make most efficient use of the memory bandwidth if the
* library uses a short segment to achieve 64-byte alignment for the
* bulk of the transfer.
* - If the source and destination buffers do not have the same alignment,
* the starting offset in the source buffer should be specified to align the
* first packet to a 64-byte boundary at the destination.
* - If the source and destination buffer alignments differ by an amount which
* is not a multiple of 4, the alignment must be adjusted by a software copy
* before or after the transfer.
*/
if( (align = (src_ptr ^ dst_ptr) & 0x3) ) {
opal_output(0, "Impossible to do a direct PUT of %d bytes. Data will have to be moved around by %d bytes (src_ptr %p dst_ptr %p)",
total_length, (int)((src_ptr | dst_ptr) & 0x3), (void*)src_ptr, (void*)dst_ptr);
align = dst_ptr & 0x3;
src_ptr = ((src_ptr + 4) & ~0x3) | align;
align = des->des_src->seg_addr.lval - src_ptr;
total_length -= align;
/*return OMPI_ERR_OUT_OF_RESOURCE;*/
}
/* Send a small amount of data in such a way that the destination PUT will
* be aligned on the cache boundaries (64 bytes).
*/
if( (align = (0x20 - (dst_ptr & 0x1F))) ) {
uint32_t length_on_wire = scdma_send_event_len(align + 3 * sizeof(uint64_t));
scdma_hw_cmd_t* cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx);
cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT,
length_on_wire,
endpoint->route->route_handles[sicortex_btl->handle_count],
endpoint->route->ports[sicortex_btl->handle_count]);
cmd->payload[0] = (((uint64_t)0 << 16) | (EVENT_SEND_PUT_ALIGN << 8)
| (align + 3 * sizeof(uint64_t)) );
cmd->payload[1] = dst_ptr;
memcpy( &(cmd->payload[2]), (void*)src_ptr, align );
cmd->payload[15] = 0;
scdma_cq_post_fastpath(sicortex_btl->ctx, cmd->header);
src_ptr += align;
dst_ptr += align;
total_length -= align;
dst_offset += align;
/*opal_output(0, "Align the begining of the destination buffer by %d (dst_ptr %llx dst_offset %d)\n",
align, (unsigned long long)dst_ptr, dst_offset );*/
if( dst_offset >= (64*1024) ) {
assert( 0 == (dst_offset % (64*1024)) );
dst_offset = 0;
MCA_BTL_SICORTEX_NEXT_BDS(dst_bd_index);
dst_num_entries--;
}
src_offset += align;
if( src_offset >= (64*1024) ) {
src_offset %= (64*1024);
MCA_BTL_SICORTEX_NEXT_BDS(src_bd_index);
src_num_entries--;
}
}
/* Send a small amount of data to make sure that the last PUT will
* stop on a cache boundary alignment (64 bytes).
*/
if( (align = (total_length & 0x1F)) ) {
uint32_t length_on_wire = scdma_send_event_len(align + 3 * sizeof(uint64_t));
scdma_hw_cmd_t* cmd = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(sicortex_btl->ctx);
cmd->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT,
length_on_wire,
endpoint->route->route_handles[sicortex_btl->handle_count],
endpoint->route->ports[sicortex_btl->handle_count]);
cmd->payload[0] = (((uint64_t)0 << 16) | (EVENT_SEND_PUT_ALIGN << 8)
| (align + 3 * sizeof(uint64_t)) );
cmd->payload[1] = dst_ptr + total_length - align;
memcpy( &(cmd->payload[2]), (void*)(src_ptr + total_length - align), align );
cmd->payload[15] = 0;
scdma_cq_post_fastpath(sicortex_btl->ctx, cmd->header);
total_length -= align;
/*opal_output(0, "Align the end of the destination buffer by %d (dst_ptr %llx)\n",
align, (unsigned long long)(dst_ptr + total_length) );*/
}
/* We can use overlapping on the remote side, but not locally. Therefore, we will
* compute the size of the first chunck, which will allow us to align locally on
* the 64K boundary. Then we will put 64K by 64K, except for the last put,
* where we will have to recompute the overhead.
*/
seg_length = (64 * 1024) - dst_offset;
do {
if( seg_length > total_length ) {
seg_length = total_length;
}
frag->count++; /* count how many puts per fragment */
/* Prepare the PUT order */
pb = (scdma_hw_put_bf_bf_cmd_t*)scdma_cq_head_spinwait(sicortex_btl->ctx);
scdma_build_put_bf_bf_cmd( (void*)pb,
endpoint->route->route_handles[sicortex_btl->handle_count],
endpoint->route->ports[sicortex_btl->handle_count],
src_bd_index, src_offset,
dst_bd_index, dst_offset,
seg_length, 0, (uintptr_t)frag );
scdma_cq_post(sicortex_btl->ctx);
/*opal_output(0, "put src(index %d:offset %d) dst(index %d:offset %d) length %d route %d",
src_bd_index, src_offset, dst_bd_index, dst_offset, seg_length,
sicortex_btl->handle_count);*/
/* Update local informations: the route and the count */
sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM;
/* Now let's update the values */
src_offset = src_offset + seg_length;
if( src_offset >= (64 * 1024) ) {
src_offset = src_offset % (64 * 1024);
MCA_BTL_SICORTEX_NEXT_BDS(src_bd_index);
}
dst_offset = 0; /* We're now indexed by destination pages */
MCA_BTL_SICORTEX_NEXT_BDS(dst_bd_index);
total_length -= seg_length; /* Update the remaining length */
seg_length = 64 * 1024; /* We're dealing now with full pages */
} while (total_length != 0);
return OMPI_SUCCESS;
}
/**
* Initiate an asynchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferre
*
*/
#if 0
static int
mca_btl_sicortex_get( mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* des)
{
mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl;
mca_btl_sicortex_frag_t* frag = (mca_btl_sicortex_frag_t*) des;
void * cmd;
mca_btl_base_segment_t *src = des->des_src;
mca_btl_base_segment_t *dst = des->des_dst;
int seg_length = src->seg_len;
uint32_t tx_bd_index =(uint32_t)((src->seg_key.key64 >> 48) &0xffff);/* src->seg_key.key32[0];*/
uint32_t tx_bd_offset =(uint32_t)((src->seg_key.key64 >> 32) &0xffff); /*src->seg_key.key32[1];*/
uint32_t remote_route_handle = (uint32_t)(src->seg_key.key64 &0xffffffff);
uint32_t rx_begin_bd_index =(uint32_t)((dst->seg_key.key64 >> 48) &0xffff);
uint32_t rx_begin_bd_offset=(uint32_t) ((dst->seg_key.key64 >> 32) & 0xffff);
uint32_t rx_end_bd_index = (uint32_t)((dst->seg_key.key64 >>16) &0xffff);
uint32_t rx_end_bd_offset = (uint32_t)(dst->seg_key.key64 & 0xffff);
uint64_t sw_bucket = (uintptr_t)frag;
int length = rx_begin_bd_offset + seg_length;
uint32_t counter = rx_begin_bd_index;
int rx_bd_offset;
int tx_bd_index_now;
int tx_bd_index_next = tx_bd_index;
int tx_bd_offset_next = tx_bd_offset;
int tx_bd_offset_now;
sw_bucket = sw_bucket | 0x1;
/* opal_output(0,"sender %llx tx_bd_index %d\t tx_bd_offset %d\trx_begin_bd_index %d\t rx_begin_bd_offset %d\trx_end_bd_index%d\trx_end_bd_offset%d\t ",sw_bucket, tx_bd_index,tx_bd_offset,rx_begin_bd_index,rx_begin_bd_offset,rx_end_bd_index,rx_end_bd_offset);*/
frag->endpoint = endpoint;
frag->count = 0;
frag->bds_index = tx_bd_index;
frag->remote_bds_start= rx_begin_bd_offset;
do {
frag->count++;
rx_bd_offset = 0;
tx_bd_offset_now = tx_bd_offset_next;
tx_bd_index_now = tx_bd_index_next;
if(counter == rx_begin_bd_index) {
rx_bd_offset = rx_begin_bd_offset;
if(length > BD_SIZE) {
seg_length = BD_SIZE - rx_begin_bd_offset;
length -= BD_SIZE;
} else {
seg_length = seg_length;
length =0;
}
} else if(counter == rx_end_bd_index) {
seg_length = rx_end_bd_offset;
length = 0;
} else if(counter> rx_begin_bd_index && counter< rx_end_bd_index) {
seg_length = BD_SIZE;
length -= BD_SIZE;
} else {
opal_output(0,"hahaha, u r wrong!!!");
}
if( tx_bd_offset_now + seg_length >= BD_SIZE )
tx_bd_index_next = tx_bd_index_now+1;
tx_bd_offset_next = (tx_bd_offset_now + seg_length) %(BD_SIZE);
cmd = (void*)scdma_cq_head_spinwait(sicortex_btl->ctx);
scdma_build_send_put_bf_bf_cmd(cmd,
endpoint->route->route_handles[sicortex_btl->handle_count],
endpoint->route->ports[sicortex_btl->handle_count],
remote_route_handle,
tx_bd_index_now,
tx_bd_offset_now,
counter,
rx_bd_offset,
seg_length,
0,
sw_bucket);
sicortex_btl->handle_count = (sicortex_btl->handle_count + 1) % SICORTEX_PORTS_NUM;
scdma_cq_post(sicortex_btl->ctx);
} while( counter++ != rx_end_bd_index );
return OMPI_SUCCESS;
}
#endif
/*
* Cleanup/release module resources.
*/
static int mca_btl_sicortex_finalize(struct mca_btl_base_module_t* btl)
{
mca_btl_sicortex_module_t* sicortex_btl = (mca_btl_sicortex_module_t*) btl;
OBJ_DESTRUCT(&sicortex_btl->sicortex_lock);
free(sicortex_btl);
return OMPI_SUCCESS;
}
static int mca_btl_sicortex_ft_event(int state)
{
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
;
}
else if(OPAL_CRS_RESTART == state) {
;
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}
mca_btl_sicortex_module_t mca_btl_sicortex_module = {
{
&mca_btl_sicortex_component.super,
0, /* max size of first fragment */
0, /* min send fragment size*/
0, /* max send fragment size */
0, /* rdma pipeline offset */
0, /* rdma pipeline frag size */
0, /* min rdma pipeline size */
0, /* exclusivity */
0, /* latency */
0, /* bandwidth */
0, /* flags */
mca_btl_sicortex_add_procs,
mca_btl_sicortex_del_procs,
NULL,
mca_btl_sicortex_finalize,
mca_btl_sicortex_alloc,
mca_btl_sicortex_free,
mca_btl_sicortex_prepare_src,
mca_btl_sicortex_prepare_dst,
mca_btl_sicortex_send,
mca_btl_sicortex_sendi,/* send immediate */
mca_btl_sicortex_put, /* put */
#if 0
mca_btl_sicortex_get, /* get */
#else
NULL, /* get */
#endif
NULL, /*dump */
NULL, /* mpool */
NULL, /* register error cb */
mca_btl_sicortex_ft_event
}
};

166
ompi/mca/btl/sicortex/btl_sicortex.h Исполняемый файл
Просмотреть файл

@ -0,0 +1,166 @@
/*
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_SICORTEX_H
#define MCA_PTL_SICORTEX_H
/* Standard system includes */
#include <sys/types.h>
#include <string.h>
/* Open MPI includes */
#include "opal/class/opal_bitmap.h"
#include "opal/event/event.h"
#include "orte/util/show_help.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/mca/btl/btl.h"
#include <unistd.h>
#include <linux/sicortex/scdma_hw.h>
#include <linux/sicortex/scdma.h>
#include <assert.h>
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
#define MCA_BTL_HAS_MPOOL 0
#define SICORTEX_PORTS_NUM 3
#define BD_SIZE 64*1024
/*
* Infiniband (SICORTEX) BTL component.
*/
enum eventcodes { EVENT_SEND = (SCDMA_HW_EVENT_TYPE_LIMIT + 1),
EVENT_SEND_IM,
EVENT_SEND_PUT_ALIGN,
EVENT_SEND_IM_PUT_POST,
EVENT_PUT };
struct peer_struct_t {
uint64_t context_id;
uint32_t route_handles[3];
uint32_t ports[3];
};
typedef struct peer_struct_t peer_struct_t;
struct mca_btl_sicortex_component_t {
mca_btl_base_component_2_0_0_t super; /**< base BTL component */
size_t sicortex_num_btls;
/**< number of hcas available to the SICORTEX component */
size_t sicortex_max_btls;
struct mca_btl_sicortex_module_t **sicortex_btls;
/**< array of available BTL modules */
int sicortex_free_list_num;
/**< initial size of free lists */
int sicortex_free_list_max;
/**< maximum size of free lists */
int sicortex_free_list_inc;
/**< number of elements to alloc when growing free lists */
opal_list_t sicortex_procs;
/**< list of sicortex proc structures */
opal_mutex_t sicortex_lock;
/**< lock for accessing module state */
ompi_free_list_t sicortex_frag_eager;
ompi_free_list_t sicortex_frag_max;
ompi_free_list_t sicortex_frag_user;
char* sicortex_mpool_name;
/**< name of memory pool */
bool leave_pinned;
/**< pin memory on first use and leave pinned */
};
typedef struct mca_btl_sicortex_component_t mca_btl_sicortex_component_t;
OMPI_MODULE_DECLSPEC extern mca_btl_sicortex_component_t mca_btl_sicortex_component;
/**
* BTL Module Interface
*/
struct mca_btl_sicortex_module_t {
mca_btl_base_module_t super;
scdma_context_t *ctx;
uint64_t localContext;
struct mca_btl_base_endpoint_t** peers;
uint32_t bds_size;
uint32_t bds_limit;
uint32_t bds_head;
int handle_count;
opal_mutex_t sicortex_lock;
#if MCA_BTL_HAS_MPOOL
struct mca_mpool_base_module_t* sicortex_mpool;
#endif
};
typedef struct mca_btl_sicortex_module_t mca_btl_sicortex_module_t;
extern mca_btl_sicortex_module_t mca_btl_sicortex_module;
/**
* Register SICORTEX component parameters with the MCA framework
*/
extern int mca_btl_sicortex_component_open(void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_btl_sicortex_component_close(void);
/**
* SICORTEX component initialization.
*
* @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
* @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
* @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
*/
extern mca_btl_base_module_t** mca_btl_sicortex_component_init(
int *num_btl_modules,
bool allow_multi_user_threads,
bool have_hidden_threads
);
/**
* SICORTEX component progress.
*/
extern int mca_btl_sicortex_component_progress(void);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

415
ompi/mca/btl/sicortex/btl_sicortex_component.c Исполняемый файл
Просмотреть файл

@ -0,0 +1,415 @@
/*
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "opal/event/event.h"
#include "opal/util/if.h"
#include "opal/util/argv.h"
#include "orte/util/show_help.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "orte/mca/errmgr/errmgr.h"
#include "ompi/mca/mpool/base/base.h"
#include "btl_sicortex.h"
#include "btl_sicortex_frag.h"
#include "btl_sicortex_endpoint.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/datatype/convertor.h"
mca_btl_sicortex_component_t mca_btl_sicortex_component = {
{
/* First, the mca_base_component_t struct containing meta information
about the component itself */
{
/* Indicate that we are a pml v1.0.0 component (which also implies a
specific MCA version) */
MCA_BTL_BASE_VERSION_2_0_0,
"sicortex", /* MCA component name */
OMPI_MAJOR_VERSION, /* MCA component major version */
OMPI_MINOR_VERSION, /* MCA component minor version */
OMPI_RELEASE_VERSION, /* MCA component release version */
mca_btl_sicortex_component_open, /* component open */
mca_btl_sicortex_component_close /* component close */
},
/* Next the MCA v1.0.0 component meta data */
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
mca_btl_sicortex_component_init,
mca_btl_sicortex_component_progress,
}
};
/*
* utility routines for parameter registration
*/
static inline char*
mca_btl_sicortex_param_register_string( const char* param_name,
const char* default_value )
{
char *param_value;
int id = mca_base_param_register_string( "btl", "sicortex", param_name,
NULL, default_value );
mca_base_param_lookup_string(id, &param_value);
return param_value;
}
static inline int
mca_btl_sicortex_param_register_int( const char* param_name,
int default_value )
{
int id = mca_base_param_register_int( "btl", "sicortex", param_name,
NULL, default_value );
int param_value = default_value;
mca_base_param_lookup_int(id,&param_value);
return param_value;
}
/*
* Called by MCA framework to open the component, registers
* component parameters.
*/
int mca_btl_sicortex_component_open(void)
{
/* initialize state */
mca_btl_sicortex_component.sicortex_num_btls = 0;
mca_btl_sicortex_component.sicortex_btls = NULL;
/* initialize objects */
OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_procs, opal_list_t);
OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_lock, opal_mutex_t);
/* register SICORTEX component parameters */
mca_btl_sicortex_component.sicortex_max_btls =
mca_btl_sicortex_param_register_int("max_modules", 4);
mca_btl_sicortex_component.sicortex_free_list_num =
mca_btl_sicortex_param_register_int ("free_list_num", 8);
mca_btl_sicortex_component.sicortex_free_list_max =
mca_btl_sicortex_param_register_int ("free_list_max", 1024);
mca_btl_sicortex_component.sicortex_free_list_inc =
mca_btl_sicortex_param_register_int ("free_list_inc", 32);
mca_btl_sicortex_component.sicortex_mpool_name =
mca_btl_sicortex_param_register_string("mpool", "sicortex");
mca_btl_sicortex_module.super.btl_exclusivity = 0;
mca_btl_sicortex_module.super.btl_eager_limit = 2400;/* 4*1024 - sizeof(mca_btl_base_header_t);*/
mca_btl_sicortex_module.super.btl_rndv_eager_limit = 4*1024;/*4*1024 - sizeof(mca_btl_base_header_t);*/
mca_btl_sicortex_module.super.btl_max_send_size = 4*1024;/*4*1024 - sizeof(mca_btl_base_header_t);*/
mca_btl_sicortex_module.super.btl_min_rdma_pipeline_size = 1024*1024;
mca_btl_sicortex_module.super.btl_rdma_pipeline_frag_size = 1024*1024;
mca_btl_sicortex_module.super.btl_rdma_pipeline_send_length = 1024*1024;
mca_btl_sicortex_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_PUT;
mca_btl_sicortex_module.super.btl_latency = 3; /* Microsec */
mca_btl_sicortex_module.super.btl_bandwidth = 14000; /* Mbs */
mca_btl_base_param_register(&mca_btl_sicortex_component.super.btl_version,
&mca_btl_sicortex_module.super);
return OMPI_SUCCESS;
}
/*
* component cleanup - sanity checking of queue lengths
*/
int mca_btl_sicortex_component_close(void)
{
if( 0 != mca_btl_sicortex_component.sicortex_num_btls ) {
OBJ_DESTRUCT(&mca_btl_sicortex_component.sicortex_frag_eager);
OBJ_DESTRUCT(&mca_btl_sicortex_component.sicortex_frag_max);
OBJ_DESTRUCT(&mca_btl_sicortex_component.sicortex_frag_user);
mca_btl_sicortex_component.sicortex_num_btls = 0;
free(mca_btl_sicortex_component.sicortex_btls);
mca_btl_sicortex_component.sicortex_btls = NULL;
}
return OMPI_SUCCESS;
}
static int mca_btl_sicortex_modex_send(void)
{
int rc;
size_t i;
size_t size;
uint64_t *addrs = NULL;
size = mca_btl_sicortex_component.sicortex_num_btls * sizeof( uint64_t);
if(0 != size){
addrs = (uint64_t*) malloc(size);
for( i = 0; i < mca_btl_sicortex_component.sicortex_num_btls; i++ ) {
mca_btl_sicortex_module_t *btl = mca_btl_sicortex_component.sicortex_btls[i];
addrs[i]= btl->localContext;
}
}
rc = ompi_modex_send(&mca_btl_sicortex_component.super.btl_version, addrs, size);
if( NULL != addrs ) {
free(addrs);
}
return rc;
}
mca_btl_base_module_t**
mca_btl_sicortex_component_init( int *num_btl_modules,
bool enable_progress_threads,
bool enable_mpi_threads )
{
int rc;
mca_btl_base_module_t **btls;
uint64_t localContext;
mca_btl_sicortex_module_t *btl;
scdma_context_t *ctx = (scdma_context_t*) malloc(sizeof(scdma_context_t));
*num_btl_modules = 0;
/*init scdma*/
rc = scdma_open( -1, /* Process (-1 for ANY) */
ctx, /* the context... */
131072, /* event queue size */
16384, /* cmd queue size */
65536, /* heap size */
4096, /* num RDT entries */
4096); /* num BDT entries */
if( rc ) {
opal_output( 0, "[%s:%d] error in initializing the sicortex library\n", __FILE__, __LINE__ );
return NULL;
}
/* get context of each card */
localContext = scdma_get_my_context_id(ctx);
mca_btl_sicortex_component.sicortex_btls = malloc( mca_btl_sicortex_component.sicortex_max_btls * sizeof (mca_btl_sicortex_module_t*));
if( NULL == mca_btl_sicortex_component.sicortex_btls ) {
return NULL;
}
mca_btl_sicortex_component.sicortex_num_btls = 0;
btl = (mca_btl_sicortex_module_t *)malloc( sizeof(mca_btl_sicortex_module_t) );
memcpy (btl, &mca_btl_sicortex_module, sizeof(mca_btl_sicortex_module_t));
btl->localContext = localContext;
btl->ctx = ctx;
btl->handle_count = 0;
OBJ_CONSTRUCT(&btl->sicortex_lock, opal_mutex_t);
mca_btl_sicortex_component.sicortex_btls[mca_btl_sicortex_component.sicortex_num_btls] = btl;
mca_btl_sicortex_component.sicortex_num_btls++;
if (OMPI_SUCCESS != mca_btl_sicortex_modex_send()) {
return NULL;
}
/* initialize free lists */
OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_frag_eager, ompi_free_list_t);
ompi_free_list_init_new( &mca_btl_sicortex_component.sicortex_frag_eager,
sizeof(mca_btl_sicortex_frag_eager_t) + mca_btl_sicortex_module.super.btl_eager_limit,
CACHE_LINE_SIZE,
OBJ_CLASS (mca_btl_sicortex_frag_eager_t),
0,
CACHE_LINE_SIZE,
mca_btl_sicortex_component.sicortex_free_list_num,
mca_btl_sicortex_component.sicortex_free_list_max,
mca_btl_sicortex_component.sicortex_free_list_inc,
btl->super.btl_mpool );
OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_frag_max, ompi_free_list_t);
ompi_free_list_init_new( &mca_btl_sicortex_component.sicortex_frag_max,
sizeof (mca_btl_sicortex_frag_max_t) + mca_btl_sicortex_module.super.btl_max_send_size,
CACHE_LINE_SIZE,
OBJ_CLASS (mca_btl_sicortex_frag_max_t),
0,
CACHE_LINE_SIZE,
mca_btl_sicortex_component.sicortex_free_list_num,
mca_btl_sicortex_component.sicortex_free_list_max,
mca_btl_sicortex_component.sicortex_free_list_inc,
btl->super.btl_mpool );
OBJ_CONSTRUCT(&mca_btl_sicortex_component.sicortex_frag_user, ompi_free_list_t);
ompi_free_list_init_new( &mca_btl_sicortex_component.sicortex_frag_user,
sizeof (mca_btl_sicortex_frag_user_t),
CACHE_LINE_SIZE,
OBJ_CLASS (mca_btl_sicortex_frag_user_t),
0,
CACHE_LINE_SIZE,
mca_btl_sicortex_component.sicortex_free_list_num,
mca_btl_sicortex_component.sicortex_free_list_max,
mca_btl_sicortex_component.sicortex_free_list_inc,
NULL );
/* return array of BTLs */
btls = (mca_btl_base_module_t**) malloc (
mca_btl_sicortex_component.sicortex_num_btls * sizeof(mca_btl_base_module_t *));
if( NULL == btls ){
return NULL;
}
memcpy(btls, mca_btl_sicortex_component.sicortex_btls,
mca_btl_sicortex_component.sicortex_num_btls * sizeof(mca_btl_sicortex_module_t *));
*num_btl_modules = mca_btl_sicortex_component.sicortex_num_btls;
return btls;
}
int mca_btl_sicortex_component_progress(void)
{
int count = 0;
size_t i;
uint64_t *evt;
for( i = 0; i < mca_btl_sicortex_component.sicortex_num_btls; i++ ) {
mca_btl_sicortex_module_t* btl = mca_btl_sicortex_component.sicortex_btls[i];
int pooling = 0;
while( pooling++ < 5000 ) {
evt = (uint64_t *) scdma_eq_next_event(btl->ctx);
if( NULL != evt ) break;
}
if( NULL == evt ) continue;
switch ((evt[0] >> 8) & 0xff) {
case SCDMA_HW_EVENT_TYPE_RDT_FAULT: {
uint64_t sw_bucket = evt[1];
opal_output( 0, "Error RDT_FAULT sw_bucket %llx", (unsigned long long)sw_bucket );
scdma_eq_processed_event(btl->ctx);
break;
}
case SCDMA_HW_EVENT_TYPE_BDT_FAULT: {
uint64_t sw_bucket = evt[1];
opal_output( 0, "Error BDT_FAULT sw_bucket %llx", (unsigned long long)sw_bucket );
scdma_eq_processed_event(btl->ctx);
break;
}
case SCDMA_HW_EVENT_TYPE_SEG_ABORT: {
uint64_t sw_bucket = evt[1];
opal_output( 0, "Error SEG_ABORT sw_bucket %llx", (unsigned long long)sw_bucket );
scdma_eq_processed_event(btl->ctx);
break;
}
case SCDMA_HW_EVENT_TYPE_DEFERRED_CMD: {
void *cmd = (void *) scdma_cq_head_spinwait(btl->ctx);
memcpy(cmd, &evt[1], 0x78);
scdma_cq_post(btl->ctx);
scdma_eq_processed_event(btl->ctx);
break;
}
case SCDMA_HW_EVENT_TYPE_RX_END_SEG: {
scdma_hw_cmd_t *cmd_2;
uint64_t swbucket = evt[1];
scdma_eq_processed_event(btl->ctx);
cmd_2 = (scdma_hw_cmd_t *)scdma_cq_head_spinwait(btl->ctx);
cmd_2->header = scdma_cmd_header(SCDMA_HW_CMD_TYPE_SEND_EVENT, 0x18,
btl->peers[0]->route->route_handles[btl->handle_count],
btl->peers[0]->route->ports[btl->handle_count]);
btl->handle_count = (btl->handle_count +1) % SICORTEX_PORTS_NUM;
cmd_2->payload[0] = (EVENT_PUT << 8) + 0x18;
cmd_2->payload[1] = swbucket;
cmd_2->payload[2] = 0;
scdma_cq_post_fastpath(btl->ctx,cmd_2->header);
break;
}
/* Receive small message */
case EVENT_SEND_IM: {
mca_btl_active_message_callback_t* reg;
mca_btl_sicortex_frag_t frag;
frag.base.des_dst = &frag.segment;
frag.base.des_dst->seg_addr.pval = (void*) (evt+1);
frag.base.des_dst->seg_len = (size_t)((evt[0] & SCDMA_HW_CMD_LEN_MASK)
- 2 * sizeof(uint64_t));
frag.base.des_dst_cnt = 1;
frag.tag = (mca_btl_base_tag_t)((evt[0] >> 16) & 0xff);
reg = mca_btl_base_active_message_trigger + frag.tag;
reg->cbfunc( &(btl->super), frag.tag, &(frag.base), reg->cbdata );
scdma_eq_processed_event(btl->ctx);
count++;
break;
}
case EVENT_SEND_PUT_ALIGN: {
/* There are at most 2 PUT_ALIGN per segment, one of the begining and
* one for the end. As the DMA guarantee that the orders in each route and port
* are completed in order, we don't have to care about these two PUT_ALIGN
* as they will always be followed by a real PUT on the same route. When the
* PUT complete, these two PUT_ALIGN have to be completed.
*/
uint32_t length = ((evt[0] & SCDMA_HW_CMD_LEN_MASK) - 3 * sizeof(uint64_t));
void* ptr = (void*)evt[1];
memcpy( ptr, &(evt[2]), length );
/*opal_output(0, "PUT_ALIGN ptr %p length %d\n", ptr, length );*/
scdma_eq_processed_event(btl->ctx);
break;
}
case EVENT_SEND_IM_PUT_POST: {
mca_btl_active_message_callback_t* reg;
mca_btl_sicortex_frag_t frag;
frag.base.des_dst = &frag.segment;
frag.base.des_dst->seg_addr.pval = (void*)((char*)scdma_heap_addr(btl->ctx) + SCDMA_USABLE_HEAP_OFFSET);
frag.base.des_dst->seg_len = (size_t)((evt[0]>> 32 & 0xffffffff));
frag.base.des_dst_cnt = 1;
frag.tag = (mca_btl_base_tag_t)((evt[0]>>16)& 0xff);
reg = mca_btl_base_active_message_trigger + frag.tag;
reg->cbfunc( &(btl->super), frag.tag, &(frag.base), reg->cbdata );
scdma_eq_processed_event(btl->ctx);
count++;
break;
}
case EVENT_PUT: {
mca_btl_sicortex_frag_t *frag = (mca_btl_sicortex_frag_t *)(uintptr_t) evt[1];
if( --frag->count == 0 ) {
int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP );
count++;
frag->base.des_cbfunc( &(btl->super), frag->endpoint,&(frag->base), OMPI_SUCCESS );
if( btl_ownership ) {
MCA_BTL_SICORTEX_FRAG_RETURN(btl, frag);
}
}
scdma_eq_processed_event(btl->ctx);
break;
}
default:
opal_output(0, "Got a badley formatted event (evt[0] = 0x%llx)\n",
(long long unsigned int)evt[0]);
scdma_eq_processed_event(btl->ctx);
sleep(5);
break;
}
}
return count;
}

53
ompi/mca/btl/sicortex/btl_sicortex_endpoint.c Исполняемый файл
Просмотреть файл

@ -0,0 +1,53 @@
/*
* Copyright (c) 2008-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <sys/time.h>
#include <time.h>
#include "ompi/types.h"
#include "orte/mca/oob/base/base.h"
#include "orte/mca/rml/rml.h"
#include "orte/mca/errmgr/errmgr.h"
#include "opal/dss/dss.h"
#include "btl_sicortex.h"
#include "btl_sicortex_endpoint.h"
#include "btl_sicortex_proc.h"
#include "btl_sicortex_frag.h"
/*
* Initialize state of the endpoint instance.
*
*/
static void mca_btl_sicortex_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
{
endpoint->endpoint_btl = 0;
endpoint->endpoint_proc = 0;
}
/*
* Destroy a endpoint
*
*/
static void mca_btl_sicortex_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
{
}
OBJ_CLASS_INSTANCE(
mca_btl_sicortex_endpoint_t,
opal_list_item_t,
mca_btl_sicortex_endpoint_construct,
mca_btl_sicortex_endpoint_destruct);

51
ompi/mca/btl/sicortex/btl_sicortex_endpoint.h Исполняемый файл
Просмотреть файл

@ -0,0 +1,51 @@
/*
* Copyright (c) 2008-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_SICORTEX_ENDPOINT_H
#define MCA_BTL_SICORTEX_ENDPOINT_H
#include "opal/class/opal_list.h"
#include "opal/event/event.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "btl_sicortex_frag.h"
#include "btl_sicortex.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* An abstraction that represents a connection to a endpoint process.
* An instance of mca_btl_base_endpoint_t is associated w/ each process
* and BTL pair at startup. However, connections to the endpoint
* are established dynamically on an as-needed basis:
*/
struct mca_btl_base_endpoint_t {
opal_list_item_t super;
struct mca_btl_sicortex_module_t* endpoint_btl;
/**< BTL instance that created this connection */
struct mca_btl_sicortex_proc_t* endpoint_proc;
/**< proc structure corresponding to endpoint */
struct peer_struct_t* route;
};
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
typedef mca_btl_base_endpoint_t mca_btl_sicortex_endpoint_t;
OBJ_CLASS_DECLARATION(mca_btl_sicortex_endpoint_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

73
ompi/mca/btl/sicortex/btl_sicortex_frag.c Исполняемый файл
Просмотреть файл

@ -0,0 +1,73 @@
/*
* Copyright (c) 2008-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "btl_sicortex_frag.h"
static void mca_btl_sicortex_frag_common_constructor(mca_btl_sicortex_frag_t* frag)
{
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = &(frag->segment);
frag->base.des_dst_cnt = 1;
#if MCA_BTL_HAS_MPOOL
frag->registration = NULL;
#endif
}
static void mca_btl_sicortex_frag_eager_constructor(mca_btl_sicortex_frag_t* frag)
{
frag->size = mca_btl_sicortex_module.super.btl_eager_limit;
frag->my_list = &mca_btl_sicortex_component.sicortex_frag_eager;
mca_btl_sicortex_frag_common_constructor(frag);
}
static void mca_btl_sicortex_frag_max_constructor(mca_btl_sicortex_frag_t* frag)
{
frag->size = mca_btl_sicortex_module.super.btl_max_send_size;
frag->my_list = &mca_btl_sicortex_component.sicortex_frag_max;
mca_btl_sicortex_frag_common_constructor(frag);
}
static void mca_btl_sicortex_frag_user_constructor(mca_btl_sicortex_frag_t* frag)
{
frag->size = 0;
frag->my_list = &mca_btl_sicortex_component.sicortex_frag_user;
mca_btl_sicortex_frag_common_constructor(frag);
}
OBJ_CLASS_INSTANCE(
mca_btl_sicortex_frag_t,
mca_btl_base_descriptor_t,
NULL,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_sicortex_frag_eager_t,
mca_btl_base_descriptor_t,
mca_btl_sicortex_frag_eager_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_sicortex_frag_max_t,
mca_btl_base_descriptor_t,
mca_btl_sicortex_frag_max_constructor,
NULL);
OBJ_CLASS_INSTANCE(
mca_btl_sicortex_frag_user_t,
mca_btl_base_descriptor_t,
mca_btl_sicortex_frag_user_constructor,
NULL);

81
ompi/mca/btl/sicortex/btl_sicortex_frag.h Исполняемый файл
Просмотреть файл

@ -0,0 +1,81 @@
/*
* Copyright (c) 2008-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_SICORTEX_FRAG_H
#define MCA_BTL_SICORTEX_FRAG_H
#include "ompi_config.h"
#include "ompi/class/ompi_free_list.h"
#include "btl_sicortex.h"
BEGIN_C_DECLS
/**
* SICORTEX send frasicortexent derived type.
*/
struct mca_btl_sicortex_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_base_segment_t segment;
ompi_free_list_t* my_list;
struct mca_btl_base_endpoint_t* endpoint;
mca_btl_base_tag_t tag;
int count;
size_t size;
#if MCA_BTL_HAS_MPOOL
struct mca_mpool_base_registration_t* registration;
#endif
};
typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_t);
typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_eager_t;
OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_eager_t);
typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_max_t;
OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_max_t);
typedef struct mca_btl_sicortex_frag_t mca_btl_sicortex_frag_user_t;
OBJ_CLASS_DECLARATION(mca_btl_sicortex_frag_user_t);
/*
* Macros to allocate/return descriptors from module specific
* free list(s).
*/
#define MCA_BTL_SICORTEX_FRAG_ALLOC(list, frag, rc) \
do { \
ompi_free_list_item_t *_item; \
OMPI_FREE_LIST_GET((list), _item, rc); \
frag = (mca_btl_sicortex_frag_t*)_item; \
} while (0)
#define MCA_BTL_SICORTEX_FRAG_RETURN(btl, frag) \
do { \
OMPI_FREE_LIST_RETURN(frag->my_list, \
(ompi_free_list_item_t*)(frag)); \
} while (0)
#define MCA_BTL_SICORTEX_FRAG_ALLOC_EAGER(btl, frag, rc) \
MCA_BTL_SICORTEX_FRAG_ALLOC(&mca_btl_sicortex_component.sicortex_frag_eager, frag, rc )
#define MCA_BTL_SICORTEX_FRAG_ALLOC_MAX(btl, frag, rc) \
MCA_BTL_SICORTEX_FRAG_ALLOC(&mca_btl_sicortex_component.sicortex_frag_max, frag, rc )
#define MCA_BTL_SICORTEX_FRAG_ALLOC_USER(btl, frag, rc) \
MCA_BTL_SICORTEX_FRAG_ALLOC(&mca_btl_sicortex_component.sicortex_frag_user, frag, rc )
END_C_DECLS
#endif

173
ompi/mca/btl/sicortex/btl_sicortex_proc.c Исполняемый файл
Просмотреть файл

@ -0,0 +1,173 @@
/*
* Copyright (c) 2008-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "opal/class/opal_hash_table.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "btl_sicortex.h"
#include "btl_sicortex_proc.h"
static void mca_btl_sicortex_proc_construct(mca_btl_sicortex_proc_t* proc);
static void mca_btl_sicortex_proc_destruct(mca_btl_sicortex_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_btl_sicortex_proc_t,
opal_list_item_t, mca_btl_sicortex_proc_construct,
mca_btl_sicortex_proc_destruct);
void mca_btl_sicortex_proc_construct(mca_btl_sicortex_proc_t* sicortex_proc)
{
sicortex_proc->proc_ompi = 0;
sicortex_proc->proc_addr_count = 0;
sicortex_proc->proc_endpoints = 0;
sicortex_proc->proc_endpoint_count = 0;
OBJ_CONSTRUCT(&sicortex_proc->proc_lock, opal_mutex_t);
/* add to list of all proc instance */
OPAL_THREAD_LOCK(&mca_btl_sicortex_component.sicortex_lock);
opal_list_append(&mca_btl_sicortex_component.sicortex_procs, &sicortex_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock);
}
/*
* Cleanup ib proc instance
*/
void mca_btl_sicortex_proc_destruct(mca_btl_sicortex_proc_t* sicortex_proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK(&mca_btl_sicortex_component.sicortex_lock);
opal_list_remove_item(&mca_btl_sicortex_component.sicortex_procs, &sicortex_proc->super);
OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock);
/* release resources */
if(NULL != sicortex_proc->proc_endpoints) {
free(sicortex_proc->proc_endpoints);
}
}
/*
* Look for an existing SICORTEX process instances based on the associated
* ompi_proc_t instance.
*/
static mca_btl_sicortex_proc_t* mca_btl_sicortex_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_btl_sicortex_proc_t* sicortex_proc;
OPAL_THREAD_LOCK(&mca_btl_sicortex_component.sicortex_lock);
for(sicortex_proc = (mca_btl_sicortex_proc_t*)
opal_list_get_first(&mca_btl_sicortex_component.sicortex_procs);
sicortex_proc != (mca_btl_sicortex_proc_t*)
opal_list_get_end(&mca_btl_sicortex_component.sicortex_procs);
sicortex_proc = (mca_btl_sicortex_proc_t*)opal_list_get_next(sicortex_proc)) {
if(sicortex_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock);
return sicortex_proc;
}
}
OPAL_THREAD_UNLOCK(&mca_btl_sicortex_component.sicortex_lock);
return NULL;
}
/*
* Create a SICORTEX process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_btl_sicortex_proc_t instance. We cache
* additional data (specifically the list of mca_btl_sicortex_endpoint_t instances,
* and published addresses) associated w/ a given destination on this
* datastructure.
*/
mca_btl_sicortex_proc_t* mca_btl_sicortex_proc_create(ompi_proc_t* ompi_proc)
{
mca_btl_sicortex_proc_t* module_proc = NULL;
int rc;
size_t size;
/* Check if we have already created a SICORTEX proc
* structure for this ompi process */
module_proc = mca_btl_sicortex_proc_lookup_ompi(ompi_proc);
if(module_proc != NULL) {
return module_proc;
}
/* Oops! First time, gotta create a new SICORTEX proc
* out of the ompi_proc ... */
module_proc = OBJ_NEW(mca_btl_sicortex_proc_t);
/* Initialize number of peer */
module_proc->proc_endpoint_count = 0;
module_proc->proc_ompi = ompi_proc;
/* build a unique identifier (of arbitrary
* size) to represent the proc */
module_proc->proc_guid = ompi_proc->proc_name;
rc = ompi_modex_recv(
&mca_btl_sicortex_component.super.btl_version,
ompi_proc,
(void*)&module_proc->proc_addrs,
&size);
if(OMPI_SUCCESS != rc) {
OBJ_RELEASE(module_proc);
return NULL;
}
if((size % sizeof(uint64_t))!= 0 ){
OBJ_RELEASE(module_proc);
return NULL;
}
module_proc->proc_addr_count = size/sizeof(uint64_t);
if(0 == module_proc->proc_addr_count){
module_proc->proc_endpoints = NULL;
}else{
module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc(module_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
}
if(NULL == module_proc->proc_endpoints) {
OBJ_RELEASE(module_proc);
return NULL;
}
return module_proc;
}
/*
* Note that this routine must be called with the lock on the process
* already held. Insert a btl instance into the proc array and assign
* it an address.
*/
int mca_btl_sicortex_proc_insert(mca_btl_sicortex_proc_t* module_proc,
mca_btl_base_endpoint_t* module_endpoint)
{
/* insert into endpoint array */
module_endpoint->route = (peer_struct_t*)malloc(sizeof(peer_struct_t));
module_endpoint->route->context_id = module_proc->proc_addrs[module_proc->proc_endpoint_count];
module_endpoint->route->route_handles[0]=((module_proc->proc_endpoint_count*3)+0)<<3;
module_endpoint->route->route_handles[1]=((module_proc->proc_endpoint_count*3)+1)<<3;
module_endpoint->route->route_handles[2]=((module_proc->proc_endpoint_count*3)+2)<<3;
module_endpoint->endpoint_proc = module_proc;
module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint;
return OMPI_SUCCESS;
}

63
ompi/mca/btl/sicortex/btl_sicortex_proc.h Исполняемый файл
Просмотреть файл

@ -0,0 +1,63 @@
/*
* Copyright (c) 2008-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_BTL_SICORTEX_PROC_H
#define MCA_BTL_SICORTEX_PROC_H
#include "opal/class/opal_object.h"
#include "ompi/proc/proc.h"
#include "btl_sicortex.h"
#include "btl_sicortex_endpoint.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* Represents the state of a remote process and the set of addresses
* that it exports. Also cache an instance of mca_btl_base_endpoint_t for
* each
* BTL instance that attempts to open a connection to the process.
*/
struct mca_btl_sicortex_proc_t {
opal_list_item_t super;
/**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi;
/**< pointer to corresponding ompi_proc_t */
orte_process_name_t proc_guid;
/**< globally unique identifier for the process */
uint64_t* proc_addrs;
size_t proc_addr_count;
/**< number of addresses published by endpoint */
struct mca_btl_base_endpoint_t **proc_endpoints;
/**< array of endpoints that have been created to access this proc */
size_t proc_endpoint_count;
/**< number of endpoints */
opal_mutex_t proc_lock;
/**< lock to protect against concurrent access to proc state */
};
typedef struct mca_btl_sicortex_proc_t mca_btl_sicortex_proc_t;
OBJ_CLASS_DECLARATION(mca_btl_sicortex_proc_t);
mca_btl_sicortex_proc_t* mca_btl_sicortex_proc_create(ompi_proc_t* ompi_proc);
int mca_btl_sicortex_proc_insert(mca_btl_sicortex_proc_t*, mca_btl_base_endpoint_t*);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

71
ompi/mca/btl/sicortex/configure.m4 Исполняемый файл
Просмотреть файл

@ -0,0 +1,71 @@
# -*- shell-script -*-
#
# Copyright (c) 2008-2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# OMPI_CHECK_SICORTEX(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if Sicortex support can be found. sets prefix_{CPPFLAGS,
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
# support, otherwise executes action-if-not-found
AC_DEFUN([OMPI_CHECK_SICORTEX],[
AC_ARG_WITH([sicortex],
[AC_HELP_STRING([--with-sicortex(=DIR)],
[Build Sicortex (scdma) support, searching for libraries in DIR])])
AC_ARG_WITH([sicortex-libdir],
[AC_HELP_STRING([--with-sicortex-libdir=DIR],
[Search for Sicortex (scdma) libraries in DIR])])
AS_IF([test "$with_sicortex" != "no"],
[AS_IF([test ! -z "$with_sicortex" -a "$with_sicortex" != "yes"],
[ompi_check_sicortex_dir="$with_sicortex"])
AS_IF([test ! -z "$with_sicortex_libdir" -a "$with_sicortex_libdir" != "yes"],
[ompi_check_sicortex_libdir="$with_sicortex_libdir"])
OMPI_CHECK_PACKAGE([$1],
[sicortex/scdma.h],
[scdma],
[scdma_open],
[],
[$ompi_check_sicortex_dir],
[$ompi_check_sicortex_libdir],
[ompi_check_sicortex_happy="yes"],
[ompi_check_sicortex_happy="no"])
],
[ompi_check_sicortex_happy="no"])
AS_IF([test "$ompi_check_sicortex_happy" = "yes"],
[$2],
[AS_IF([test ! -z "$with_sicortex" -a "$with_sicortex" != "no"],
[AC_MSG_ERROR([Sicortex (scdma) support requested but not found. Aborting])])
$3])
])
# MCA_btl_sicortex_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_btl_sicortex_CONFIG],[
OMPI_CHECK_SICORTEX([btl_sicortex],
[btl_sicortex_happy="yes"],
[btl_sicortex_happy="no"])
AS_IF([test "$btl_sicortex_happy" = "yes"],
[btl_sicortex_WRAPPER_EXTRA_LDFLAGS="$btl_sicortex_LDFLAGS"
btl_sicortex_WRAPPER_EXTRA_LIBS="$btl_sicortex_LIBS"
$1],
[$2])
# substitute in the things needed to build sicortex
AC_SUBST([btl_sicortex_CFLAGS])
AC_SUBST([btl_sicortex_CPPFLAGS])
AC_SUBST([btl_sicortex_LDFLAGS])
AC_SUBST([btl_sicortex_LIBS])
])dnl

15
ompi/mca/btl/sicortex/configure.params Исполняемый файл
Просмотреть файл

@ -0,0 +1,15 @@
# -*- shell-script -*-
#
# Copyright (c) 2008-2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_CONFIG_FILES="Makefile"