2005-06-15 23:10:26 +04:00
|
|
|
/*
|
2007-03-17 02:11:45 +03:00
|
|
|
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
2005-11-05 22:57:48 +03:00
|
|
|
* University Research and Technology
|
|
|
|
* Corporation. All rights reserved.
|
2006-08-24 20:38:08 +04:00
|
|
|
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
2005-11-05 22:57:48 +03:00
|
|
|
* of Tennessee Research Foundation. All rights
|
|
|
|
* reserved.
|
2005-06-15 23:10:26 +04:00
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
|
|
* University of Stuttgart. All rights reserved.
|
|
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
|
|
* All rights reserved.
|
|
|
|
* $COPYRIGHT$
|
|
|
|
*
|
|
|
|
* Additional copyrights may follow
|
|
|
|
*
|
|
|
|
* $HEADER$
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "ompi_config.h"
|
|
|
|
|
|
|
|
#include <string.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <errno.h>
|
|
|
|
|
2005-07-04 02:45:48 +04:00
|
|
|
#include "opal/threads/mutex.h"
|
2005-09-13 00:22:59 +04:00
|
|
|
#include "ompi/datatype/convertor.h"
|
2005-12-10 02:27:55 +03:00
|
|
|
#include "ompi/datatype/datatype.h"
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "opal/sys/atomic.h"
|
2005-07-04 03:31:27 +04:00
|
|
|
#include "opal/util/output.h"
|
2005-07-04 05:36:20 +04:00
|
|
|
#include "opal/util/if.h"
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "orte/util/proc_info.h"
|
2005-07-04 06:16:57 +04:00
|
|
|
#include "opal/util/printf.h"
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "orte/util/sys_info.h"
|
2005-09-13 00:22:59 +04:00
|
|
|
#include "ompi/class/ompi_fifo.h"
|
|
|
|
#include "ompi/class/ompi_free_list.h"
|
|
|
|
#include "ompi/mca/pml/pml.h"
|
|
|
|
#include "ompi/mca/btl/btl.h"
|
2006-02-12 04:33:29 +03:00
|
|
|
#include "ompi/mca/mpool/base/base.h"
|
2005-06-30 09:50:55 +04:00
|
|
|
#include "btl_self.h"
|
|
|
|
#include "btl_self_frag.h"
|
2005-09-13 00:22:59 +04:00
|
|
|
#include "ompi/proc/proc.h"
|
2005-06-30 09:50:55 +04:00
|
|
|
|
|
|
|
mca_btl_base_module_t mca_btl_self = {
|
|
|
|
&mca_btl_self_component.super,
|
|
|
|
0, /* btl_eager_limit */
|
|
|
|
0, /* btl_min_send_size */
|
|
|
|
0, /* btl_max_send_size */
|
|
|
|
0, /* btl_min_rdma_size */
|
|
|
|
0, /* btl_max_rdma_size */
|
|
|
|
0, /* btl_exclusivity */
|
|
|
|
0, /* btl_latency */
|
|
|
|
0, /* btl_bandwidth */
|
|
|
|
0, /* btl flags */
|
|
|
|
mca_btl_self_add_procs,
|
|
|
|
mca_btl_self_del_procs,
|
|
|
|
mca_btl_self_register,
|
|
|
|
mca_btl_self_finalize,
|
|
|
|
mca_btl_self_alloc,
|
|
|
|
mca_btl_self_free,
|
|
|
|
mca_btl_self_prepare_src,
|
|
|
|
mca_btl_self_prepare_dst,
|
|
|
|
mca_btl_self_send,
|
|
|
|
mca_btl_self_rdma, /* put */
|
2006-10-27 03:11:26 +04:00
|
|
|
mca_btl_self_rdma, /* get */
|
2006-08-18 02:02:01 +04:00
|
|
|
mca_btl_base_dump,
|
|
|
|
NULL, /* mpool */
|
2007-03-17 02:11:45 +03:00
|
|
|
NULL, /* register error cb */
|
|
|
|
mca_btl_self_ft_event
|
2005-06-15 23:10:26 +04:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2006-10-27 03:11:26 +04:00
|
|
|
int mca_btl_self_add_procs( struct mca_btl_base_module_t* btl,
|
|
|
|
size_t nprocs,
|
|
|
|
struct ompi_proc_t **procs,
|
|
|
|
struct mca_btl_base_endpoint_t **peers,
|
|
|
|
ompi_bitmap_t* reachability )
|
2005-06-15 23:10:26 +04:00
|
|
|
{
|
2006-11-08 20:02:46 +03:00
|
|
|
int i;
|
2006-10-27 03:11:26 +04:00
|
|
|
|
2006-11-08 20:02:46 +03:00
|
|
|
for( i = 0; i < (int)nprocs; i++ ) {
|
2006-10-27 03:11:26 +04:00
|
|
|
if( procs[i] == ompi_proc_local_proc ) {
|
|
|
|
ompi_bitmap_set_bit( reachability, i );
|
|
|
|
break; /* there will always be only one ... */
|
2005-06-20 20:38:19 +04:00
|
|
|
}
|
2005-06-15 23:10:26 +04:00
|
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2006-10-27 03:11:26 +04:00
|
|
|
int mca_btl_self_del_procs( struct mca_btl_base_module_t* btl,
|
|
|
|
size_t nprocs,
|
|
|
|
struct ompi_proc_t **procs,
|
|
|
|
struct mca_btl_base_endpoint_t **peers )
|
2005-06-15 23:10:26 +04:00
|
|
|
{
|
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
2005-06-30 09:50:55 +04:00
|
|
|
* MCA->BTL Clean up any resources held by BTL module
|
2005-06-15 23:10:26 +04:00
|
|
|
* before the module is unloaded.
|
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* @param btl (IN) BTL module.
|
2005-06-15 23:10:26 +04:00
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* Prior to unloading a BTL module, the MCA framework will call
|
|
|
|
* the BTL finalize method of the module. Any resources held by
|
|
|
|
* the BTL should be released and if required the memory corresponding
|
|
|
|
* to the BTL module freed.
|
2005-06-15 23:10:26 +04:00
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2005-06-30 09:50:55 +04:00
|
|
|
int mca_btl_self_finalize(struct mca_btl_base_module_t* btl)
|
2005-06-15 23:10:26 +04:00
|
|
|
{
|
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Register a callback function that is called on receipt
|
|
|
|
* of a fragment.
|
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* @param btl (IN) BTL module
|
2005-06-15 23:10:26 +04:00
|
|
|
* @return Status indicating if cleanup was successful
|
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* When the process list changes, the PML notifies the BTL of the
|
2005-06-15 23:10:26 +04:00
|
|
|
* change, to provide the opportunity to cleanup or release any
|
|
|
|
* resources associated with the peer.
|
|
|
|
*/
|
|
|
|
|
2006-10-27 03:11:26 +04:00
|
|
|
int mca_btl_self_register( struct mca_btl_base_module_t* btl,
|
|
|
|
mca_btl_base_tag_t tag,
|
|
|
|
mca_btl_base_module_recv_cb_fn_t cbfunc,
|
|
|
|
void* cbdata )
|
2005-06-15 23:10:26 +04:00
|
|
|
{
|
2005-06-30 09:50:55 +04:00
|
|
|
mca_btl_self_component.self_reg[tag].cbfunc = cbfunc;
|
|
|
|
mca_btl_self_component.self_reg[tag].cbdata = cbdata;
|
2005-06-15 23:10:26 +04:00
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Allocate a segment.
|
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* @param btl (IN) BTL module
|
2005-06-15 23:10:26 +04:00
|
|
|
* @param size (IN) Request segment size.
|
|
|
|
*/
|
2006-10-27 03:11:26 +04:00
|
|
|
mca_btl_base_descriptor_t* mca_btl_self_alloc( struct mca_btl_base_module_t* btl,
|
|
|
|
size_t size )
|
2005-06-15 23:10:26 +04:00
|
|
|
{
|
2005-06-30 09:50:55 +04:00
|
|
|
mca_btl_self_frag_t* frag;
|
2005-06-15 23:10:26 +04:00
|
|
|
int rc;
|
2005-06-30 09:50:55 +04:00
|
|
|
if(size <= mca_btl_self.btl_eager_limit) {
|
|
|
|
MCA_BTL_SELF_FRAG_ALLOC_EAGER(frag,rc);
|
2006-02-22 20:03:04 +03:00
|
|
|
frag->segment.seg_len = size;
|
2006-02-22 20:37:59 +03:00
|
|
|
} else if (size <= btl->btl_max_send_size) {
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_ALLOC_SEND(frag,rc);
|
2006-02-22 20:37:59 +03:00
|
|
|
frag->segment.seg_len = size;
|
|
|
|
} else {
|
|
|
|
return NULL;
|
2005-06-15 23:10:26 +04:00
|
|
|
}
|
2006-02-22 20:37:59 +03:00
|
|
|
|
2006-10-27 03:11:26 +04:00
|
|
|
frag->base.des_flags = 0;
|
|
|
|
frag->base.des_src = &(frag->segment);
|
|
|
|
frag->base.des_src_cnt = 1;
|
2005-06-30 09:50:55 +04:00
|
|
|
return (mca_btl_base_descriptor_t*)frag;
|
2005-06-15 23:10:26 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2005-06-30 09:50:55 +04:00
|
|
|
* Return a segment allocated by this BTL.
|
2005-06-15 23:10:26 +04:00
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* @param btl (IN) BTL module
|
2005-06-15 23:10:26 +04:00
|
|
|
* @param segment (IN) Allocated segment.
|
|
|
|
*/
|
2006-10-27 03:11:26 +04:00
|
|
|
int mca_btl_self_free( struct mca_btl_base_module_t* btl,
|
|
|
|
mca_btl_base_descriptor_t* des )
|
2005-06-15 23:10:26 +04:00
|
|
|
{
|
2005-06-30 09:50:55 +04:00
|
|
|
mca_btl_self_frag_t* frag = (mca_btl_self_frag_t*)des;
|
2006-10-27 03:11:26 +04:00
|
|
|
|
|
|
|
frag->base.des_src = NULL;
|
|
|
|
frag->base.des_src_cnt = 0;
|
|
|
|
frag->base.des_dst = NULL;
|
|
|
|
frag->base.des_dst_cnt = 0;
|
|
|
|
|
2006-10-25 12:45:29 +04:00
|
|
|
if(frag->size == mca_btl_self.btl_eager_limit) {
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_RETURN_EAGER(frag);
|
2006-10-25 12:45:29 +04:00
|
|
|
} else if (frag->size == mca_btl_self.btl_max_send_size) {
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_RETURN_SEND(frag);
|
2005-06-15 23:10:26 +04:00
|
|
|
} else {
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_RETURN_RDMA(frag);
|
2005-06-15 23:10:26 +04:00
|
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Prepare data for send/put
|
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* @param btl (IN) BTL module
|
2005-06-15 23:10:26 +04:00
|
|
|
*/
|
2005-06-30 09:50:55 +04:00
|
|
|
struct mca_btl_base_descriptor_t* mca_btl_self_prepare_src(
|
|
|
|
struct mca_btl_base_module_t* btl,
|
|
|
|
struct mca_btl_base_endpoint_t* endpoint,
|
2005-06-25 01:12:38 +04:00
|
|
|
mca_mpool_base_registration_t* registration,
|
2005-06-15 23:10:26 +04:00
|
|
|
struct ompi_convertor_t* convertor,
|
|
|
|
size_t reserve,
|
|
|
|
size_t* size)
|
|
|
|
{
|
2005-06-30 09:50:55 +04:00
|
|
|
mca_btl_self_frag_t* frag;
|
2005-06-15 23:10:26 +04:00
|
|
|
struct iovec iov;
|
|
|
|
uint32_t iov_count = 1;
|
|
|
|
size_t max_data = *size;
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
/* non-contigous data */
|
2006-10-27 03:11:26 +04:00
|
|
|
if( ompi_convertor_need_buffers(convertor) ||
|
|
|
|
max_data < mca_btl_self.btl_max_send_size ||
|
|
|
|
reserve != 0 ) {
|
|
|
|
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_ALLOC_SEND(frag, rc);
|
2005-06-15 23:10:26 +04:00
|
|
|
if(NULL == frag) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(reserve + max_data > frag->size) {
|
|
|
|
max_data = frag->size - reserve;
|
|
|
|
}
|
|
|
|
iov.iov_len = max_data;
|
2006-08-24 20:38:08 +04:00
|
|
|
iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)(frag+1) + reserve);
|
2005-06-15 23:10:26 +04:00
|
|
|
|
2006-10-27 03:11:26 +04:00
|
|
|
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data );
|
2005-06-15 23:10:26 +04:00
|
|
|
if(rc < 0) {
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_RETURN_SEND(frag);
|
2005-06-15 23:10:26 +04:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
frag->base.des_flags = 0;
|
2007-01-05 01:07:37 +03:00
|
|
|
frag->segment.seg_addr.pval = frag+1;
|
2005-06-15 23:10:26 +04:00
|
|
|
frag->segment.seg_len = reserve + max_data;
|
|
|
|
*size = max_data;
|
|
|
|
} else {
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_ALLOC_RDMA(frag, rc);
|
2005-06-15 23:10:26 +04:00
|
|
|
if(NULL == frag) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
iov.iov_len = max_data;
|
|
|
|
iov.iov_base = NULL;
|
|
|
|
|
|
|
|
/* convertor should return offset into users buffer */
|
2006-10-27 03:11:26 +04:00
|
|
|
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data );
|
2005-06-15 23:10:26 +04:00
|
|
|
if(rc < 0) {
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_RETURN_RDMA(frag);
|
2005-06-15 23:10:26 +04:00
|
|
|
return NULL;
|
|
|
|
}
|
2007-01-05 01:07:37 +03:00
|
|
|
frag->segment.seg_addr.pval = iov.iov_base;
|
2005-12-06 02:36:33 +03:00
|
|
|
frag->segment.seg_len = max_data;
|
2005-06-20 20:38:19 +04:00
|
|
|
frag->base.des_flags = 0;
|
2005-06-15 23:10:26 +04:00
|
|
|
*size = max_data;
|
|
|
|
}
|
2006-10-27 03:11:26 +04:00
|
|
|
frag->base.des_src = &frag->segment;
|
|
|
|
frag->base.des_src_cnt = 1;
|
|
|
|
frag->segment.seg_key.key64 = (uint64_t)(intptr_t)convertor;
|
2005-06-15 23:10:26 +04:00
|
|
|
return &frag->base;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Prepare data for receive.
|
|
|
|
*/
|
2005-06-30 09:50:55 +04:00
|
|
|
struct mca_btl_base_descriptor_t* mca_btl_self_prepare_dst(
|
|
|
|
struct mca_btl_base_module_t* btl,
|
|
|
|
struct mca_btl_base_endpoint_t* endpoint,
|
2005-06-25 01:12:38 +04:00
|
|
|
mca_mpool_base_registration_t* registration,
|
2005-06-15 23:10:26 +04:00
|
|
|
struct ompi_convertor_t* convertor,
|
|
|
|
size_t reserve,
|
|
|
|
size_t* size)
|
|
|
|
{
|
2005-06-30 09:50:55 +04:00
|
|
|
mca_btl_self_frag_t* frag;
|
2005-06-15 23:10:26 +04:00
|
|
|
size_t max_data = *size;
|
2006-10-18 01:05:04 +04:00
|
|
|
ptrdiff_t lb;
|
2005-06-15 23:10:26 +04:00
|
|
|
int rc;
|
|
|
|
|
2005-06-30 09:50:55 +04:00
|
|
|
MCA_BTL_SELF_FRAG_ALLOC_RDMA(frag, rc);
|
2005-06-15 23:10:26 +04:00
|
|
|
if(NULL == frag) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* setup descriptor to point directly to user buffer */
|
2005-12-10 02:27:55 +03:00
|
|
|
ompi_ddt_type_lb( convertor->pDesc, &lb );
|
2007-01-05 01:07:37 +03:00
|
|
|
frag->segment.seg_addr.pval = (unsigned char*)convertor->pBaseBuf + lb + convertor->bConverted;
|
2005-06-15 23:10:26 +04:00
|
|
|
frag->segment.seg_len = reserve + max_data;
|
2006-10-27 03:11:26 +04:00
|
|
|
frag->segment.seg_key.key64 = (uint64_t)(intptr_t)convertor;
|
2005-06-20 20:38:19 +04:00
|
|
|
frag->base.des_dst = &frag->segment;
|
|
|
|
frag->base.des_dst_cnt = 1;
|
|
|
|
frag->base.des_flags = 0;
|
2005-06-15 23:10:26 +04:00
|
|
|
return &frag->base;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Initiate a send to the peer.
|
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* @param btl (IN) BTL module
|
|
|
|
* @param peer (IN) BTL peer addressing
|
2005-06-15 23:10:26 +04:00
|
|
|
*/
|
|
|
|
|
2006-10-27 03:11:26 +04:00
|
|
|
int mca_btl_self_send( struct mca_btl_base_module_t* btl,
|
|
|
|
struct mca_btl_base_endpoint_t* endpoint,
|
|
|
|
struct mca_btl_base_descriptor_t* des,
|
|
|
|
mca_btl_base_tag_t tag )
|
2005-06-15 23:10:26 +04:00
|
|
|
{
|
2006-06-20 18:11:09 +04:00
|
|
|
/**
|
|
|
|
* We have to set the dst before the call to the function and reset them
|
|
|
|
* after.
|
|
|
|
*/
|
|
|
|
des->des_dst = des->des_src;
|
|
|
|
des->des_dst_cnt = des->des_src_cnt;
|
2005-06-20 20:38:19 +04:00
|
|
|
/* upcall */
|
2006-08-24 20:38:08 +04:00
|
|
|
mca_btl_self_component.self_reg[tag].cbfunc( btl, tag, des, (void*)OMPI_SUCCESS );
|
2006-06-20 18:11:09 +04:00
|
|
|
des->des_dst = NULL;
|
|
|
|
des->des_dst_cnt = 0;
|
2005-06-20 20:38:19 +04:00
|
|
|
/* send completion */
|
2005-06-30 09:50:55 +04:00
|
|
|
des->des_cbfunc(btl,endpoint,des,OMPI_SUCCESS);
|
2005-06-20 20:38:19 +04:00
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Initiate a put to the peer.
|
|
|
|
*
|
2005-06-30 09:50:55 +04:00
|
|
|
* @param btl (IN) BTL module
|
|
|
|
* @param peer (IN) BTL peer addressing
|
2005-06-20 20:38:19 +04:00
|
|
|
*/
|
|
|
|
|
2006-10-27 03:11:26 +04:00
|
|
|
int mca_btl_self_rdma( struct mca_btl_base_module_t* btl,
|
|
|
|
struct mca_btl_base_endpoint_t* endpoint,
|
|
|
|
struct mca_btl_base_descriptor_t* des )
|
2005-06-20 20:38:19 +04:00
|
|
|
{
|
2005-06-30 09:50:55 +04:00
|
|
|
mca_btl_base_segment_t* src = des->des_src;
|
|
|
|
mca_btl_base_segment_t* dst = des->des_dst;
|
2005-06-20 20:38:19 +04:00
|
|
|
size_t src_cnt = des->des_src_cnt;
|
|
|
|
size_t dst_cnt = des->des_dst_cnt;
|
2007-01-05 01:07:37 +03:00
|
|
|
unsigned char* src_addr = (unsigned char*)src->seg_addr.pval;
|
2005-06-20 20:38:19 +04:00
|
|
|
size_t src_len = src->seg_len;
|
2007-01-14 23:34:41 +03:00
|
|
|
unsigned char* dst_addr = (unsigned char*)ompi_ptr_ltop(dst->seg_addr.lval);
|
2005-06-20 20:38:19 +04:00
|
|
|
size_t dst_len = dst->seg_len;
|
|
|
|
|
|
|
|
while(src_len && dst_len) {
|
|
|
|
|
|
|
|
if(src_len == dst_len) {
|
|
|
|
memcpy(dst_addr, src_addr, src_len);
|
|
|
|
|
|
|
|
/* advance src */
|
|
|
|
if(--src_cnt != 0) {
|
|
|
|
src++;
|
2007-01-05 01:07:37 +03:00
|
|
|
src_addr = (unsigned char*)src->seg_addr.pval;
|
2005-06-20 20:38:19 +04:00
|
|
|
src_len = src->seg_len;
|
|
|
|
} else {
|
|
|
|
src_len = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* advance dst */
|
|
|
|
if(--dst_cnt != 0) {
|
|
|
|
dst++;
|
2007-01-05 01:07:37 +03:00
|
|
|
dst_addr = (unsigned char*)dst->seg_addr.pval;
|
2005-06-20 20:38:19 +04:00
|
|
|
dst_len = dst->seg_len;
|
|
|
|
} else {
|
|
|
|
dst_len = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
} else {
|
|
|
|
size_t bytes = src_len < dst_len ? src_len : dst_len;
|
|
|
|
memcpy(dst_addr, src_addr, bytes);
|
|
|
|
|
|
|
|
/* advance src */
|
|
|
|
src_len -= bytes;
|
|
|
|
if(src_len == 0) {
|
|
|
|
if(--src_cnt != 0) {
|
|
|
|
src++;
|
2007-01-05 01:07:37 +03:00
|
|
|
src_addr = (unsigned char*)src->seg_addr.pval;
|
2005-06-20 20:38:19 +04:00
|
|
|
src_len = src->seg_len;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
src_addr += bytes;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* advance dst */
|
|
|
|
dst_len -= bytes;
|
|
|
|
if(dst_len == 0) {
|
|
|
|
if(--dst_cnt != 0) {
|
|
|
|
dst++;
|
2007-01-05 01:07:37 +03:00
|
|
|
dst_addr = (unsigned char*)src->seg_addr.pval;
|
2005-06-20 20:38:19 +04:00
|
|
|
dst_len = src->seg_len;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
dst_addr += bytes;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* rdma completion */
|
2005-06-30 09:50:55 +04:00
|
|
|
des->des_cbfunc(btl,endpoint,des,OMPI_SUCCESS);
|
2005-06-15 23:10:26 +04:00
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|
2007-03-17 02:11:45 +03:00
|
|
|
|
|
|
|
int mca_btl_self_ft_event(int state) {
|
|
|
|
if(OPAL_CRS_CHECKPOINT == state) {
|
|
|
|
;
|
|
|
|
}
|
|
|
|
else if(OPAL_CRS_CONTINUE == state) {
|
|
|
|
;
|
|
|
|
}
|
|
|
|
else if(OPAL_CRS_RESTART == state) {
|
|
|
|
;
|
|
|
|
}
|
|
|
|
else if(OPAL_CRS_TERM == state ) {
|
|
|
|
;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
;
|
|
|
|
}
|
|
|
|
|
|
|
|
return OMPI_SUCCESS;
|
|
|
|
}
|