512 строки
20 KiB
C
512 строки
20 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2014 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
|
|
* Copyright (c) 2011-2012 NVIDIA Corporation. All rights reserved.
|
|
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#ifndef OMPI_PML_OB1_SEND_REQUEST_H
|
|
#define OMPI_PML_OB1_SEND_REQUEST_H
|
|
|
|
#include "ompi/mca/btl/btl.h"
|
|
#include "ompi/mca/pml/base/pml_base_sendreq.h"
|
|
#include "ompi/mca/mpool/base/base.h"
|
|
#include "pml_ob1_comm.h"
|
|
#include "pml_ob1_hdr.h"
|
|
#include "pml_ob1_rdma.h"
|
|
#include "pml_ob1_rdmafrag.h"
|
|
#include "opal/datatype/opal_convertor.h"
|
|
#include "ompi/mca/bml/bml.h"
|
|
|
|
BEGIN_C_DECLS
|
|
|
|
typedef enum {
|
|
MCA_PML_OB1_SEND_PENDING_NONE,
|
|
MCA_PML_OB1_SEND_PENDING_SCHEDULE,
|
|
MCA_PML_OB1_SEND_PENDING_START
|
|
} mca_pml_ob1_send_pending_t;
|
|
|
|
struct mca_pml_ob1_send_request_t {
|
|
mca_pml_base_send_request_t req_send;
|
|
mca_bml_base_endpoint_t* req_endpoint;
|
|
ompi_ptr_t req_recv;
|
|
int32_t req_state;
|
|
int32_t req_lock;
|
|
bool req_throttle_sends;
|
|
size_t req_pipeline_depth;
|
|
size_t req_bytes_delivered;
|
|
uint32_t req_rdma_cnt;
|
|
mca_pml_ob1_send_pending_t req_pending;
|
|
opal_mutex_t req_send_range_lock;
|
|
opal_list_t req_send_ranges;
|
|
mca_btl_base_descriptor_t *src_des;
|
|
mca_pml_ob1_com_btl_t req_rdma[1];
|
|
};
|
|
typedef struct mca_pml_ob1_send_request_t mca_pml_ob1_send_request_t;
|
|
|
|
OBJ_CLASS_DECLARATION(mca_pml_ob1_send_request_t);
|
|
|
|
struct mca_pml_ob1_send_range_t {
|
|
ompi_free_list_item_t base;
|
|
uint64_t range_send_offset;
|
|
uint64_t range_send_length;
|
|
int range_btl_idx;
|
|
int range_btl_cnt;
|
|
mca_pml_ob1_com_btl_t range_btls[1];
|
|
};
|
|
typedef struct mca_pml_ob1_send_range_t mca_pml_ob1_send_range_t;
|
|
OBJ_CLASS_DECLARATION(mca_pml_ob1_send_range_t);
|
|
|
|
static inline bool lock_send_request(mca_pml_ob1_send_request_t *sendreq)
|
|
{
|
|
return OPAL_THREAD_ADD32(&sendreq->req_lock, 1) == 1;
|
|
}
|
|
|
|
static inline bool unlock_send_request(mca_pml_ob1_send_request_t *sendreq)
|
|
{
|
|
return OPAL_THREAD_ADD32(&sendreq->req_lock, -1) == 0;
|
|
}
|
|
|
|
static inline void
|
|
add_request_to_send_pending(mca_pml_ob1_send_request_t* sendreq,
|
|
const mca_pml_ob1_send_pending_t type,
|
|
const bool append)
|
|
{
|
|
opal_list_item_t *item = (opal_list_item_t*)sendreq;
|
|
|
|
OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
|
|
sendreq->req_pending = type;
|
|
if(append)
|
|
opal_list_append(&mca_pml_ob1.send_pending, item);
|
|
else
|
|
opal_list_prepend(&mca_pml_ob1.send_pending, item);
|
|
|
|
OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);
|
|
}
|
|
|
|
static inline mca_pml_ob1_send_request_t*
|
|
get_request_from_send_pending(mca_pml_ob1_send_pending_t *type)
|
|
{
|
|
mca_pml_ob1_send_request_t *sendreq;
|
|
|
|
OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
|
|
sendreq = (mca_pml_ob1_send_request_t*)
|
|
opal_list_remove_first(&mca_pml_ob1.send_pending);
|
|
if(sendreq) {
|
|
*type = sendreq->req_pending;
|
|
sendreq->req_pending = MCA_PML_OB1_SEND_PENDING_NONE;
|
|
}
|
|
OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);
|
|
|
|
return sendreq;
|
|
}
|
|
|
|
#define MCA_PML_OB1_SEND_REQUEST_ALLOC( comm, \
|
|
dst, \
|
|
sendreq) \
|
|
{ \
|
|
ompi_proc_t *proc = ompi_comm_peer_lookup( comm, dst ); \
|
|
ompi_free_list_item_t* item; \
|
|
\
|
|
if( OPAL_LIKELY(NULL != proc) ) { \
|
|
OMPI_FREE_LIST_WAIT_MT(&mca_pml_base_send_requests, item); \
|
|
sendreq = (mca_pml_ob1_send_request_t*)item; \
|
|
sendreq->req_send.req_base.req_proc = proc; \
|
|
sendreq->src_des = NULL; \
|
|
} \
|
|
}
|
|
|
|
|
|
#define MCA_PML_OB1_SEND_REQUEST_INIT( sendreq, \
|
|
buf, \
|
|
count, \
|
|
datatype, \
|
|
dst, \
|
|
tag, \
|
|
comm, \
|
|
sendmode, \
|
|
persistent) \
|
|
{ \
|
|
MCA_PML_BASE_SEND_REQUEST_INIT(&(sendreq)->req_send, \
|
|
buf, \
|
|
count, \
|
|
datatype, \
|
|
dst, \
|
|
tag, \
|
|
comm, \
|
|
sendmode, \
|
|
persistent, \
|
|
0); /* convertor_flags */ \
|
|
(sendreq)->req_recv.pval = NULL; \
|
|
}
|
|
|
|
#define MCA_PML_OB1_SEND_REQUEST_RESET(sendreq) \
|
|
if ((sendreq)->req_send.req_bytes_packed > 0) { \
|
|
size_t _position = 0; \
|
|
opal_convertor_set_position(&(sendreq)->req_send.req_base.req_convertor, \
|
|
&_position); \
|
|
assert( 0 == _position ); \
|
|
}
|
|
|
|
static inline void mca_pml_ob1_free_rdma_resources(mca_pml_ob1_send_request_t* sendreq)
|
|
{
|
|
size_t r;
|
|
|
|
/* return mpool resources */
|
|
for(r = 0; r < sendreq->req_rdma_cnt; r++) {
|
|
mca_mpool_base_registration_t* reg = sendreq->req_rdma[r].btl_reg;
|
|
if( NULL != reg && reg->mpool != NULL ) {
|
|
reg->mpool->mpool_deregister(reg->mpool, reg);
|
|
}
|
|
}
|
|
sendreq->req_rdma_cnt = 0;
|
|
}
|
|
|
|
|
|
/**
|
|
* Start a send request.
|
|
*/
|
|
|
|
#define MCA_PML_OB1_SEND_REQUEST_START(sendreq, rc) \
|
|
do { \
|
|
rc = mca_pml_ob1_send_request_start(sendreq); \
|
|
} while (0)
|
|
|
|
#define MCA_PML_OB1_SEND_REQUEST_START_W_SEQ(sendreq, endpoint, seq, rc) \
|
|
do { \
|
|
rc = mca_pml_ob1_send_request_start_seq (sendreq, endpoint, seq); \
|
|
} while (0)
|
|
|
|
|
|
/*
|
|
* Mark a send request as completed at the MPI level.
|
|
*/
|
|
|
|
#define MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, with_signal) \
|
|
do { \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status.MPI_SOURCE = \
|
|
(sendreq)->req_send.req_base.req_comm->c_my_rank; \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status.MPI_TAG = \
|
|
(sendreq)->req_send.req_base.req_tag; \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS; \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status._ucount = \
|
|
(sendreq)->req_send.req_bytes_packed; \
|
|
ompi_request_complete( &((sendreq)->req_send.req_base.req_ompi), (with_signal) ); \
|
|
\
|
|
PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_COMPLETE, \
|
|
&(sendreq->req_send.req_base), PERUSE_SEND); \
|
|
} while(0)
|
|
|
|
/*
|
|
* Release resources associated with a request
|
|
*/
|
|
|
|
#define MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq) \
|
|
do { \
|
|
/* Let the base handle the reference counts */ \
|
|
MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send)); \
|
|
OMPI_FREE_LIST_RETURN_MT( &mca_pml_base_send_requests, \
|
|
(ompi_free_list_item_t*)sendreq); \
|
|
} while(0)
|
|
|
|
|
|
/*
|
|
* The PML has completed a send request. Note that this request
|
|
* may have been orphaned by the user or have already completed
|
|
* at the MPI level.
|
|
* This function will never be called directly from the upper level, as it
|
|
* should only be an internal call to the PML.
|
|
*
|
|
*/
|
|
static inline void
|
|
send_request_pml_complete(mca_pml_ob1_send_request_t *sendreq)
|
|
{
|
|
assert(false == sendreq->req_send.req_base.req_pml_complete);
|
|
|
|
if(sendreq->req_send.req_bytes_packed > 0) {
|
|
PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_END,
|
|
&(sendreq->req_send.req_base), PERUSE_SEND);
|
|
}
|
|
|
|
/* return mpool resources */
|
|
mca_pml_ob1_free_rdma_resources(sendreq);
|
|
|
|
if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED &&
|
|
sendreq->req_send.req_addr != sendreq->req_send.req_base.req_addr) {
|
|
mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq);
|
|
}
|
|
|
|
OPAL_THREAD_LOCK(&ompi_request_lock);
|
|
if(false == sendreq->req_send.req_base.req_ompi.req_complete) {
|
|
/* Should only be called for long messages (maybe synchronous) */
|
|
MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, true);
|
|
} else {
|
|
if( MPI_SUCCESS != sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR ) {
|
|
ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST);
|
|
}
|
|
}
|
|
sendreq->req_send.req_base.req_pml_complete = true;
|
|
|
|
if(sendreq->req_send.req_base.req_free_called) {
|
|
MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq);
|
|
}
|
|
OPAL_THREAD_UNLOCK(&ompi_request_lock);
|
|
}
|
|
|
|
/* returns true if request was completed on PML level */
|
|
static inline bool
|
|
send_request_pml_complete_check(mca_pml_ob1_send_request_t *sendreq)
|
|
{
|
|
#if OPAL_ENABLE_MULTI_THREADS
|
|
opal_atomic_rmb();
|
|
#endif
|
|
/* if no more events are expected for the request and the whole message is
|
|
* already sent and send fragment scheduling isn't running in another
|
|
* thread then complete the request on PML level. From now on, if user
|
|
* called free on this request, the request structure can be reused for
|
|
* another request or if the request is persistent it can be restarted */
|
|
if(sendreq->req_state == 0 &&
|
|
sendreq->req_bytes_delivered >= sendreq->req_send.req_bytes_packed
|
|
&& lock_send_request(sendreq)) {
|
|
send_request_pml_complete(sendreq);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Schedule additional fragments
|
|
*/
|
|
int
|
|
mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t*);
|
|
|
|
static inline int
|
|
mca_pml_ob1_send_request_schedule_exclusive(mca_pml_ob1_send_request_t* sendreq)
|
|
{
|
|
int rc;
|
|
do {
|
|
rc = mca_pml_ob1_send_request_schedule_once(sendreq);
|
|
if(rc == OMPI_ERR_OUT_OF_RESOURCE)
|
|
break;
|
|
} while(!unlock_send_request(sendreq));
|
|
|
|
if(OMPI_SUCCESS == rc)
|
|
send_request_pml_complete_check(sendreq);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static inline void
|
|
mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq)
|
|
{
|
|
/*
|
|
* Only allow one thread in this routine for a given request.
|
|
* However, we cannot block callers on a mutex, so simply keep track
|
|
* of the number of times the routine has been called and run through
|
|
* the scheduling logic once for every call.
|
|
*/
|
|
|
|
if(!lock_send_request(sendreq))
|
|
return;
|
|
|
|
mca_pml_ob1_send_request_schedule_exclusive(sendreq);
|
|
}
|
|
|
|
#if OPAL_CUDA_SUPPORT
|
|
int mca_pml_ob1_send_request_start_cuda(
|
|
mca_pml_ob1_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
#endif /* OPAL_CUDA_SUPPORT */
|
|
|
|
/**
|
|
* Start the specified request
|
|
*/
|
|
|
|
int mca_pml_ob1_send_request_start_buffered(
|
|
mca_pml_ob1_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_ob1_send_request_start_copy(
|
|
mca_pml_ob1_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_ob1_send_request_start_prepare(
|
|
mca_pml_ob1_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_ob1_send_request_start_rdma(
|
|
mca_pml_ob1_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_ob1_send_request_start_rndv(
|
|
mca_pml_ob1_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size,
|
|
int flags);
|
|
|
|
static inline int
|
|
mca_pml_ob1_send_request_start_btl( mca_pml_ob1_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl )
|
|
{
|
|
size_t size = sendreq->req_send.req_bytes_packed;
|
|
mca_btl_base_module_t* btl = bml_btl->btl;
|
|
size_t eager_limit = btl->btl_eager_limit - sizeof(mca_pml_ob1_hdr_t);
|
|
int rc;
|
|
|
|
#if OPAL_CUDA_GDR_SUPPORT
|
|
if (btl->btl_cuda_eager_limit && (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA)) {
|
|
eager_limit = btl->btl_cuda_eager_limit - sizeof(mca_pml_ob1_hdr_t);
|
|
}
|
|
#endif /* OPAL_CUDA_GDR_SUPPORT */
|
|
|
|
if( OPAL_LIKELY(size <= eager_limit) ) {
|
|
switch(sendreq->req_send.req_send_mode) {
|
|
case MCA_PML_BASE_SEND_SYNCHRONOUS:
|
|
rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, 0);
|
|
break;
|
|
case MCA_PML_BASE_SEND_BUFFERED:
|
|
rc = mca_pml_ob1_send_request_start_copy(sendreq, bml_btl, size);
|
|
break;
|
|
case MCA_PML_BASE_SEND_COMPLETE:
|
|
rc = mca_pml_ob1_send_request_start_prepare(sendreq, bml_btl, size);
|
|
break;
|
|
default:
|
|
if (size != 0 && bml_btl->btl_flags & MCA_BTL_FLAGS_SEND_INPLACE) {
|
|
rc = mca_pml_ob1_send_request_start_prepare(sendreq, bml_btl, size);
|
|
} else {
|
|
rc = mca_pml_ob1_send_request_start_copy(sendreq, bml_btl, size);
|
|
}
|
|
break;
|
|
}
|
|
} else {
|
|
size = eager_limit;
|
|
if(OPAL_UNLIKELY(btl->btl_rndv_eager_limit < eager_limit))
|
|
size = btl->btl_rndv_eager_limit;
|
|
if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
|
|
rc = mca_pml_ob1_send_request_start_buffered(sendreq, bml_btl, size);
|
|
} else if
|
|
(opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) {
|
|
unsigned char *base;
|
|
opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base );
|
|
|
|
if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_ob1_rdma_btls(
|
|
sendreq->req_endpoint,
|
|
base,
|
|
sendreq->req_send.req_bytes_packed,
|
|
sendreq->req_rdma))) {
|
|
rc = mca_pml_ob1_send_request_start_rdma(sendreq, bml_btl,
|
|
sendreq->req_send.req_bytes_packed);
|
|
if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
|
|
mca_pml_ob1_free_rdma_resources(sendreq);
|
|
}
|
|
} else {
|
|
rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size,
|
|
MCA_PML_OB1_HDR_FLAGS_CONTIG);
|
|
}
|
|
} else {
|
|
#if OPAL_CUDA_SUPPORT
|
|
if (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) {
|
|
return mca_pml_ob1_send_request_start_cuda(sendreq, bml_btl, size);
|
|
}
|
|
#endif /* OPAL_CUDA_SUPPORT */
|
|
rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, 0);
|
|
}
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static inline int
|
|
mca_pml_ob1_send_request_start_seq (mca_pml_ob1_send_request_t* sendreq, mca_bml_base_endpoint_t* endpoint, int32_t seqn)
|
|
{
|
|
sendreq->req_endpoint = endpoint;
|
|
sendreq->req_state = 0;
|
|
sendreq->req_lock = 0;
|
|
sendreq->req_pipeline_depth = 0;
|
|
sendreq->req_bytes_delivered = 0;
|
|
sendreq->req_pending = MCA_PML_OB1_SEND_PENDING_NONE;
|
|
sendreq->req_send.req_base.req_sequence = seqn;
|
|
|
|
MCA_PML_BASE_SEND_START( &sendreq->req_send.req_base );
|
|
|
|
for(size_t i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) {
|
|
mca_bml_base_btl_t* bml_btl;
|
|
int rc;
|
|
|
|
/* select a btl */
|
|
bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
|
|
rc = mca_pml_ob1_send_request_start_btl(sendreq, bml_btl);
|
|
if( OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != rc) )
|
|
return rc;
|
|
}
|
|
add_request_to_send_pending(sendreq, MCA_PML_OB1_SEND_PENDING_START, true);
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static inline int
|
|
mca_pml_ob1_send_request_start( mca_pml_ob1_send_request_t* sendreq )
|
|
{
|
|
mca_bml_base_endpoint_t* endpoint = (mca_bml_base_endpoint_t*)
|
|
sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
|
|
mca_pml_ob1_comm_t* comm = sendreq->req_send.req_base.req_comm->c_pml_comm;
|
|
int32_t seqn;
|
|
|
|
if (OPAL_UNLIKELY(NULL == endpoint)) {
|
|
return OMPI_ERR_UNREACH;
|
|
}
|
|
|
|
seqn = OPAL_THREAD_ADD32(&comm->procs[sendreq->req_send.req_base.req_peer].send_sequence, 1);
|
|
|
|
return mca_pml_ob1_send_request_start_seq (sendreq, endpoint, seqn);
|
|
}
|
|
|
|
/**
|
|
* Initiate a put scheduled by the receiver.
|
|
*/
|
|
|
|
void mca_pml_ob1_send_request_put( mca_pml_ob1_send_request_t* sendreq,
|
|
mca_btl_base_module_t* btl,
|
|
mca_pml_ob1_rdma_hdr_t* hdr );
|
|
|
|
int mca_pml_ob1_send_request_put_frag(mca_pml_ob1_rdma_frag_t* frag);
|
|
|
|
/* This function tries to continue sendreq that was stuck because of resource
|
|
* unavailability. A sendreq may be added to send_pending list if there is no
|
|
* resource to send initial packet or there is not resource to schedule data
|
|
* for sending. The reason the sendreq was added to the list is stored inside
|
|
* sendreq struct and appropriate operation is retried when resource became
|
|
* available. bml_btl passed to the function doesn't represents sendreq
|
|
* destination, it represents BTL on which resource was freed, so only this BTL
|
|
* should be considered for sending packets */
|
|
void mca_pml_ob1_send_request_process_pending(mca_bml_base_btl_t *bml_btl);
|
|
|
|
void mca_pml_ob1_send_request_copy_in_out(mca_pml_ob1_send_request_t *sendreq,
|
|
uint64_t send_offset, uint64_t send_length);
|
|
|
|
END_C_DECLS
|
|
|
|
#endif /* OMPI_PML_OB1_SEND_REQUEST_H */
|