552c9ca5a0
WHAT: Open our low-level communication infrastructure by moving all necessary components (btl/rcache/allocator/mpool) down in OPAL All the components required for inter-process communications are currently deeply integrated in the OMPI layer. Several groups/institutions have express interest in having a more generic communication infrastructure, without all the OMPI layer dependencies. This communication layer should be made available at a different software level, available to all layers in the Open MPI software stack. As an example, our ORTE layer could replace the current OOB and instead use the BTL directly, gaining access to more reactive network interfaces than TCP. Similarly, external software libraries could take advantage of our highly optimized AM (active message) communication layer for their own purpose. UTK with support from Sandia, developped a version of Open MPI where the entire communication infrastucture has been moved down to OPAL (btl/rcache/allocator/mpool). Most of the moved components have been updated to match the new schema, with few exceptions (mainly BTLs where I have no way of compiling/testing them). Thus, the completion of this RFC is tied to being able to completing this move for all BTLs. For this we need help from the rest of the Open MPI community, especially those supporting some of the BTLs. A non-exhaustive list of BTLs that qualify here is: mx, portals4, scif, udapl, ugni, usnic. This commit was SVN r32317.
499 строки
19 KiB
C
499 строки
19 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2014 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved.
|
|
* Copyright (c) 2011-2012 Los Alamos National Security, LLC.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#ifndef OMPI_PML_BFO_SEND_REQUEST_H
|
|
#define OMPI_PML_BFO_SEND_REQUEST_H
|
|
|
|
#include "opal/mca/btl/btl.h"
|
|
#include "opal/mca/mpool/base/base.h"
|
|
#include "opal/datatype/opal_convertor.h"
|
|
#include "ompi/mca/pml/base/pml_base_sendreq.h"
|
|
#include "pml_bfo_comm.h"
|
|
#include "pml_bfo_hdr.h"
|
|
#include "pml_bfo_rdma.h"
|
|
#include "pml_bfo_rdmafrag.h"
|
|
#include "ompi/mca/bml/bml.h"
|
|
|
|
BEGIN_C_DECLS
|
|
|
|
typedef enum {
|
|
MCA_PML_BFO_SEND_PENDING_NONE,
|
|
MCA_PML_BFO_SEND_PENDING_SCHEDULE,
|
|
MCA_PML_BFO_SEND_PENDING_START
|
|
} mca_pml_bfo_send_pending_t;
|
|
|
|
struct mca_pml_bfo_send_request_t {
|
|
mca_pml_base_send_request_t req_send;
|
|
mca_bml_base_endpoint_t* req_endpoint;
|
|
opal_ptr_t req_recv;
|
|
#if PML_BFO
|
|
int32_t req_events; /* number of outstanding events on request */
|
|
int32_t req_restartseq; /* sequence number of restarted request */
|
|
int32_t req_restart; /* state of restarted request */
|
|
int32_t req_error; /* non-zero when error has occurred on request */
|
|
#endif /* PML_BFO */
|
|
int32_t req_state;
|
|
int32_t req_lock;
|
|
bool req_throttle_sends;
|
|
size_t req_pipeline_depth;
|
|
size_t req_bytes_delivered;
|
|
uint32_t req_rdma_cnt;
|
|
mca_pml_bfo_send_pending_t req_pending;
|
|
opal_mutex_t req_send_range_lock;
|
|
opal_list_t req_send_ranges;
|
|
mca_pml_bfo_com_btl_t req_rdma[1];
|
|
};
|
|
typedef struct mca_pml_bfo_send_request_t mca_pml_bfo_send_request_t;
|
|
|
|
OBJ_CLASS_DECLARATION(mca_pml_bfo_send_request_t);
|
|
|
|
struct mca_pml_bfo_send_range_t {
|
|
ompi_free_list_item_t base;
|
|
uint64_t range_send_offset;
|
|
uint64_t range_send_length;
|
|
int range_btl_idx;
|
|
int range_btl_cnt;
|
|
mca_pml_bfo_com_btl_t range_btls[1];
|
|
};
|
|
typedef struct mca_pml_bfo_send_range_t mca_pml_bfo_send_range_t;
|
|
OBJ_CLASS_DECLARATION(mca_pml_bfo_send_range_t);
|
|
|
|
static inline bool lock_send_request(mca_pml_bfo_send_request_t *sendreq)
|
|
{
|
|
return OPAL_THREAD_ADD32(&sendreq->req_lock, 1) == 1;
|
|
}
|
|
|
|
static inline bool unlock_send_request(mca_pml_bfo_send_request_t *sendreq)
|
|
{
|
|
return OPAL_THREAD_ADD32(&sendreq->req_lock, -1) == 0;
|
|
}
|
|
|
|
static inline void
|
|
add_request_to_send_pending(mca_pml_bfo_send_request_t* sendreq,
|
|
const mca_pml_bfo_send_pending_t type,
|
|
const bool append)
|
|
{
|
|
opal_list_item_t *item = (opal_list_item_t*)sendreq;
|
|
|
|
OPAL_THREAD_LOCK(&mca_pml_bfo.lock);
|
|
sendreq->req_pending = type;
|
|
if(append)
|
|
opal_list_append(&mca_pml_bfo.send_pending, item);
|
|
else
|
|
opal_list_prepend(&mca_pml_bfo.send_pending, item);
|
|
|
|
OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock);
|
|
}
|
|
|
|
static inline mca_pml_bfo_send_request_t*
|
|
get_request_from_send_pending(mca_pml_bfo_send_pending_t *type)
|
|
{
|
|
mca_pml_bfo_send_request_t *sendreq;
|
|
|
|
OPAL_THREAD_LOCK(&mca_pml_bfo.lock);
|
|
sendreq = (mca_pml_bfo_send_request_t*)
|
|
opal_list_remove_first(&mca_pml_bfo.send_pending);
|
|
if(sendreq) {
|
|
*type = sendreq->req_pending;
|
|
sendreq->req_pending = MCA_PML_BFO_SEND_PENDING_NONE;
|
|
}
|
|
OPAL_THREAD_UNLOCK(&mca_pml_bfo.lock);
|
|
|
|
return sendreq;
|
|
}
|
|
|
|
#define MCA_PML_BFO_SEND_REQUEST_ALLOC( comm, \
|
|
dst, \
|
|
sendreq) \
|
|
{ \
|
|
ompi_proc_t *proc = ompi_comm_peer_lookup( comm, dst ); \
|
|
ompi_free_list_item_t* item; \
|
|
\
|
|
sendreq = NULL; \
|
|
if( OPAL_LIKELY(NULL != proc) ) { \
|
|
OMPI_FREE_LIST_WAIT_MT(&mca_pml_base_send_requests, item); \
|
|
sendreq = (mca_pml_bfo_send_request_t*)item; \
|
|
sendreq->req_send.req_base.req_proc = proc; \
|
|
} \
|
|
}
|
|
|
|
|
|
#define MCA_PML_BFO_SEND_REQUEST_INIT( sendreq, \
|
|
buf, \
|
|
count, \
|
|
datatype, \
|
|
dst, \
|
|
tag, \
|
|
comm, \
|
|
sendmode, \
|
|
persistent) \
|
|
{ \
|
|
MCA_PML_BASE_SEND_REQUEST_INIT(&sendreq->req_send, \
|
|
buf, \
|
|
count, \
|
|
datatype, \
|
|
dst, \
|
|
tag, \
|
|
comm, \
|
|
sendmode, \
|
|
persistent, \
|
|
0); /* convertor_flags */ \
|
|
(sendreq)->req_recv.pval = NULL; \
|
|
}
|
|
|
|
|
|
static inline void mca_pml_bfo_free_rdma_resources(mca_pml_bfo_send_request_t* sendreq)
|
|
{
|
|
size_t r;
|
|
|
|
/* return mpool resources */
|
|
for(r = 0; r < sendreq->req_rdma_cnt; r++) {
|
|
mca_mpool_base_registration_t* reg = sendreq->req_rdma[r].btl_reg;
|
|
if( NULL != reg && reg->mpool != NULL ) {
|
|
reg->mpool->mpool_deregister(reg->mpool, reg);
|
|
}
|
|
}
|
|
sendreq->req_rdma_cnt = 0;
|
|
}
|
|
|
|
|
|
/**
|
|
* Start a send request.
|
|
*/
|
|
|
|
#define MCA_PML_BFO_SEND_REQUEST_START(sendreq, rc) \
|
|
do { \
|
|
rc = mca_pml_bfo_send_request_start(sendreq); \
|
|
} while (0)
|
|
|
|
|
|
/*
|
|
* Mark a send request as completed at the MPI level.
|
|
*/
|
|
|
|
#define MCA_PML_BFO_SEND_REQUEST_MPI_COMPLETE(sendreq, with_signal) \
|
|
do { \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status.MPI_SOURCE = \
|
|
(sendreq)->req_send.req_base.req_comm->c_my_rank; \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status.MPI_TAG = \
|
|
(sendreq)->req_send.req_base.req_tag; \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS; \
|
|
(sendreq)->req_send.req_base.req_ompi.req_status._ucount = \
|
|
(sendreq)->req_send.req_bytes_packed; \
|
|
ompi_request_complete( &((sendreq)->req_send.req_base.req_ompi), (with_signal) ); \
|
|
\
|
|
PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_COMPLETE, \
|
|
&(sendreq->req_send.req_base), PERUSE_SEND); \
|
|
} while(0)
|
|
|
|
/*
|
|
* Release resources associated with a request
|
|
*/
|
|
|
|
#define MCA_PML_BFO_SEND_REQUEST_RETURN(sendreq) \
|
|
do { \
|
|
/* Let the base handle the reference counts */ \
|
|
MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send)); \
|
|
OMPI_FREE_LIST_RETURN_MT( &mca_pml_base_send_requests, \
|
|
(ompi_free_list_item_t*)sendreq); \
|
|
} while(0)
|
|
|
|
|
|
/*
|
|
* The PML has completed a send request. Note that this request
|
|
* may have been orphaned by the user or have already completed
|
|
* at the MPI level.
|
|
* This function will never be called directly from the upper level, as it
|
|
* should only be an internal call to the PML.
|
|
*
|
|
*/
|
|
static inline void
|
|
send_request_pml_complete(mca_pml_bfo_send_request_t *sendreq)
|
|
{
|
|
assert(false == sendreq->req_send.req_base.req_pml_complete);
|
|
|
|
if(sendreq->req_send.req_bytes_packed > 0) {
|
|
PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_END,
|
|
&(sendreq->req_send.req_base), PERUSE_SEND);
|
|
}
|
|
|
|
/* return mpool resources */
|
|
mca_pml_bfo_free_rdma_resources(sendreq);
|
|
|
|
if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED &&
|
|
sendreq->req_send.req_addr != sendreq->req_send.req_base.req_addr) {
|
|
mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq);
|
|
}
|
|
|
|
OPAL_THREAD_LOCK(&ompi_request_lock);
|
|
if(false == sendreq->req_send.req_base.req_ompi.req_complete) {
|
|
/* Should only be called for long messages (maybe synchronous) */
|
|
MCA_PML_BFO_SEND_REQUEST_MPI_COMPLETE(sendreq, true);
|
|
} else {
|
|
if( MPI_SUCCESS != sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR ) {
|
|
ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST);
|
|
}
|
|
}
|
|
sendreq->req_send.req_base.req_pml_complete = true;
|
|
#if PML_BFO
|
|
sendreq->req_send.req_base.req_sequence -= 100;
|
|
#endif /* PML_BFO */
|
|
|
|
if(sendreq->req_send.req_base.req_free_called) {
|
|
MCA_PML_BFO_SEND_REQUEST_RETURN(sendreq);
|
|
}
|
|
OPAL_THREAD_UNLOCK(&ompi_request_lock);
|
|
}
|
|
|
|
/* returns true if request was completed on PML level */
|
|
static inline bool
|
|
send_request_pml_complete_check(mca_pml_bfo_send_request_t *sendreq)
|
|
{
|
|
#if OPAL_ENABLE_MULTI_THREADS
|
|
opal_atomic_rmb();
|
|
#endif
|
|
/* if no more events are expected for the request and the whole message is
|
|
* already sent and send fragment scheduling isn't running in another
|
|
* thread then complete the request on PML level. From now on, if user
|
|
* called free on this request, the request structure can be reused for
|
|
* another request or if the request is persistent it can be restarted */
|
|
if(sendreq->req_state == 0 &&
|
|
sendreq->req_bytes_delivered >= sendreq->req_send.req_bytes_packed
|
|
&& lock_send_request(sendreq)) {
|
|
send_request_pml_complete(sendreq);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Schedule additional fragments
|
|
*/
|
|
int
|
|
mca_pml_bfo_send_request_schedule_once(mca_pml_bfo_send_request_t*);
|
|
|
|
static inline int
|
|
mca_pml_bfo_send_request_schedule_exclusive(mca_pml_bfo_send_request_t* sendreq)
|
|
{
|
|
int rc;
|
|
do {
|
|
rc = mca_pml_bfo_send_request_schedule_once(sendreq);
|
|
if(rc == OMPI_ERR_OUT_OF_RESOURCE)
|
|
break;
|
|
} while(!unlock_send_request(sendreq));
|
|
|
|
if(OMPI_SUCCESS == rc)
|
|
send_request_pml_complete_check(sendreq);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static inline void
|
|
mca_pml_bfo_send_request_schedule(mca_pml_bfo_send_request_t* sendreq)
|
|
{
|
|
/*
|
|
* Only allow one thread in this routine for a given request.
|
|
* However, we cannot block callers on a mutex, so simply keep track
|
|
* of the number of times the routine has been called and run through
|
|
* the scheduling logic once for every call.
|
|
*/
|
|
|
|
if(!lock_send_request(sendreq))
|
|
return;
|
|
|
|
mca_pml_bfo_send_request_schedule_exclusive(sendreq);
|
|
}
|
|
|
|
#if OPAL_CUDA_SUPPORT
|
|
int mca_pml_bfo_send_request_start_cuda(
|
|
mca_pml_bfo_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
#endif /* OPAL_CUDA_SUPPORT */
|
|
|
|
/**
|
|
* Start the specified request
|
|
*/
|
|
|
|
int mca_pml_bfo_send_request_start_buffered(
|
|
mca_pml_bfo_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_bfo_send_request_start_copy(
|
|
mca_pml_bfo_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_bfo_send_request_start_prepare(
|
|
mca_pml_bfo_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_bfo_send_request_start_rdma(
|
|
mca_pml_bfo_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size);
|
|
|
|
int mca_pml_bfo_send_request_start_rndv(
|
|
mca_pml_bfo_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl,
|
|
size_t size,
|
|
int flags);
|
|
|
|
static inline int
|
|
mca_pml_bfo_send_request_start_btl( mca_pml_bfo_send_request_t* sendreq,
|
|
mca_bml_base_btl_t* bml_btl )
|
|
{
|
|
size_t size = sendreq->req_send.req_bytes_packed;
|
|
mca_btl_base_module_t* btl = bml_btl->btl;
|
|
size_t eager_limit = btl->btl_eager_limit - sizeof(mca_pml_bfo_hdr_t);
|
|
int rc;
|
|
|
|
if( OPAL_LIKELY(size <= eager_limit) ) {
|
|
switch(sendreq->req_send.req_send_mode) {
|
|
case MCA_PML_BASE_SEND_SYNCHRONOUS:
|
|
rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, 0);
|
|
break;
|
|
case MCA_PML_BASE_SEND_BUFFERED:
|
|
rc = mca_pml_bfo_send_request_start_copy(sendreq, bml_btl, size);
|
|
break;
|
|
case MCA_PML_BASE_SEND_COMPLETE:
|
|
rc = mca_pml_bfo_send_request_start_prepare(sendreq, bml_btl, size);
|
|
break;
|
|
default:
|
|
if (size != 0 && bml_btl->btl_flags & MCA_BTL_FLAGS_SEND_INPLACE) {
|
|
rc = mca_pml_bfo_send_request_start_prepare(sendreq, bml_btl, size);
|
|
} else {
|
|
rc = mca_pml_bfo_send_request_start_copy(sendreq, bml_btl, size);
|
|
}
|
|
break;
|
|
}
|
|
} else {
|
|
size = eager_limit;
|
|
if(OPAL_UNLIKELY(btl->btl_rndv_eager_limit < eager_limit))
|
|
size = btl->btl_rndv_eager_limit;
|
|
if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
|
|
rc = mca_pml_bfo_send_request_start_buffered(sendreq, bml_btl, size);
|
|
} else if
|
|
(opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) {
|
|
unsigned char *base;
|
|
opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base );
|
|
|
|
if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_bfo_rdma_btls(
|
|
sendreq->req_endpoint,
|
|
base,
|
|
sendreq->req_send.req_bytes_packed,
|
|
sendreq->req_rdma))) {
|
|
rc = mca_pml_bfo_send_request_start_rdma(sendreq, bml_btl,
|
|
sendreq->req_send.req_bytes_packed);
|
|
if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
|
|
mca_pml_bfo_free_rdma_resources(sendreq);
|
|
}
|
|
} else {
|
|
rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size,
|
|
MCA_PML_BFO_HDR_FLAGS_CONTIG);
|
|
}
|
|
} else {
|
|
#if OPAL_CUDA_SUPPORT
|
|
if (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) {
|
|
return mca_pml_bfo_send_request_start_cuda(sendreq, bml_btl, size);
|
|
}
|
|
#endif /* OPAL_CUDA_SUPPORT */
|
|
rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, 0);
|
|
}
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static inline int
|
|
mca_pml_bfo_send_request_start( mca_pml_bfo_send_request_t* sendreq )
|
|
{
|
|
mca_pml_bfo_comm_t* comm = sendreq->req_send.req_base.req_comm->c_pml_comm;
|
|
mca_bml_base_endpoint_t* endpoint = (mca_bml_base_endpoint_t*)
|
|
sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
|
|
size_t i;
|
|
|
|
if( OPAL_UNLIKELY(endpoint == NULL) ) {
|
|
return OMPI_ERR_UNREACH;
|
|
}
|
|
|
|
sendreq->req_endpoint = endpoint;
|
|
sendreq->req_state = 0;
|
|
sendreq->req_lock = 0;
|
|
sendreq->req_pipeline_depth = 0;
|
|
sendreq->req_bytes_delivered = 0;
|
|
sendreq->req_pending = MCA_PML_BFO_SEND_PENDING_NONE;
|
|
sendreq->req_send.req_base.req_sequence = OPAL_THREAD_ADD32(
|
|
&comm->procs[sendreq->req_send.req_base.req_peer].send_sequence,1);
|
|
#if PML_BFO
|
|
sendreq->req_restartseq = 0; /* counts up restarts */
|
|
sendreq->req_restart = 0; /* reset in case we restart again */
|
|
sendreq->req_error = 0; /* clear error state */
|
|
sendreq->req_events = 0; /* clear events, probably 0 anyways */
|
|
#endif /* PML_BFO */
|
|
|
|
MCA_PML_BASE_SEND_START( &sendreq->req_send.req_base );
|
|
|
|
for(i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) {
|
|
mca_bml_base_btl_t* bml_btl;
|
|
int rc;
|
|
|
|
/* select a btl */
|
|
bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
|
|
rc = mca_pml_bfo_send_request_start_btl(sendreq, bml_btl);
|
|
if( OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != rc) )
|
|
return rc;
|
|
}
|
|
add_request_to_send_pending(sendreq, MCA_PML_BFO_SEND_PENDING_START, true);
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
/**
|
|
* Initiate a put scheduled by the receiver.
|
|
*/
|
|
|
|
void mca_pml_bfo_send_request_put( mca_pml_bfo_send_request_t* sendreq,
|
|
mca_btl_base_module_t* btl,
|
|
mca_pml_bfo_rdma_hdr_t* hdr );
|
|
|
|
int mca_pml_bfo_send_request_put_frag(mca_pml_bfo_rdma_frag_t* frag);
|
|
|
|
/* This function tries to continue sendreq that was stuck because of resource
|
|
* unavailability. A sendreq may be added to send_pending list if there is no
|
|
* resource to send initial packet or there is not resource to schedule data
|
|
* for sending. The reason the sendreq was added to the list is stored inside
|
|
* sendreq struct and appropriate operation is retried when resource became
|
|
* available. bml_btl passed to the function doesn't represents sendreq
|
|
* destination, it represents BTL on which resource was freed, so only this BTL
|
|
* should be considered for sending packets */
|
|
void mca_pml_bfo_send_request_process_pending(mca_bml_base_btl_t *bml_btl);
|
|
|
|
void mca_pml_bfo_send_request_copy_in_out(mca_pml_bfo_send_request_t *sendreq,
|
|
uint64_t send_offset, uint64_t send_length);
|
|
|
|
END_C_DECLS
|
|
|
|
#endif /* OMPI_PML_BFO_SEND_REQUEST_H */
|