/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2011-2012 NVIDIA Corporation. All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef OMPI_PML_OB1_SEND_REQUEST_H #define OMPI_PML_OB1_SEND_REQUEST_H #include "opal/datatype/opal_convertor.h" #include "opal/mca/mpool/base/base.h" #include "ompi/mca/pml/base/pml_base_sendreq.h" #include "pml_ob1_comm.h" #include "pml_ob1_hdr.h" #include "pml_ob1_rdma.h" #include "pml_ob1_rdmafrag.h" #include "ompi/mca/bml/bml.h" BEGIN_C_DECLS typedef enum { MCA_PML_OB1_SEND_PENDING_NONE, MCA_PML_OB1_SEND_PENDING_SCHEDULE, MCA_PML_OB1_SEND_PENDING_START } mca_pml_ob1_send_pending_t; struct mca_pml_ob1_send_request_t { mca_pml_base_send_request_t req_send; mca_bml_base_endpoint_t* req_endpoint; opal_ptr_t req_recv; int32_t req_state; int32_t req_lock; bool req_throttle_sends; size_t req_pipeline_depth; size_t req_bytes_delivered; uint32_t req_rdma_cnt; mca_pml_ob1_send_pending_t req_pending; opal_mutex_t req_send_range_lock; opal_list_t req_send_ranges; mca_btl_base_descriptor_t *src_des; mca_pml_ob1_com_btl_t req_rdma[1]; }; typedef struct mca_pml_ob1_send_request_t mca_pml_ob1_send_request_t; OBJ_CLASS_DECLARATION(mca_pml_ob1_send_request_t); struct mca_pml_ob1_send_range_t { ompi_free_list_item_t base; uint64_t range_send_offset; uint64_t range_send_length; int range_btl_idx; int range_btl_cnt; mca_pml_ob1_com_btl_t range_btls[1]; }; typedef struct mca_pml_ob1_send_range_t mca_pml_ob1_send_range_t; OBJ_CLASS_DECLARATION(mca_pml_ob1_send_range_t); static inline bool lock_send_request(mca_pml_ob1_send_request_t *sendreq) { return OPAL_THREAD_ADD32(&sendreq->req_lock, 1) == 1; } static inline bool unlock_send_request(mca_pml_ob1_send_request_t *sendreq) { return OPAL_THREAD_ADD32(&sendreq->req_lock, -1) == 0; } static inline void add_request_to_send_pending(mca_pml_ob1_send_request_t* sendreq, const mca_pml_ob1_send_pending_t type, const bool append) { opal_list_item_t *item = (opal_list_item_t*)sendreq; OPAL_THREAD_LOCK(&mca_pml_ob1.lock); sendreq->req_pending = type; if(append) opal_list_append(&mca_pml_ob1.send_pending, item); else opal_list_prepend(&mca_pml_ob1.send_pending, item); OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock); } static inline mca_pml_ob1_send_request_t* get_request_from_send_pending(mca_pml_ob1_send_pending_t *type) { mca_pml_ob1_send_request_t *sendreq; OPAL_THREAD_LOCK(&mca_pml_ob1.lock); sendreq = (mca_pml_ob1_send_request_t*) opal_list_remove_first(&mca_pml_ob1.send_pending); if(sendreq) { *type = sendreq->req_pending; sendreq->req_pending = MCA_PML_OB1_SEND_PENDING_NONE; } OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock); return sendreq; } #define MCA_PML_OB1_SEND_REQUEST_ALLOC( comm, \ dst, \ sendreq) \ { \ ompi_proc_t *proc = ompi_comm_peer_lookup( comm, dst ); \ ompi_free_list_item_t* item; \ \ if( OPAL_LIKELY(NULL != proc) ) { \ OMPI_FREE_LIST_WAIT_MT(&mca_pml_base_send_requests, item); \ sendreq = (mca_pml_ob1_send_request_t*)item; \ sendreq->req_send.req_base.req_proc = proc; \ sendreq->src_des = NULL; \ } \ } #define MCA_PML_OB1_SEND_REQUEST_INIT( sendreq, \ buf, \ count, \ datatype, \ dst, \ tag, \ comm, \ sendmode, \ persistent) \ { \ MCA_PML_BASE_SEND_REQUEST_INIT(&(sendreq)->req_send, \ buf, \ count, \ datatype, \ dst, \ tag, \ comm, \ sendmode, \ persistent, \ 0); /* convertor_flags */ \ (sendreq)->req_recv.pval = NULL; \ } #define MCA_PML_OB1_SEND_REQUEST_RESET(sendreq) \ if ((sendreq)->req_send.req_bytes_packed > 0) { \ size_t _position = 0; \ opal_convertor_set_position(&(sendreq)->req_send.req_base.req_convertor, \ &_position); \ assert( 0 == _position ); \ } static inline void mca_pml_ob1_free_rdma_resources(mca_pml_ob1_send_request_t* sendreq) { size_t r; /* return mpool resources */ for(r = 0; r < sendreq->req_rdma_cnt; r++) { mca_mpool_base_registration_t* reg = sendreq->req_rdma[r].btl_reg; if( NULL != reg && reg->mpool != NULL ) { reg->mpool->mpool_deregister(reg->mpool, reg); } } sendreq->req_rdma_cnt = 0; } /** * Start a send request. */ #define MCA_PML_OB1_SEND_REQUEST_START(sendreq, rc) \ do { \ rc = mca_pml_ob1_send_request_start(sendreq); \ } while (0) #define MCA_PML_OB1_SEND_REQUEST_START_W_SEQ(sendreq, endpoint, seq, rc) \ do { \ rc = mca_pml_ob1_send_request_start_seq (sendreq, endpoint, seq); \ } while (0) /* * Mark a send request as completed at the MPI level. */ #define MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, with_signal) \ do { \ (sendreq)->req_send.req_base.req_ompi.req_status.MPI_SOURCE = \ (sendreq)->req_send.req_base.req_comm->c_my_rank; \ (sendreq)->req_send.req_base.req_ompi.req_status.MPI_TAG = \ (sendreq)->req_send.req_base.req_tag; \ (sendreq)->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS; \ (sendreq)->req_send.req_base.req_ompi.req_status._ucount = \ (sendreq)->req_send.req_bytes_packed; \ ompi_request_complete( &((sendreq)->req_send.req_base.req_ompi), (with_signal) ); \ \ PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_COMPLETE, \ &(sendreq->req_send.req_base), PERUSE_SEND); \ } while(0) /* * Release resources associated with a request */ #define MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq) \ do { \ /* Let the base handle the reference counts */ \ MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send)); \ OMPI_FREE_LIST_RETURN_MT( &mca_pml_base_send_requests, \ (ompi_free_list_item_t*)sendreq); \ } while(0) /* * The PML has completed a send request. Note that this request * may have been orphaned by the user or have already completed * at the MPI level. * This function will never be called directly from the upper level, as it * should only be an internal call to the PML. * */ static inline void send_request_pml_complete(mca_pml_ob1_send_request_t *sendreq) { assert(false == sendreq->req_send.req_base.req_pml_complete); if(sendreq->req_send.req_bytes_packed > 0) { PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_END, &(sendreq->req_send.req_base), PERUSE_SEND); } /* return mpool resources */ mca_pml_ob1_free_rdma_resources(sendreq); if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED && sendreq->req_send.req_addr != sendreq->req_send.req_base.req_addr) { mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); } OPAL_THREAD_LOCK(&ompi_request_lock); if(false == sendreq->req_send.req_base.req_ompi.req_complete) { /* Should only be called for long messages (maybe synchronous) */ MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, true); } else { if( MPI_SUCCESS != sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR ) { ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST); } } sendreq->req_send.req_base.req_pml_complete = true; if(sendreq->req_send.req_base.req_free_called) { MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq); } OPAL_THREAD_UNLOCK(&ompi_request_lock); } /* returns true if request was completed on PML level */ static inline bool send_request_pml_complete_check(mca_pml_ob1_send_request_t *sendreq) { #if OPAL_ENABLE_MULTI_THREADS opal_atomic_rmb(); #endif /* if no more events are expected for the request and the whole message is * already sent and send fragment scheduling isn't running in another * thread then complete the request on PML level. From now on, if user * called free on this request, the request structure can be reused for * another request or if the request is persistent it can be restarted */ if(sendreq->req_state == 0 && sendreq->req_bytes_delivered >= sendreq->req_send.req_bytes_packed && lock_send_request(sendreq)) { send_request_pml_complete(sendreq); return true; } return false; } /** * Schedule additional fragments */ int mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t*); static inline int mca_pml_ob1_send_request_schedule_exclusive(mca_pml_ob1_send_request_t* sendreq) { int rc; do { rc = mca_pml_ob1_send_request_schedule_once(sendreq); if(rc == OMPI_ERR_OUT_OF_RESOURCE) break; } while(!unlock_send_request(sendreq)); if(OMPI_SUCCESS == rc) send_request_pml_complete_check(sendreq); return rc; } static inline void mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq) { /* * Only allow one thread in this routine for a given request. * However, we cannot block callers on a mutex, so simply keep track * of the number of times the routine has been called and run through * the scheduling logic once for every call. */ if(!lock_send_request(sendreq)) return; mca_pml_ob1_send_request_schedule_exclusive(sendreq); } #if OPAL_CUDA_SUPPORT int mca_pml_ob1_send_request_start_cuda( mca_pml_ob1_send_request_t* sendreq, mca_bml_base_btl_t* bml_btl, size_t size); #endif /* OPAL_CUDA_SUPPORT */ /** * Start the specified request */ int mca_pml_ob1_send_request_start_buffered( mca_pml_ob1_send_request_t* sendreq, mca_bml_base_btl_t* bml_btl, size_t size); int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq, mca_bml_base_btl_t* bml_btl, size_t size); int mca_pml_ob1_send_request_start_prepare( mca_pml_ob1_send_request_t* sendreq, mca_bml_base_btl_t* bml_btl, size_t size); int mca_pml_ob1_send_request_start_rdma( mca_pml_ob1_send_request_t* sendreq, mca_bml_base_btl_t* bml_btl, size_t size); int mca_pml_ob1_send_request_start_rndv( mca_pml_ob1_send_request_t* sendreq, mca_bml_base_btl_t* bml_btl, size_t size, int flags); static inline int mca_pml_ob1_send_request_start_btl( mca_pml_ob1_send_request_t* sendreq, mca_bml_base_btl_t* bml_btl ) { size_t size = sendreq->req_send.req_bytes_packed; mca_btl_base_module_t* btl = bml_btl->btl; size_t eager_limit = btl->btl_eager_limit - sizeof(mca_pml_ob1_hdr_t); int rc; #if OPAL_CUDA_GDR_SUPPORT if (btl->btl_cuda_eager_limit && (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA)) { eager_limit = btl->btl_cuda_eager_limit - sizeof(mca_pml_ob1_hdr_t); } #endif /* OPAL_CUDA_GDR_SUPPORT */ if( OPAL_LIKELY(size <= eager_limit) ) { switch(sendreq->req_send.req_send_mode) { case MCA_PML_BASE_SEND_SYNCHRONOUS: rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, 0); break; case MCA_PML_BASE_SEND_BUFFERED: rc = mca_pml_ob1_send_request_start_copy(sendreq, bml_btl, size); break; case MCA_PML_BASE_SEND_COMPLETE: rc = mca_pml_ob1_send_request_start_prepare(sendreq, bml_btl, size); break; default: if (size != 0 && bml_btl->btl_flags & MCA_BTL_FLAGS_SEND_INPLACE) { rc = mca_pml_ob1_send_request_start_prepare(sendreq, bml_btl, size); } else { rc = mca_pml_ob1_send_request_start_copy(sendreq, bml_btl, size); } break; } } else { size = eager_limit; if(OPAL_UNLIKELY(btl->btl_rndv_eager_limit < eager_limit)) size = btl->btl_rndv_eager_limit; if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { rc = mca_pml_ob1_send_request_start_buffered(sendreq, bml_btl, size); } else if (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) { unsigned char *base; opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base ); if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_ob1_rdma_btls( sendreq->req_endpoint, base, sendreq->req_send.req_bytes_packed, sendreq->req_rdma))) { rc = mca_pml_ob1_send_request_start_rdma(sendreq, bml_btl, sendreq->req_send.req_bytes_packed); if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { mca_pml_ob1_free_rdma_resources(sendreq); } } else { rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, MCA_PML_OB1_HDR_FLAGS_CONTIG); } } else { #if OPAL_CUDA_SUPPORT if (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) { return mca_pml_ob1_send_request_start_cuda(sendreq, bml_btl, size); } #endif /* OPAL_CUDA_SUPPORT */ rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, 0); } } return rc; } static inline int mca_pml_ob1_send_request_start_seq (mca_pml_ob1_send_request_t* sendreq, mca_bml_base_endpoint_t* endpoint, int32_t seqn) { sendreq->req_endpoint = endpoint; sendreq->req_state = 0; sendreq->req_lock = 0; sendreq->req_pipeline_depth = 0; sendreq->req_bytes_delivered = 0; sendreq->req_pending = MCA_PML_OB1_SEND_PENDING_NONE; sendreq->req_send.req_base.req_sequence = seqn; MCA_PML_BASE_SEND_START( &sendreq->req_send.req_base ); for(size_t i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) { mca_bml_base_btl_t* bml_btl; int rc; /* select a btl */ bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager); rc = mca_pml_ob1_send_request_start_btl(sendreq, bml_btl); if( OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != rc) ) return rc; } add_request_to_send_pending(sendreq, MCA_PML_OB1_SEND_PENDING_START, true); return OMPI_SUCCESS; } static inline int mca_pml_ob1_send_request_start( mca_pml_ob1_send_request_t* sendreq ) { mca_bml_base_endpoint_t* endpoint = (mca_bml_base_endpoint_t*) sendreq->req_send.req_base.req_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]; mca_pml_ob1_comm_t* comm = sendreq->req_send.req_base.req_comm->c_pml_comm; int32_t seqn; if (OPAL_UNLIKELY(NULL == endpoint)) { return OMPI_ERR_UNREACH; } seqn = OPAL_THREAD_ADD32(&comm->procs[sendreq->req_send.req_base.req_peer].send_sequence, 1); return mca_pml_ob1_send_request_start_seq (sendreq, endpoint, seqn); } /** * Initiate a put scheduled by the receiver. */ void mca_pml_ob1_send_request_put( mca_pml_ob1_send_request_t* sendreq, mca_btl_base_module_t* btl, mca_pml_ob1_rdma_hdr_t* hdr ); int mca_pml_ob1_send_request_put_frag(mca_pml_ob1_rdma_frag_t* frag); /* This function tries to continue sendreq that was stuck because of resource * unavailability. A sendreq may be added to send_pending list if there is no * resource to send initial packet or there is not resource to schedule data * for sending. The reason the sendreq was added to the list is stored inside * sendreq struct and appropriate operation is retried when resource became * available. bml_btl passed to the function doesn't represents sendreq * destination, it represents BTL on which resource was freed, so only this BTL * should be considered for sending packets */ void mca_pml_ob1_send_request_process_pending(mca_bml_base_btl_t *bml_btl); void mca_pml_ob1_send_request_copy_in_out(mca_pml_ob1_send_request_t *sendreq, uint64_t send_offset, uint64_t send_length); END_C_DECLS #endif /* OMPI_PML_OB1_SEND_REQUEST_H */