openmpi/ompi/mca/pml/ob1/pml_ob1_sendreq.h

/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2016 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
 * Copyright (c) 2011-2012 NVIDIA Corporation.  All rights reserved.
 * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights
 *                         reserved.
 * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

#ifndef OMPI_PML_OB1_SEND_REQUEST_H
#define OMPI_PML_OB1_SEND_REQUEST_H

#include "opal/datatype/opal_convertor.h"
#include "opal/mca/mpool/base/base.h"
#include "ompi/mca/pml/base/pml_base_sendreq.h"
#include "pml_ob1_comm.h"
#include "pml_ob1_hdr.h"
#include "pml_ob1_rdma.h"
#include "pml_ob1_rdmafrag.h"
#include "ompi/mca/bml/bml.h"

BEGIN_C_DECLS

typedef enum {
    MCA_PML_OB1_SEND_PENDING_NONE,
    MCA_PML_OB1_SEND_PENDING_SCHEDULE,
    MCA_PML_OB1_SEND_PENDING_START
} mca_pml_ob1_send_pending_t;

struct mca_pml_ob1_send_request_t {
    mca_pml_base_send_request_t req_send;
    mca_bml_base_endpoint_t* req_endpoint;
    opal_ptr_t req_recv;
    int32_t  req_state;
    int32_t  req_lock;
    bool     req_throttle_sends;
    int32_t  req_pipeline_depth;
    size_t   req_bytes_delivered;
    uint32_t req_rdma_cnt;
    mca_pml_ob1_send_pending_t req_pending;
    opal_mutex_t req_send_range_lock;
    opal_list_t req_send_ranges;
    mca_pml_ob1_rdma_frag_t *rdma_frag;
    /** The size of this array is set from mca_pml_ob1.max_rdma_per_request */
    mca_pml_ob1_com_btl_t req_rdma[];
};
typedef struct mca_pml_ob1_send_request_t mca_pml_ob1_send_request_t;

OBJ_CLASS_DECLARATION(mca_pml_ob1_send_request_t);

struct mca_pml_ob1_send_range_t {
    opal_free_list_item_t base;
    uint64_t range_send_offset;
    uint64_t range_send_length;
    int range_btl_idx;
    int range_btl_cnt;
    /** The size of this array is set from mca_pml_ob1.max_send_per_range */
    mca_pml_ob1_com_btl_t range_btls[];
};
typedef struct mca_pml_ob1_send_range_t mca_pml_ob1_send_range_t;
OBJ_CLASS_DECLARATION(mca_pml_ob1_send_range_t);

static inline bool lock_send_request(mca_pml_ob1_send_request_t *sendreq)
{
    return OPAL_THREAD_ADD_FETCH32(&sendreq->req_lock,  1) == 1;
}

static inline bool unlock_send_request(mca_pml_ob1_send_request_t *sendreq)
{
    return OPAL_THREAD_ADD_FETCH32(&sendreq->req_lock, -1) == 0;
}

static inline void
add_request_to_send_pending(mca_pml_ob1_send_request_t* sendreq,
                            const mca_pml_ob1_send_pending_t type,
                            const bool append)
{
    opal_list_item_t *item = (opal_list_item_t*)sendreq;

    OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
    sendreq->req_pending = type;
    if(append)
        opal_list_append(&mca_pml_ob1.send_pending, item);
    else
        opal_list_prepend(&mca_pml_ob1.send_pending, item);

    OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);
    mca_pml_ob1_enable_progress(1);
}

static inline mca_pml_ob1_send_request_t*
get_request_from_send_pending(mca_pml_ob1_send_pending_t *type)
{
    mca_pml_ob1_send_request_t *sendreq;

    OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
    sendreq = (mca_pml_ob1_send_request_t*)
        opal_list_remove_first(&mca_pml_ob1.send_pending);
    if(sendreq) {
        *type = sendreq->req_pending;
        sendreq->req_pending = MCA_PML_OB1_SEND_PENDING_NONE;
    }
    OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);

    return sendreq;
}

#define MCA_PML_OB1_SEND_REQUEST_ALLOC( comm,                           \
                                        dst,                            \
                                        sendreq)                        \
    {                                                                   \
        ompi_proc_t *proc = ompi_comm_peer_lookup( comm, dst );         \
                                                                        \
        if( OPAL_LIKELY(NULL != proc) ) {                               \
            sendreq = (mca_pml_ob1_send_request_t*)                     \
                opal_free_list_wait (&mca_pml_base_send_requests);      \
            sendreq->req_send.req_base.req_proc = proc;                 \
        }                                                               \
    }


#define MCA_PML_OB1_SEND_REQUEST_INIT( sendreq,                         \
                                       buf,                             \
                                       count,                           \
                                       datatype,                        \
                                       dst,                             \
                                       tag,                             \
                                       comm,                            \
                                       sendmode,                        \
                                       persistent)                      \
    {                                                                   \
        MCA_PML_BASE_SEND_REQUEST_INIT(&(sendreq)->req_send,            \
                                       buf,                             \
                                       count,                           \
                                       datatype,                        \
                                       dst,                             \
                                       tag,                             \
                                       comm,                            \
                                       sendmode,                        \
                                       persistent,                      \
                                       0); /* convertor_flags */        \
        (sendreq)->req_recv.pval = NULL;                                \
    }

#define MCA_PML_OB1_SEND_REQUEST_RESET(sendreq)                         \
    MCA_PML_BASE_SEND_REQUEST_RESET(&(sendreq)->req_send)

static inline void mca_pml_ob1_free_rdma_resources (mca_pml_ob1_send_request_t* sendreq)
{
    size_t r;

    /* return mpool resources */
    for(r = 0; r < sendreq->req_rdma_cnt; r++) {
        struct mca_btl_base_registration_handle_t *handle = sendreq->req_rdma[r].btl_reg;
        mca_bml_base_btl_t *bml_btl = sendreq->req_rdma[r].bml_btl;

        if (NULL != handle) {
            mca_bml_base_deregister_mem (bml_btl, handle);
            sendreq->req_rdma[r].btl_reg = NULL;
        }
    }
    sendreq->req_rdma_cnt = 0;
}


/**
 * Start a send request.
 */

#define MCA_PML_OB1_SEND_REQUEST_START(sendreq, rc)       \
    do {                                                  \
        rc = mca_pml_ob1_send_request_start(sendreq);     \
    } while (0)

#define MCA_PML_OB1_SEND_REQUEST_START_W_SEQ(sendreq, endpoint, seq, rc) \
    do {                                                                \
        rc = mca_pml_ob1_send_request_start_seq (sendreq, endpoint, seq); \
    } while (0)


/*
 * Mark a send request as completed at the MPI level.
 */

#define MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, with_signal)                  \
do {                                                                                 \
   (sendreq)->req_send.req_base.req_ompi.req_status.MPI_SOURCE =                     \
       (sendreq)->req_send.req_base.req_comm->c_my_rank;                             \
   (sendreq)->req_send.req_base.req_ompi.req_status.MPI_TAG =                        \
        (sendreq)->req_send.req_base.req_tag;                                        \
   (sendreq)->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;        \
   (sendreq)->req_send.req_base.req_ompi.req_status._ucount =                        \
        (sendreq)->req_send.req_bytes_packed;                                        \
   PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_COMPLETE,                                \
                            &(sendreq->req_send.req_base), PERUSE_SEND);             \
                                                                                     \
   ompi_request_complete( &((sendreq)->req_send.req_base.req_ompi), (with_signal) ); \
} while(0)

static inline void mca_pml_ob1_send_request_fini (mca_pml_ob1_send_request_t *sendreq)
{
    /*  Let the base handle the reference counts */
    MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send));
    assert( NULL == sendreq->rdma_frag );
}

/*
 * Release resources associated with a request and return the request.
 */

#define MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq)                        \
    do {                                                                \
        mca_pml_ob1_send_request_fini (sendreq);                        \
        opal_free_list_return ( &mca_pml_base_send_requests,            \
                                (opal_free_list_item_t*)sendreq);       \
        sendreq = NULL;  /* for safety */                               \
    } while(0)


/*
 * The PML has completed a send request. Note that this request
 * may have been orphaned by the user or have already completed
 * at the MPI level.
 * This function will never be called directly from the upper level, as it
 * should only be an internal call to the PML.
 *
 */
static inline void
send_request_pml_complete(mca_pml_ob1_send_request_t *sendreq)
{
    if(false == sendreq->req_send.req_base.req_pml_complete) {
        if(sendreq->req_send.req_bytes_packed > 0) {
            PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_END,
                                     &(sendreq->req_send.req_base), PERUSE_SEND);
        }

        /* return mpool resources */
        mca_pml_ob1_free_rdma_resources(sendreq);

        if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED &&
            sendreq->req_send.req_addr != sendreq->req_send.req_base.req_addr) {
            mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq);
        }

        if (!sendreq->req_send.req_base.req_free_called) {
            sendreq->req_send.req_base.req_pml_complete = true;

            if( !REQUEST_COMPLETE( &((sendreq->req_send).req_base.req_ompi)) ) {
                /* Should only be called for long messages (maybe synchronous) */
                MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, true);
            } else {
                if( MPI_SUCCESS != sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR ) {
                    ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST);
                }
            }
        } else {
            MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq);
        }
    }
}

/* returns true if request was completed on PML level */
static inline bool
send_request_pml_complete_check(mca_pml_ob1_send_request_t *sendreq)
{
#if OPAL_ENABLE_MULTI_THREADS
    opal_atomic_rmb();
#endif
    /* if no more events are expected for the request and the whole message is
     * already sent and send fragment scheduling isn't running in another
     * thread then complete the request on PML level. From now on, if user
     * called free on this request, the request structure can be reused for
     * another request or if the request is persistent it can be restarted */
    if(sendreq->req_state == 0 &&
            sendreq->req_bytes_delivered >= sendreq->req_send.req_bytes_packed
            && lock_send_request(sendreq)) {
        send_request_pml_complete(sendreq);
        return true;
    }

    return false;
}

/**
 *  Schedule additional fragments
 */
int
mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t*);

static inline int
mca_pml_ob1_send_request_schedule_exclusive(mca_pml_ob1_send_request_t* sendreq)
{
    int rc;
    do {
        rc = mca_pml_ob1_send_request_schedule_once(sendreq);
        if(rc == OMPI_ERR_OUT_OF_RESOURCE)
            break;
    } while(!unlock_send_request(sendreq));

    if(OMPI_SUCCESS == rc)
        send_request_pml_complete_check(sendreq);

    return rc;
}

static inline void
mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq)
{
    /*
     * Only allow one thread in this routine for a given request.
     * However, we cannot block callers on a mutex, so simply keep track
     * of the number of times the routine has been called and run through
     * the scheduling logic once for every call.
     */

    if(!lock_send_request(sendreq))
        return;

    mca_pml_ob1_send_request_schedule_exclusive(sendreq);
}

#if OPAL_CUDA_SUPPORT
int mca_pml_ob1_send_request_start_cuda(
    mca_pml_ob1_send_request_t* sendreq,
    mca_bml_base_btl_t* bml_btl,
    size_t size);
#endif /* OPAL_CUDA_SUPPORT */

/**
 *  Start the specified request
 */

int mca_pml_ob1_send_request_start_buffered(
    mca_pml_ob1_send_request_t* sendreq,
    mca_bml_base_btl_t* bml_btl,
    size_t size);

int mca_pml_ob1_send_request_start_copy(
    mca_pml_ob1_send_request_t* sendreq,
    mca_bml_base_btl_t* bml_btl,
    size_t size);

int mca_pml_ob1_send_request_start_prepare(
    mca_pml_ob1_send_request_t* sendreq,
    mca_bml_base_btl_t* bml_btl,
    size_t size);

int mca_pml_ob1_send_request_start_rdma(
    mca_pml_ob1_send_request_t* sendreq,
    mca_bml_base_btl_t* bml_btl,
    size_t size);

int mca_pml_ob1_send_request_start_rndv(
    mca_pml_ob1_send_request_t* sendreq,
    mca_bml_base_btl_t* bml_btl,
    size_t size,
    int flags);

static inline int
mca_pml_ob1_send_request_start_btl( mca_pml_ob1_send_request_t* sendreq,
                                    mca_bml_base_btl_t* bml_btl )
{
    size_t size = sendreq->req_send.req_bytes_packed;
    mca_btl_base_module_t* btl = bml_btl->btl;
    size_t eager_limit = btl->btl_eager_limit - sizeof(mca_pml_ob1_hdr_t);
    int rc;

#if OPAL_CUDA_GDR_SUPPORT
    if (btl->btl_cuda_eager_limit && (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA)) {
        eager_limit = btl->btl_cuda_eager_limit - sizeof(mca_pml_ob1_hdr_t);
    }
#endif /* OPAL_CUDA_GDR_SUPPORT */

    if( OPAL_LIKELY(size <= eager_limit) ) {
        switch(sendreq->req_send.req_send_mode) {
        case MCA_PML_BASE_SEND_SYNCHRONOUS:
            rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, 0);
            break;
        case MCA_PML_BASE_SEND_BUFFERED:
            rc = mca_pml_ob1_send_request_start_copy(sendreq, bml_btl, size);
            break;
        case MCA_PML_BASE_SEND_COMPLETE:
            rc = mca_pml_ob1_send_request_start_prepare(sendreq, bml_btl, size);
            break;
        default:
            if (size != 0 && bml_btl->btl_flags & MCA_BTL_FLAGS_SEND_INPLACE) {
                rc = mca_pml_ob1_send_request_start_prepare(sendreq, bml_btl, size);
            } else {
                rc = mca_pml_ob1_send_request_start_copy(sendreq, bml_btl, size);
            }
            break;
        }
    } else {
        size = eager_limit;
        if(OPAL_UNLIKELY(btl->btl_rndv_eager_limit < eager_limit))
            size = btl->btl_rndv_eager_limit;
        if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
            rc = mca_pml_ob1_send_request_start_buffered(sendreq, bml_btl, size);
        } else if
                (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) {
            unsigned char *base;
            opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base );

            if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_ob1_rdma_btls(
                                                                              sendreq->req_endpoint,
                                                                              base,
                                                                              sendreq->req_send.req_bytes_packed,
                                                                              sendreq->req_rdma))) {
                rc = mca_pml_ob1_send_request_start_rdma(sendreq, bml_btl,
                                                         sendreq->req_send.req_bytes_packed);
                if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
                    mca_pml_ob1_free_rdma_resources(sendreq);
                }
            } else {
                rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size,
                                                         MCA_PML_OB1_HDR_FLAGS_CONTIG);
            }
        } else {
#if OPAL_CUDA_SUPPORT
            if (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) {
                return mca_pml_ob1_send_request_start_cuda(sendreq, bml_btl, size);
            }
#endif /* OPAL_CUDA_SUPPORT */
            rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, 0);
        }
    }

    return rc;
}

static inline int
mca_pml_ob1_send_request_start_seq (mca_pml_ob1_send_request_t* sendreq, mca_bml_base_endpoint_t* endpoint, int32_t seqn)
{
    sendreq->req_endpoint = endpoint;
    sendreq->req_state = 0;
    sendreq->req_lock = 0;
    sendreq->req_pipeline_depth = 0;
    sendreq->req_bytes_delivered = 0;
    sendreq->req_pending = MCA_PML_OB1_SEND_PENDING_NONE;
    sendreq->req_send.req_base.req_sequence = seqn;

    MCA_PML_BASE_SEND_START( &sendreq->req_send );

    for(size_t i = 0; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager); i++) {
        mca_bml_base_btl_t* bml_btl;
        int rc;

        /* select a btl */
        bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
        rc = mca_pml_ob1_send_request_start_btl(sendreq, bml_btl);
        if( OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != rc) )
            return rc;
    }
    if(MCA_PML_BASE_SEND_BUFFERED == sendreq->req_send.req_send_mode &&
       sendreq->req_send.req_addr == sendreq->req_send.req_base.req_addr) {
        /* in the buffered mode, the send buffer must be saved to
         * the attached buffer before returning it to the user */
        int rc;
        rc = mca_pml_base_bsend_request_start((ompi_request_t*)sendreq);
        if(OMPI_SUCCESS != rc){
            return rc;
        }
    }
    add_request_to_send_pending(sendreq, MCA_PML_OB1_SEND_PENDING_START, true);

    return OMPI_SUCCESS;
}

static inline int
mca_pml_ob1_send_request_start( mca_pml_ob1_send_request_t* sendreq )
{
    mca_bml_base_endpoint_t *endpoint = mca_bml_base_get_endpoint (sendreq->req_send.req_base.req_proc);
    ompi_communicator_t *comm = sendreq->req_send.req_base.req_comm;
    mca_pml_ob1_comm_proc_t *ob1_proc = mca_pml_ob1_peer_lookup (comm, sendreq->req_send.req_base.req_peer);
    int32_t seqn;

    if (OPAL_UNLIKELY(NULL == endpoint)) {
        return OMPI_ERR_UNREACH;
    }

    seqn = OPAL_THREAD_ADD_FETCH32(&ob1_proc->send_sequence, 1);

    return mca_pml_ob1_send_request_start_seq (sendreq, endpoint, seqn);
}

/**
 *  Initiate a put scheduled by the receiver.
 */

void mca_pml_ob1_send_request_put( mca_pml_ob1_send_request_t* sendreq,
                                   mca_btl_base_module_t* btl,
                                   mca_pml_ob1_rdma_hdr_t* hdr );

int mca_pml_ob1_send_request_put_frag(mca_pml_ob1_rdma_frag_t* frag);

/* This function tries to continue sendreq that was stuck because of resource
 * unavailability. A sendreq may be added to send_pending list if there is no
 * resource to send initial packet or there is not resource to schedule data
 * for sending. The reason the sendreq was added to the list is stored inside
 * sendreq struct and appropriate operation is retried when resource became
 * available. bml_btl passed to the function doesn't represents sendreq
 * destination, it represents BTL on which resource was freed, so only this BTL
 * should be considered for sending packets */
void mca_pml_ob1_send_request_process_pending(mca_bml_base_btl_t *bml_btl);

void mca_pml_ob1_send_request_copy_in_out(mca_pml_ob1_send_request_t *sendreq,
                uint64_t send_offset, uint64_t send_length);

END_C_DECLS

#endif  /* OMPI_PML_OB1_SEND_REQUEST_H */