openmpi/ompi/mca/bcol/iboffload/bcol_iboffload_bcast.h

/*
 * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
 * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

#ifndef MCA_BCOL_IBOFFLOAD_BCAST_H
#define MCA_BCOL_IBOFFLOAD_BCAST_H

#include "ompi_config.h"

#include "bcol_iboffload.h"
#include "bcol_iboffload_frag.h"
#include "bcol_iboffload_task.h"
#include "bcol_iboffload_collreq.h"
#include "bcol_iboffload_collfrag.h"
#include "bcol_iboffload_endpoint.h"

#include "opal/include/opal/types.h"

BEGIN_C_DECLS

int mca_bcol_iboffload_small_msg_bcast_progress(
                        bcol_function_args_t *input_args,
                        struct mca_bcol_base_function_t *const_args);
int mca_bcol_iboffload_small_msg_bcast_extra_intra(bcol_function_args_t *fn_arguments,
        struct mca_bcol_base_function_t *const_args);
int mca_bcol_iboffload_small_msg_bcast_intra(bcol_function_args_t *fn_arguments,
        struct mca_bcol_base_function_t *const_args);
int mca_bcol_iboffload_bcast_scatter_allgather_intra(bcol_function_args_t *fn_arguments,
        struct mca_bcol_base_function_t *const_args);
int mca_bcol_iboffload_zero_copy_progress(bcol_function_args_t *fn_arguments,
        struct mca_bcol_base_function_t *const_args);
int mca_bcol_iboffload_bcast_scatter_allgather_extra_intra(bcol_function_args_t *fn_arguments,
        struct mca_bcol_base_function_t *const_args);
int mca_bcol_iboffload_bcast_register(mca_bcol_base_module_t *super);

static inline __opal_attribute_always_inline__ int
mca_bcol_iboffload_recv_rtr_setup(
        struct mqe_task **last_wait,
        uint32_t dest_rank,
        mca_bcol_iboffload_module_t *iboffload,
        mca_bcol_iboffload_collfrag_t *coll_fragment)
{
    mca_bcol_iboffload_task_t *task;
    mca_bcol_iboffload_frag_t *fragment;

    /* Wait for RTR message over credit QP */
    fragment = mca_bcol_iboffload_get_preposted_recv_frag(
            iboffload, dest_rank,
            MCA_BCOL_IBOFFLOAD_QP_CREDIT);
    if (OPAL_UNLIKELY(NULL == fragment)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get recv frag.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    task = mca_bcol_iboffload_get_wait_task(
            iboffload, dest_rank, 1, fragment, MCA_BCOL_IBOFFLOAD_QP_CREDIT,
            iboffload->endpoints[dest_rank]->qps[MCA_BCOL_IBOFFLOAD_QP_LARGE_BUFF].qp->lcl_qp);
    if (OPAL_UNLIKELY(NULL == task)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get wait task.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    MCA_BCOL_IBOFFLOAD_APPEND_TASK_TO_LIST(coll_fragment->task_next, task);
    MCA_BCOL_IBOFFLOAD_APPEND_MQ_TASK_TO_LIST(coll_fragment->tail_next, task);

    return OMPI_SUCCESS;
}

static inline __opal_attribute_always_inline__ int
mca_bcol_iboffload_send_small_buff_setup(
        struct mqe_task **last_send,
        size_t len, uint32_t dest_rank,
        mca_bcol_iboffload_module_t *iboffload,
        mca_bcol_iboffload_collfrag_t *coll_fragment)
{
    mca_bcol_iboffload_task_t *task;
    mca_bcol_iboffload_frag_t *fragment;

    mca_bcol_iboffload_collreq_t *coll_request =
        coll_fragment->coll_full_req;

    IBOFFLOAD_VERBOSE(10,("Get ml frag that I will send dest rank %d, len %d, lkey %d",
                            dest_rank, len, iboffload->rdma_block.ib_info.lkey));

    fragment = mca_bcol_iboffload_get_send_frag(coll_request, dest_rank,
                                 coll_request->qp_index, len, 0,
                                 SBUF, /* this could be problematic */
                                 MCA_BCOL_IBOFFLOAD_SEND_FRAG_ML);
    if (OPAL_UNLIKELY(NULL == fragment)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get frag.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    IBOFFLOAD_VERBOSE(10,("Get an rdma task for dest %d for packet size %d",
                            dest_rank,len));
    task = mca_bcol_iboffload_get_rdma_task(
                            dest_rank, 0,
                            fragment, iboffload, coll_fragment);

    if (OPAL_UNLIKELY(NULL == task)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get send task.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    *last_send = &task->element;

    MCA_BCOL_IBOFFLOAD_APPEND_TASK_TO_LIST(coll_fragment->task_next, task);
    MCA_BCOL_IBOFFLOAD_APPEND_MQ_TASK_TO_LIST(coll_fragment->tail_next, task);

    return OMPI_SUCCESS;
}

static inline __opal_attribute_always_inline__ int
mca_bcol_iboffload_send_large_buff_setup(
        struct mqe_task **last_send,
        int buf_index, int offset,
        size_t len, uint32_t dest_rank,
        mca_bcol_iboffload_module_t *iboffload,
        mca_bcol_iboffload_collfrag_t *coll_fragment)
{
    mca_bcol_iboffload_task_t *task;
    mca_bcol_iboffload_frag_t *fragment;

    mca_bcol_iboffload_collreq_t *coll_request =
        coll_fragment->coll_full_req;

    fragment = mca_bcol_iboffload_get_send_frag(coll_request, dest_rank,
                                 MCA_BCOL_IBOFFLOAD_QP_LARGE_BUFF,
                                 len,
                                 offset, buf_index, MCA_BCOL_IBOFFLOAD_SEND_FRAG_ML);
    if (OPAL_UNLIKELY(NULL == fragment)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get frag.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    task = mca_bcol_iboffload_get_send_task(
                            iboffload, dest_rank,
                            MCA_BCOL_IBOFFLOAD_QP_LARGE_BUFF,
                            fragment, coll_fragment, NO_INLINE);

    if (OPAL_UNLIKELY(NULL == task)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get send task.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    *last_send = &task->element;

    MCA_BCOL_IBOFFLOAD_APPEND_TASK_TO_LIST(coll_fragment->task_next, task);
    MCA_BCOL_IBOFFLOAD_APPEND_MQ_TASK_TO_LIST(coll_fragment->tail_next, task);

    return OMPI_SUCCESS;
}

static inline __opal_attribute_always_inline__ int
mca_bcol_iboffload_send_rtr_setup(
                            struct mqe_task **last_send,
                            uint32_t dest_rank,
                            mca_bcol_iboffload_module_t *iboffload,
                            mca_bcol_iboffload_collfrag_t *coll_fragment)
{
    mca_bcol_iboffload_task_t *task;
    mca_bcol_iboffload_frag_t *fragment;

    /* Recv is ready , Send RTR message */
    fragment = mca_bcol_iboffload_get_send_frag(coll_fragment->coll_full_req,
                                 dest_rank, MCA_BCOL_IBOFFLOAD_QP_CREDIT, 0,
                                 0, RBUF, MCA_BCOL_IBOFFLOAD_SEND_FRAG_DUMMY);
    if (OPAL_UNLIKELY(NULL == fragment)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get frag.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    task = mca_bcol_iboffload_get_send_task(iboffload, dest_rank,
            MCA_BCOL_IBOFFLOAD_QP_CREDIT,
            fragment, coll_fragment, INLINE);
    if (OPAL_UNLIKELY(NULL == task)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get send task.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    IBOFFLOAD_VERBOSE(10, ("dest_rank - %d. qp index - %d.\n",
                dest_rank, MCA_BCOL_IBOFFLOAD_QP_CREDIT));

    *last_send = &task->element;

    MCA_BCOL_IBOFFLOAD_APPEND_TASK_TO_LIST(coll_fragment->task_next, task);
    MCA_BCOL_IBOFFLOAD_APPEND_MQ_TASK_TO_LIST(coll_fragment->tail_next, task);

    return OMPI_SUCCESS;
}

static inline __opal_attribute_always_inline__ int
mca_bcol_iboffload_recv_small_preposted_buff_setup(
                            struct mqe_task **last_wait,
                            size_t len, uint32_t dest_rank,
                            int qp_index,
                            int nwaits,
                            mca_bcol_iboffload_module_t *iboffload,
                            mca_bcol_iboffload_collfrag_t *coll_fragment)
{
    mca_bcol_iboffload_task_t *task;
    mca_bcol_iboffload_frag_t *fragment;

    IBOFFLOAD_VERBOSE(10,("Get preposted recv from rank %d", dest_rank));

    fragment = mca_bcol_iboffload_get_preposted_recv_frag(
                               iboffload, dest_rank,
                               qp_index);
    if (OPAL_UNLIKELY(NULL == fragment)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get recv frag.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    task = mca_bcol_iboffload_get_wait_task(iboffload, dest_rank, nwaits,
            fragment, qp_index, NULL);
    if (OPAL_UNLIKELY(NULL == task)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get wait task.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    *last_wait = &task->element;

    MCA_BCOL_IBOFFLOAD_APPEND_TASK_TO_LIST(coll_fragment->task_next, task);
    MCA_BCOL_IBOFFLOAD_APPEND_MQ_TASK_TO_LIST(coll_fragment->tail_next, task);

    return OMPI_SUCCESS;
}

static inline __opal_attribute_always_inline__ int
mca_bcol_iboffload_recv_small_buff_setup(
                            struct mqe_task **last_wait,
                            size_t len, uint32_t dest_rank,
                            mca_bcol_iboffload_module_t *iboffload,
                            mca_bcol_iboffload_collfrag_t *coll_fragment)
{
    mca_bcol_iboffload_task_t *task;
    mca_bcol_iboffload_frag_t *fragment;

    mca_bcol_iboffload_collreq_t *coll_request =
        coll_fragment->coll_full_req;

    IBOFFLOAD_VERBOSE(10, ("Get preposted recv from rank %d", dest_rank));

    fragment = mca_bcol_iboffload_get_preposted_recv_frag(
                               iboffload, dest_rank,
                               coll_request->qp_index);
    if (OPAL_UNLIKELY(NULL == fragment)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get recv frag.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    task = mca_bcol_iboffload_get_wait_task(iboffload, dest_rank, 1,
            fragment, coll_request->qp_index, NULL);
    if (OPAL_UNLIKELY(NULL == task)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get wait task.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    *last_wait = &task->element;

    MCA_BCOL_IBOFFLOAD_APPEND_TASK_TO_LIST(coll_fragment->task_next, task);
    MCA_BCOL_IBOFFLOAD_APPEND_MQ_TASK_TO_LIST(coll_fragment->tail_next, task);

    return OMPI_SUCCESS;
}

static inline __opal_attribute_always_inline__ int
mca_bcol_iboffload_recv_large_buff_setup(
                            struct mqe_task **last_wait,
                            int buf_index, int offset,
                            size_t len, uint32_t dest_rank,
                            mca_bcol_iboffload_module_t *iboffload,
                            mca_bcol_iboffload_collfrag_t *coll_fragment)
{
    int num_preposted;

    mca_bcol_iboffload_task_t *task;
    mca_bcol_iboffload_frag_t *fragment;

    mca_bcol_iboffload_collreq_t *coll_request = coll_fragment->coll_full_req;

    /* Post message to recv queue for large messages */
    fragment = mca_bcol_iboffload_get_ml_frag(
            iboffload, MCA_BCOL_IBOFFLOAD_QP_LARGE_BUFF, len,
            coll_request->buffer_info[buf_index].iboffload_reg->mr->lkey,
            (uint64_t)((unsigned char *)coll_request->buffer_info[buf_index].buf + offset));
    if (OPAL_UNLIKELY(NULL == fragment)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get recv frag.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    num_preposted = mca_bcol_iboffload_prepost_ml_recv_frag(
            MCA_BCOL_IBOFFLOAD_QP_LARGE_BUFF,
            dest_rank, fragment, iboffload);
    if (0 >= num_preposted) {
        IBOFFLOAD_ERROR(("Failed to prepost recv fragments "
                    "return code - %d; dest_rank - %d",
                    num_preposted, dest_rank));

        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    task = mca_bcol_iboffload_get_wait_task(iboffload, dest_rank, 1,
            fragment, MCA_BCOL_IBOFFLOAD_QP_LARGE_BUFF, NULL);
    if (OPAL_UNLIKELY(NULL == task)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to get wait task.\n"));
        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
    }

    *last_wait = &task->element;

    MCA_BCOL_IBOFFLOAD_APPEND_TASK_TO_LIST(coll_fragment->task_next, task);
    MCA_BCOL_IBOFFLOAD_APPEND_MQ_TASK_TO_LIST(coll_fragment->tail_next, task);

    return OMPI_SUCCESS;
}

static inline __opal_attribute_always_inline__
int bcol_iboffload_binomial_root_to_src(int group_root, int my_rank,
        int pow2_size, int group_size, int *distance)
{
    int root, relative_rank, src,
        pow2_distance = 0, i;

    if (group_root < pow2_size) {
        root = group_root;
    } else {
        /* the source of the data is extra node,
           the real root it represented by some rank from
           pow2 group */
        root = group_root - pow2_size;
        /* shortcut for the case when my rank is root for the group */
        if (my_rank == root) {
            *distance = -1;
            return group_root;
        }
    }

    relative_rank = (my_rank - root) < 0 ? my_rank - root + pow2_size :
        my_rank - root;

    for (i = 1; i < pow2_size; i<<=1, pow2_distance++) {
        if (relative_rank & i) {
            src = my_rank ^ i;
            if (src >= pow2_size)
                src -= pow2_size;

            *distance = pow2_distance;
            IBOFFLOAD_VERBOSE(10, ("AAAAA d %d rel %d it %d root %d my %d", *distance, relative_rank, i, root, my_rank));
            return src;
        }
    }

    /* error case */
    *distance = -1;
    return -1;
}

static inline void bcol_iboffload_setup_binomial_connection(mca_bcol_iboffload_module_t *iboffload)
{
    netpatterns_pair_exchange_node_t *my_exchange_node =
                                          &iboffload->recursive_doubling_tree;

    int i, n_exchanges = my_exchange_node->n_exchanges,
        *exchanges = my_exchange_node->rank_exchanges,
        n_extra_src = my_exchange_node->n_extra_sources,
        my_rank = iboffload->ibnet->super.my_index,
        rank_extra_src = my_exchange_node->rank_extra_source;

    mca_bcol_iboffload_endpoint_t *ep;

    IBOFFLOAD_VERBOSE(10, ("Open connections.\n"));

    if (0 < n_extra_src) {
        ep = iboffload->endpoints[rank_extra_src];
        while (OMPI_SUCCESS !=
                check_endpoint_state(ep, NULL, NULL)) {
            opal_progress();
        }

#if OPAL_ENABLE_DEBUG
        {
            int qp_index, num_qps = mca_bcol_iboffload_component.num_qps;
            for (qp_index = 0; qp_index < num_qps; ++qp_index) {
                assert(NULL != ep->qps[qp_index].qp->lcl_qp);
                IBOFFLOAD_VERBOSE(10, ("Endpoint - %p, QP index - %d: qp num - %x.",
                                       ep, qp_index, ep->qps[qp_index].qp->lcl_qp->qp_num));
            }
        }
#endif

        /* Connect to all extra nodes */
        if (EXTRA_NODE == my_exchange_node->node_type) {
            for (i = iboffload->power_of_2_ranks;
                    i < iboffload->num_endpoints; ++i) {
                if (i != my_rank) {
                    ep = iboffload->endpoints[i];

                    IBOFFLOAD_VERBOSE(10, ("subgroup rank %d: Connect to rank %d.\n", my_rank, i));

                    while (OMPI_SUCCESS !=
                            check_endpoint_state(ep, NULL, NULL)) {
                        opal_progress();
                    }

#if OPAL_ENABLE_DEBUG
        {
            int qp_index, num_qps = mca_bcol_iboffload_component.num_qps;
            for (qp_index = 0; qp_index < num_qps; ++qp_index) {
                assert(NULL != ep->qps[qp_index].qp->lcl_qp);
                IBOFFLOAD_VERBOSE(10, ("Endpoint - %p, QP index - %d: qp num - %x.",
                                       ep, qp_index, ep->qps[qp_index].qp->lcl_qp->qp_num));
            }
        }
#endif
                }
            }
        }
    }

    for (i = 0; i < n_exchanges; ++i) {
        ep = iboffload->endpoints[exchanges[i]];

        while (OMPI_SUCCESS !=
                check_endpoint_state(ep, NULL, NULL)) {
            opal_progress();
        }

#if OPAL_ENABLE_DEBUG
        {
            int qp_index, num_qps = mca_bcol_iboffload_component.num_qps;
            for (qp_index = 0; qp_index < num_qps; ++qp_index) {
                assert(NULL != ep->qps[qp_index].qp->lcl_qp);
                IBOFFLOAD_VERBOSE(10, ("Endpoint - %p, QP index - %d: qp num - %x.",
                                       ep, qp_index, ep->qps[qp_index].qp->lcl_qp->qp_num));
            }
        }
#endif
    }
    /* set the connection status to connected */
    iboffload->connection_status[RECURSIVE_DOUBLING_TREE_BCAST] = true;
}

static inline __opal_attribute_always_inline__
int bcol_iboffload_bcast_binomial_gather(mca_bcol_iboffload_module_t *iboffload_module,
        struct mqe_task **last_send, struct mqe_task **last_wait,
        mca_bcol_iboffload_collfrag_t *coll_fragment,
        int count, int base_block_size, int radix_mask_pow)
{
    int rc;
    int i;
    int my_group_index = iboffload_module->ibnet->super.my_index;
    int delta, rdelta;

    IBOFFLOAD_VERBOSE(10, ("bcol_iboffload_bcast_binomial_gather %d %d",
                radix_mask_pow, my_group_index));

    /* we assume the iteration #iteration already was completed with probe */
    for (i = 0; i < iboffload_module->power_of_2; i++) {
        int pow2 = 1 << i;
        int peer_index = my_group_index ^ pow2;
        int slen, rlen,
            send_offset,
            recv_offset;

        if (i > radix_mask_pow) {
            slen = rlen = pow2 * base_block_size;
            send_offset = base_block_size * ((my_group_index) & ((~(int)0) << i));
            recv_offset = base_block_size * ((peer_index)     & ((~(int)0) << i));

            rdelta = count - recv_offset;
            if (rdelta > 0) {
                IBOFFLOAD_VERBOSE(10, ("Recv1 [ pow2 %d, radix %d ] offset %d , len %d , dest %d",
                            pow2, 1 << iboffload_module->power_of_2,
                            recv_offset, rlen, peer_index));

                rc = mca_bcol_iboffload_send_rtr_setup(last_send,
                        peer_index, iboffload_module,
                        coll_fragment);
                if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                    IBOFFLOAD_VERBOSE(10, ("Failed to setup send rtr"));
                    return OMPI_ERROR;
                }
            }

            delta = count - send_offset;
            if (delta > 0) {
                if (delta < slen) {
                    /* recv the tail */
                    slen = delta;
                }

                IBOFFLOAD_VERBOSE(10, ("Send1 [ pow2 %d, radix %d ] offset %d , len %d , dest %d",
                            pow2, 1 << iboffload_module->power_of_2,
                            send_offset, slen, peer_index));
                rc = mca_bcol_iboffload_recv_rtr_setup(last_wait, peer_index, iboffload_module, coll_fragment);
                if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                    IBOFFLOAD_VERBOSE(10, ("Failed to isend data"));
                    return OMPI_ERROR;
                }

                rc = mca_bcol_iboffload_send_large_buff_setup(last_send, SBUF, send_offset, slen, peer_index,
                        iboffload_module, coll_fragment);
                if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                    IBOFFLOAD_VERBOSE(10, ("Failed to isend data"));
                    return OMPI_ERROR;
                }
            }

            if (rdelta > 0) {
                if (rdelta < rlen) {
                    /* recv the tail */
                    rlen = rdelta;
                }

                rc = mca_bcol_iboffload_recv_large_buff_setup(last_wait,
                        SBUF, recv_offset, rlen, peer_index,
                        iboffload_module, coll_fragment);
                if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                    IBOFFLOAD_VERBOSE(10, ("Failed to setup data receive"));
                    return OMPI_ERROR;
                }
            }

        } else if (i == radix_mask_pow) {
            /* only receive data */
            rlen = pow2 * base_block_size;
            recv_offset = base_block_size * ((peer_index) & ((~(int)0) << i));
            delta = count - recv_offset;
            if (0 >= delta) {
                /* we have nothing to send, skip the iteration */
                continue;
            }
            if (delta < rlen) {
                /* recv the tail */
                rlen = delta;
            }
            /* receive data from the peer */
            IBOFFLOAD_VERBOSE(10, ("Recv2 [ pow2 %d, radix %d ] offset %d , len %d , dest %d",
                        pow2,
                        1 << iboffload_module->power_of_2,
                        recv_offset,
                        rlen, peer_index));
            rc = mca_bcol_iboffload_send_rtr_setup(last_send,
                    peer_index, iboffload_module,
                    coll_fragment);
            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                IBOFFLOAD_VERBOSE(10, ("Failed to setup send rtr"));
                return OMPI_ERROR;
            }

            rc = mca_bcol_iboffload_recv_large_buff_setup(last_wait,
                    SBUF, recv_offset, rlen, peer_index,
                    iboffload_module, coll_fragment);
            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                IBOFFLOAD_VERBOSE(10, ("Failed to setup data receive"));
                return OMPI_ERROR;
            }
        } else if (i < radix_mask_pow) {
            /* Only send data */
            slen = pow2 * base_block_size;
            send_offset = base_block_size * ((my_group_index) & ((~(int)0) << i));
            delta = count - send_offset;
            if (0 >= delta) {
                /* we have nothing to send, skip the iteration */
                continue;
            }

            if (delta < slen) {
                slen = delta;
            }

            IBOFFLOAD_VERBOSE(10, ("Send2 [ pow2 %d, radix %d ] offset %d , len %d , dest %d",
                        pow2,
                        1 << iboffload_module->power_of_2,
                        send_offset,
                        slen,
                        peer_index));

            rc = mca_bcol_iboffload_recv_rtr_setup(last_wait, peer_index, iboffload_module, coll_fragment);
            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                IBOFFLOAD_VERBOSE(10, ("Failed to isend data"));
                return OMPI_ERROR;
            }

            rc = mca_bcol_iboffload_send_large_buff_setup(last_send, SBUF, send_offset, slen, peer_index,
                    iboffload_module, coll_fragment);
            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                IBOFFLOAD_VERBOSE(10, ("Failed to isend data"));
                return OMPI_ERROR;
            }
        }
    }

    return OMPI_SUCCESS;
}

END_C_DECLS

#endif