c5c5b42307
This commit adds a new btl for one-sided and two-sided. This btl uses the uct layer in OpenUCX. This btl makes use of multiple uct contexts and per-thread device pinning to provide good performance when using threads and osc/rdma. This btl has been tested extensively with osc/rdma and passes all MTT tests on aries and IB hardware. For now this new component disables itself but can be enabled by setting the btl_ucx_transports MCA variable with a comma-delimited list of supported memory domains/transport layers. For example: --mca btl_uct_memory_domains ib/mlx5_0. The specific transports used can be selected using --mca btl_uct_transports. The default is to use any available transport. Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
63 строки
2.3 KiB
C
63 строки
2.3 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#if !defined(BTL_UCT_RDMA_H)
|
|
#define BTL_UCT_RDMA_H
|
|
|
|
#include "btl_uct.h"
|
|
#include "btl_uct_endpoint.h"
|
|
#include "btl_uct_frag.h"
|
|
|
|
/**
|
|
* @brief allocate a callback structure
|
|
*/
|
|
mca_btl_uct_uct_completion_t *mca_btl_uct_uct_completion_alloc (mca_btl_uct_module_t *btl, mca_btl_base_endpoint_t *endpoint,
|
|
void *local_address, mca_btl_base_registration_handle_t *local_handle,
|
|
mca_btl_uct_device_context_t *dev_context, mca_btl_base_rdma_completion_fn_t cbfunc,
|
|
void *cbcontext, void *cbdata);
|
|
/**
|
|
* @brief release a callback structure
|
|
*/
|
|
void mca_btl_uct_uct_completion_release (mca_btl_uct_uct_completion_t *comp);
|
|
|
|
void mca_btl_uct_uct_completion (uct_completion_t *uct_comp, ucs_status_t status);
|
|
|
|
/**
|
|
* @brief unpack the registration key and ensure the endpoint is connected
|
|
*
|
|
* @param[in] module uct btl module
|
|
* @param[in] context device context to use
|
|
* @param[in] endpoint btl endpoint
|
|
* @param[in] remote_handle buffer containing remote handle data
|
|
* @param[inout] rkey uct registration key bundle
|
|
* @param[out] ep_handle uct endpoint handle
|
|
*/
|
|
static inline int mca_btl_uct_get_rkey (mca_btl_uct_module_t *module,
|
|
mca_btl_uct_device_context_t *context,
|
|
mca_btl_base_endpoint_t *endpoint,
|
|
mca_btl_base_registration_handle_t *remote_handle,
|
|
uct_rkey_bundle_t *rkey,
|
|
uct_ep_h *ep_handle)
|
|
{
|
|
ucs_status_t ucs_status;
|
|
int rc;
|
|
|
|
rc = mca_btl_uct_endpoint_check_rdma (module, endpoint, context, ep_handle);
|
|
if (OPAL_SUCCESS != rc) {
|
|
return rc;
|
|
}
|
|
|
|
ucs_status = uct_rkey_unpack ((void *) remote_handle, rkey);
|
|
return (UCS_OK == ucs_status) ? OPAL_SUCCESS : OPAL_ERROR;
|
|
}
|
|
|
|
#endif /* !defined(BTL_UCT_RDMA_H) */
|