
The old BTL interface provided support for RDMA through the use of the btl_prepare_src and btl_prepare_dst functions. These functions were expected to prepare as much of the user buffer as possible for the RDMA operation and return a descriptor. The descriptor contained segment information on the prepared region. The btl user could then pass the RDMA segment information to a remote peer. Once the peer received that information it then packed it into a similar descriptor on the other side that could then be passed into a single btl_put or btl_get operation. Changes: - Added functions to register and deregister memory regions with the btl. If no registration is needed a btl should set these function pointers to NULL. These function take over for btl_prepare_src/dst and btl_free for RDMA operations. The caller should specify the maximum permissions needed on the memory. - Changed the function signatures for both btl_put and btl_get. In place of a prepared descriptor the caller should provide the source and destination addresses and registration handles as well as a new callback function. The callback will be provided with the local address and registration handle, callback context, callback data, and status. See mca_btl_base_rdma_completion_fn_t in btl.h. - Added a new btl constraint: MCA_BTL_REG_HANDLE_MAX_SIZE. This value specifies the maximum size of any btl's registration handle. - Removed the btl_prepare_dst function. This reflects the fact that RDMA operations no longer depend on "prepared" descriptors. - Removed the btl_seg_size member. There is no need to btl's to subclass the mca_btl_base_segment_t class anymore. - Expose the btl's put/get limitations with new struct members: btl_put_limit, btl_put_alignment, btl_get_limit, btl_get_alignment. - Remove the mca_mpool_base_registration_t argument from the btl_prepare_src function. The argument was intended to support RDMA operations and is no longer necessary. - Remove des_remote/des_remote_count from the mca_btl_base_descriptor_t structure. This structure member was originally used to specify the remote segment for RDMA operations. Since the new btl interface no longer uses desriptors for RDMA this member no longer has a purpose. In addition to removing these members the local segment structure fields have been renamed to from des_local/des_local_count to des_segments/des_segment_count. Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
171 строка
5.9 KiB
C
171 строка
5.9 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2007 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2006-2007 Sun Microsystems, Inc. All rights reserved.
|
|
* Copyright (c) 2008-2013 Cisco Systems, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
|
|
#include "opal_config.h"
|
|
#include <stdio.h>
|
|
|
|
#include "opal/mca/mca.h"
|
|
#include "opal/util/output.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "opal/mca/btl/btl.h"
|
|
#include "opal/mca/btl/base/base.h"
|
|
|
|
mca_btl_active_message_callback_t mca_btl_base_active_message_trigger[MCA_BTL_TAG_MAX];
|
|
|
|
/*
|
|
* mca_btl_base_descriptor_t
|
|
*/
|
|
|
|
static void mca_btl_base_descriptor_constructor(mca_btl_base_descriptor_t* des)
|
|
{
|
|
des->des_segments = NULL;
|
|
des->des_segment_count = 0;
|
|
des->des_cbfunc = NULL;
|
|
des->des_cbdata = NULL;
|
|
des->des_flags = 0;
|
|
}
|
|
|
|
static void mca_btl_base_descriptor_destructor(mca_btl_base_descriptor_t* des)
|
|
{
|
|
}
|
|
|
|
OBJ_CLASS_INSTANCE(
|
|
mca_btl_base_descriptor_t,
|
|
opal_list_item_t,
|
|
mca_btl_base_descriptor_constructor,
|
|
mca_btl_base_descriptor_destructor);
|
|
|
|
|
|
/*
|
|
* The following file was created by configure. It contains extern
|
|
* statements and the definition of an array of pointers to each
|
|
* component's public mca_base_component_t struct.
|
|
*/
|
|
|
|
#include "opal/mca/btl/base/static-components.h"
|
|
#include "btl_base_error.h"
|
|
|
|
/*
|
|
* Global variables
|
|
*/
|
|
char* mca_btl_base_include = NULL;
|
|
char* mca_btl_base_exclude = NULL;
|
|
int mca_btl_base_warn_component_unused = 1;
|
|
opal_list_t mca_btl_base_modules_initialized;
|
|
bool mca_btl_base_thread_multiple_override = false;
|
|
|
|
static int mca_btl_base_register(mca_base_register_flag_t flags)
|
|
{
|
|
/* Override the per-BTL "don't run if THREAD_MULTIPLE selected"
|
|
embargo? */
|
|
mca_btl_base_thread_multiple_override = false;
|
|
(void) mca_base_var_register("opal", "btl", "base", "thread_multiple_override",
|
|
"Enable BTLs that are not normally enabled when MPI_THREAD_MULTIPLE is enabled (THIS IS FOR DEVELOPERS ONLY! SHOULD NOT BE USED BY END USERS!)",
|
|
MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
|
|
MCA_BASE_VAR_FLAG_INTERNAL,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_btl_base_thread_multiple_override);
|
|
|
|
(void) mca_base_var_register("opal", "btl", "base", "include", NULL,
|
|
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_btl_base_include);
|
|
(void) mca_base_var_register("opal", "btl", "base", "exclude", NULL,
|
|
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_btl_base_exclude);
|
|
(void) mca_base_var_register("opal", "btl", "base", "warn_component_unused",
|
|
"This parameter is used to turn on warning messages when certain NICs are not used",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_btl_base_warn_component_unused);
|
|
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
/**
|
|
* Function for finding and opening either all MCA components, or the one
|
|
* that was specifically requested via a MCA parameter.
|
|
*/
|
|
static int mca_btl_base_open(mca_base_open_flag_t flags)
|
|
{
|
|
int ret;
|
|
|
|
/* Open up all available components */
|
|
|
|
if (OPAL_SUCCESS !=
|
|
(ret = mca_base_framework_components_open(&opal_btl_base_framework, flags))) {
|
|
return ret;
|
|
}
|
|
|
|
/* Initialize the list so that in mca_btl_base_close(), we can
|
|
iterate over it (even if it's empty, as in the case of
|
|
opal_info) */
|
|
|
|
OBJ_CONSTRUCT(&mca_btl_base_modules_initialized, opal_list_t);
|
|
|
|
/* get the verbosity so that BTL_VERBOSE will work */
|
|
mca_btl_base_verbose = opal_output_get_verbosity(opal_btl_base_framework.framework_output);
|
|
|
|
/* All done */
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
static int mca_btl_base_close(void)
|
|
{
|
|
mca_btl_base_selected_module_t *sm, *next;
|
|
|
|
#if 0
|
|
/* disable event processing while cleaning up btls */
|
|
opal_event_disable();
|
|
#endif
|
|
/* Finalize all the btl components and free their list items */
|
|
|
|
OPAL_LIST_FOREACH_SAFE(sm, next, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
|
|
/* Blatently ignore the return code (what would we do to recover,
|
|
anyway? This component is going away, so errors don't matter
|
|
anymore) */
|
|
|
|
sm->btl_module->btl_finalize(sm->btl_module);
|
|
opal_list_remove_item(&mca_btl_base_modules_initialized, &sm->super);
|
|
free(sm);
|
|
}
|
|
|
|
(void) mca_base_framework_components_close(&opal_btl_base_framework, NULL);
|
|
|
|
OBJ_DESTRUCT(&mca_btl_base_modules_initialized);
|
|
|
|
#if 0
|
|
/* restore event processing */
|
|
opal_event_enable();
|
|
#endif
|
|
/* All done */
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
MCA_BASE_FRAMEWORK_DECLARE(opal, btl, "Byte Transport Layer", mca_btl_base_register,
|
|
mca_btl_base_open, mca_btl_base_close, mca_btl_base_static_components,
|
|
0);
|