210 строки
11 KiB
C
210 строки
11 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2006-2007 Voltaire All rights reserved.
|
|
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
|
|
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
|
|
* Copyright (c) 2013 NVIDIA Corporation. All rights reserved.
|
|
*
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "opal_config.h"
|
|
|
|
#include <stdio.h>
|
|
|
|
#include "opal/util/output.h"
|
|
#include "opal/constants.h"
|
|
#include "opal/mca/btl/btl.h"
|
|
#include "opal/mca/btl/base/base.h"
|
|
|
|
int mca_btl_base_param_register(mca_base_component_t *version,
|
|
mca_btl_base_module_t *module)
|
|
{
|
|
char *msg;
|
|
|
|
/* If this is ever triggered change the uint32_ts in mca_btl_base_module_t to unsigned ints */
|
|
assert(sizeof(unsigned int) == sizeof(uint32_t));
|
|
|
|
(void) mca_base_component_var_register(version, "exclusivity",
|
|
"BTL exclusivity (must be >= 0)",
|
|
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_7,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_exclusivity);
|
|
|
|
asprintf(&msg, "BTL bit flags (general flags: SEND=%d, PUT=%d, GET=%d, SEND_INPLACE=%d, HETEROGENEOUS_RDMA=%d, "
|
|
"ATOMIC_OPS=%d; flags only used by the \"dr\" PML (ignored by others): ACK=%d, CHECKSUM=%d, "
|
|
"RDMA_COMPLETION=%d; flags only used by the \"bfo\" PML (ignored by others): FAILOVER_SUPPORT=%d)",
|
|
MCA_BTL_FLAGS_SEND,
|
|
MCA_BTL_FLAGS_PUT,
|
|
MCA_BTL_FLAGS_GET,
|
|
MCA_BTL_FLAGS_SEND_INPLACE,
|
|
MCA_BTL_FLAGS_HETEROGENEOUS_RDMA,
|
|
MCA_BTL_FLAGS_ATOMIC_OPS,
|
|
MCA_BTL_FLAGS_NEED_ACK,
|
|
MCA_BTL_FLAGS_NEED_CSUM,
|
|
MCA_BTL_FLAGS_RDMA_COMPLETION,
|
|
MCA_BTL_FLAGS_FAILOVER_SUPPORT);
|
|
(void) mca_base_component_var_register(version, "flags", msg,
|
|
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_5,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_flags);
|
|
free(msg);
|
|
|
|
asprintf (&msg, "BTL atomic bit flags (general flags: ADD=%d, AND=%d, OR=%d, XOR=%d",
|
|
MCA_BTL_ATOMIC_SUPPORTS_ADD, MCA_BTL_ATOMIC_SUPPORTS_AND, MCA_BTL_ATOMIC_SUPPORTS_OR,
|
|
MCA_BTL_ATOMIC_SUPPORTS_XOR);
|
|
(void) mca_base_component_var_register(version, "atomic_flags", msg, MCA_BASE_VAR_TYPE_UNSIGNED_INT,
|
|
NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5,
|
|
MCA_BASE_VAR_SCOPE_CONSTANT, &module->btl_atomic_flags);
|
|
free(msg);
|
|
|
|
(void) mca_base_component_var_register(version, "rndv_eager_limit", "Size (in bytes, including header) of \"phase 1\" fragment sent for all large messages (must be >= 0 and <= eager_limit)",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_rndv_eager_limit);
|
|
|
|
(void) mca_base_component_var_register(version, "eager_limit", "Maximum size (in bytes, including header) of \"short\" messages (must be >= 1).",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_eager_limit);
|
|
|
|
if ((module->btl_flags & MCA_BTL_FLAGS_GET) && module->btl_get) {
|
|
if (0 == module->btl_get_limit) {
|
|
module->btl_get_limit = SIZE_MAX;
|
|
}
|
|
|
|
(void) mca_base_component_var_register(version, "get_limit", "Maximum size (in bytes) for btl get",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY, &module->btl_get_limit);
|
|
|
|
/* Allow the user to set the alignment. The BTL should double-check the alignment in its open
|
|
* function. */
|
|
(void) mca_base_component_var_register(version, "get_alignment", "Alignment required for btl get",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_6,
|
|
MCA_BASE_VAR_SCOPE_CONSTANT, &module->btl_get_alignment);
|
|
}
|
|
|
|
if ((module->btl_flags & MCA_BTL_FLAGS_PUT) && module->btl_put) {
|
|
if (0 == module->btl_put_limit) {
|
|
module->btl_put_limit = SIZE_MAX;
|
|
}
|
|
(void) mca_base_component_var_register(version, "put_limit", "Maximum size (in bytes) for btl put",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY, &module->btl_put_limit);
|
|
|
|
/* Allow the user to set the alignment. The BTL should double-check the alignment in its open
|
|
* function. */
|
|
(void) mca_base_component_var_register(version, "put_alignment", "Alignment required for btl put",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_6,
|
|
MCA_BASE_VAR_SCOPE_CONSTANT, &module->btl_put_alignment);
|
|
}
|
|
|
|
|
|
#if OPAL_CUDA_GDR_SUPPORT
|
|
/* If no CUDA RDMA support, zero them out */
|
|
if (!(MCA_BTL_FLAGS_CUDA_GET & module->btl_flags)) {
|
|
module->btl_cuda_eager_limit = 0;
|
|
module->btl_cuda_rdma_limit = SIZE_MAX;
|
|
}
|
|
(void) mca_base_component_var_register(version, "cuda_eager_limit", "Maximum size (in bytes, including header) of \"GPU short\" messages (must be >= 1).",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_5,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_cuda_eager_limit);
|
|
(void) mca_base_component_var_register(version, "cuda_rdma_limit", "Size (in bytes, including header) of GPU buffer when switch to rndv protocol and pipeline.",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_5,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_cuda_rdma_limit);
|
|
#endif /* OPAL_CUDA_GDR_SUPPORT */
|
|
|
|
(void) mca_base_component_var_register(version, "max_send_size", "Maximum size (in bytes) of a single \"phase 2\" fragment of a long message when using the pipeline protocol (must be >= 1)",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_max_send_size);
|
|
|
|
if (NULL != module->btl_put) {
|
|
(void) mca_base_component_var_register(version, "rdma_pipeline_send_length", "Length of the \"phase 2\" portion of a large message (in bytes) when using the pipeline protocol. This part of the message will be split into fragments of size max_send_size and sent using send/receive semantics (must be >= 0; only relevant when the PUT flag is set)",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_rdma_pipeline_send_length);
|
|
|
|
(void) mca_base_component_var_register(version, "rdma_pipeline_frag_size", "Maximum size (in bytes) of a single \"phase 3\" fragment from a long message when using the pipeline protocol. These fragments will be sent using RDMA semantics (must be >= 1; only relevant when the PUT flag is set)",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_rdma_pipeline_frag_size);
|
|
|
|
(void) mca_base_component_var_register(version, "min_rdma_pipeline_size", "Messages smaller than this size (in bytes) will not use the RDMA pipeline protocol. Instead, they will be split into fragments of max_send_size and sent using send/receive semantics (must be >=0, and is automatically adjusted up to at least (eager_limit+btl_rdma_pipeline_send_length); only relevant when the PUT flag is set)",
|
|
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
|
|
OPAL_INFO_LVL_4,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_min_rdma_pipeline_size);
|
|
|
|
(void) mca_base_component_var_register(version, "latency", "Approximate latency of interconnect (0 = auto-detect value at run-time [not supported in all BTL modules], >= 1 = latency in microseconds)",
|
|
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_5,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_latency);
|
|
(void) mca_base_component_var_register(version, "bandwidth", "Approximate maximum bandwidth of interconnect (0 = auto-detect value at run-time [not supported in all BTL modules], >= 1 = bandwidth in Mbps)",
|
|
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_5,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&module->btl_bandwidth);
|
|
}
|
|
|
|
return mca_btl_base_param_verify(module);
|
|
}
|
|
|
|
/* Verify btl parameters make sense */
|
|
int mca_btl_base_param_verify(mca_btl_base_module_t *module)
|
|
{
|
|
if (module->btl_min_rdma_pipeline_size <
|
|
(module->btl_eager_limit + module->btl_rdma_pipeline_send_length)) {
|
|
module->btl_min_rdma_pipeline_size =
|
|
module->btl_eager_limit + module->btl_rdma_pipeline_send_length;
|
|
}
|
|
|
|
if (NULL == module->btl_put) {
|
|
module->btl_flags &= ~MCA_BTL_FLAGS_PUT;
|
|
}
|
|
|
|
if (NULL == module->btl_get) {
|
|
module->btl_flags &= ~MCA_BTL_FLAGS_GET;
|
|
}
|
|
|
|
if (0 == module->btl_atomic_flags) {
|
|
module->btl_flags &= ~MCA_BTL_FLAGS_ATOMIC_OPS;
|
|
}
|
|
|
|
if (0 == module->btl_get_limit) {
|
|
module->btl_get_limit = SIZE_MAX;
|
|
}
|
|
|
|
if (0 == module->btl_put_limit) {
|
|
module->btl_put_limit = SIZE_MAX;
|
|
}
|
|
|
|
return OPAL_SUCCESS;
|
|
}
|