64f61ebd07
This commit has the same commit message as r18450, but without the extra bonus memory corruption that was introduced. This commit was SVN r18467. The following SVN revision numbers were found above: r18450 --> open-mpi/ompi@5295902ebe The following Trac tickets were found above: Ticket 1285 --> https://svn.open-mpi.org/trac/ompi/ticket/1285
535 строки
22 KiB
C
535 строки
22 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
|
|
* Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
|
|
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* Copyright (c) 2006-2007 Voltaire All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "ompi_config.h"
|
|
|
|
#include <string.h>
|
|
|
|
#include "opal/mca/installdirs/installdirs.h"
|
|
#include "orte/util/output.h"
|
|
#include "opal/util/argv.h"
|
|
#include "opal/mca/base/mca_base_param.h"
|
|
#include "btl_openib.h"
|
|
#include "btl_openib_mca.h"
|
|
#include "connect/base.h"
|
|
|
|
/*
|
|
* Local flags
|
|
*/
|
|
enum {
|
|
REGINT_NEG_ONE_OK = 0x01,
|
|
REGINT_GE_ZERO = 0x02,
|
|
REGINT_GE_ONE = 0x04,
|
|
REGINT_NONZERO = 0x08,
|
|
|
|
REGINT_MAX = 0x88
|
|
};
|
|
|
|
|
|
enum {
|
|
REGSTR_EMPTY_OK = 0x01,
|
|
|
|
REGSTR_MAX = 0x88
|
|
};
|
|
|
|
|
|
/*
|
|
* utility routine for string parameter registration
|
|
*/
|
|
static inline int reg_string(const char* param_name, const char* param_desc,
|
|
const char* default_value, char **out_value,
|
|
int flags)
|
|
{
|
|
char *value;
|
|
mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version,
|
|
param_name, param_desc, false, false,
|
|
default_value, &value);
|
|
|
|
if (0 != (flags & REGSTR_EMPTY_OK) && 0 == strlen(value)) {
|
|
orte_output(0, "Bad parameter value for parameter \"%s\"",
|
|
param_name);
|
|
return OMPI_ERR_BAD_PARAM;
|
|
}
|
|
*out_value = value;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
/*
|
|
* utility routine for integer parameter registration
|
|
*/
|
|
static inline int reg_int(const char* param_name, const char* param_desc,
|
|
int default_value, int *out_value, int flags)
|
|
{
|
|
int value;
|
|
mca_base_param_reg_int(&mca_btl_openib_component.super.btl_version,
|
|
param_name, param_desc, false, false,
|
|
default_value, &value);
|
|
|
|
if (0 != (flags & REGINT_NEG_ONE_OK) && -1 == value) {
|
|
*out_value = value;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
if ((0 != (flags & REGINT_GE_ZERO) && value < 0) ||
|
|
(0 != (flags & REGINT_GE_ONE) && value < 1) ||
|
|
(0 != (flags & REGINT_NONZERO) && 0 == value)) {
|
|
orte_output(0, "Bad parameter value for parameter \"%s\"",
|
|
param_name);
|
|
return OMPI_ERR_BAD_PARAM;
|
|
}
|
|
*out_value = value;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
/*
|
|
* Register and check all MCA parameters
|
|
*/
|
|
int btl_openib_register_mca_params(void)
|
|
{
|
|
char default_qps[100];
|
|
uint32_t mid_qp_size;
|
|
int i;
|
|
char *msg, *str;
|
|
int ival, ival2, ret, tmp;
|
|
|
|
ret = OMPI_SUCCESS;
|
|
#define CHECK(expr) do {\
|
|
tmp = (expr); \
|
|
if (OMPI_SUCCESS != tmp) ret = tmp; \
|
|
} while (0)
|
|
|
|
/* register IB component parameters */
|
|
CHECK(reg_int("verbose",
|
|
"Output some verbose OpenIB BTL information "
|
|
"(0 = no output, nonzero = output)", 0, &ival, 0));
|
|
mca_btl_openib_component.verbose = (0 != ival);
|
|
CHECK(reg_int("warn_no_hca_params_found",
|
|
"Warn when no HCA-specific parameters are found in the INI file specified by the btl_openib_hca_param_files MCA parameter (0 = do not warn; any other value = warn)",
|
|
1, &ival, 0));
|
|
mca_btl_openib_component.warn_no_hca_params_found = (0 != ival);
|
|
CHECK(reg_int("warn_default_gid_prefix",
|
|
"Warn when there is more than one active ports and at least one of them connected to the network with only default GID prefix configured (0 = do not warn; any other value = warn)",
|
|
1, &ival, 0));
|
|
mca_btl_openib_component.warn_default_gid_prefix = (0 != ival);
|
|
CHECK(reg_int("warn_nonexistent_if",
|
|
"Warn if non-existent HCAs and/or ports are specified in the btl_openib_if_[in|ex]clude MCA parameters (0 = do not warn; any other value = warn)",
|
|
1, &ival, 0));
|
|
mca_btl_openib_component.warn_nonexistent_if = (0 != ival);
|
|
|
|
#ifdef HAVE_IBV_FORK_INIT
|
|
ival2 = -1;
|
|
#else
|
|
ival2 = 0;
|
|
#endif
|
|
CHECK(reg_int("want_fork_support",
|
|
"Whether fork support is desired or not "
|
|
"(negative = try to enable fork support, but continue even if it is not available, 0 = do not enable fork support, positive = try to enable fork support and fail if it is not available)",
|
|
ival2, &ival, 0));
|
|
#ifdef HAVE_IBV_FORK_INIT
|
|
mca_btl_openib_component.want_fork_support = ival;
|
|
#else
|
|
if (0 != ival) {
|
|
orte_show_help("help-mpi-btl-openib.txt",
|
|
"ibv_fork requested but not supported", true,
|
|
orte_process_info.nodename);
|
|
return OMPI_ERROR;
|
|
}
|
|
#endif
|
|
|
|
asprintf(&str, "%s/mca-btl-openib-hca-params.ini",
|
|
opal_install_dirs.pkgdatadir);
|
|
if (NULL == str) {
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
}
|
|
#ifdef HAVE_IBV_FORK_INIT
|
|
ival2 = -1;
|
|
#else
|
|
ival2 = 0;
|
|
#endif
|
|
CHECK(reg_int("want_fork_support",
|
|
"Whether fork support is desired or not "
|
|
"(negative = try to enable fork support, but continue even if it is not available, 0 = do not enable fork support, positive = try to enable fork support and fail if it is not available)",
|
|
ival2, &ival, 0));
|
|
#ifdef HAVE_IBV_FORK_INIT
|
|
mca_btl_openib_component.want_fork_support = ival;
|
|
#else
|
|
if (0 != ival) {
|
|
orte_show_help("help-mpi-btl-openib.txt",
|
|
"ibv_fork requested but not supported", true,
|
|
orte_process_info.nodename);
|
|
return OMPI_ERROR;
|
|
}
|
|
#endif
|
|
|
|
CHECK(reg_string("hca_param_files",
|
|
"Colon-delimited list of INI-style files that contain HCA vendor/part-specific parameters",
|
|
str, &mca_btl_openib_component.hca_params_file_names, 0));
|
|
free(str);
|
|
|
|
CHECK(reg_int("max_btls",
|
|
"Maximum number of HCA ports to use "
|
|
"(-1 = use all available, otherwise must be >= 1)",
|
|
-1, &mca_btl_openib_component.ib_max_btls,
|
|
REGINT_NEG_ONE_OK | REGINT_GE_ONE));
|
|
CHECK(reg_int("free_list_num",
|
|
"Intial size of free lists (must be >= 1)",
|
|
8, &mca_btl_openib_component.ib_free_list_num,
|
|
REGINT_GE_ONE));
|
|
CHECK(reg_int("free_list_max",
|
|
"Maximum size of free lists "
|
|
"(-1 = infinite, otherwise must be >= 0)",
|
|
-1, &mca_btl_openib_component.ib_free_list_max,
|
|
REGINT_NEG_ONE_OK | REGINT_GE_ONE));
|
|
CHECK(reg_int("free_list_inc",
|
|
"Increment size of free lists (must be >= 1)",
|
|
32, &mca_btl_openib_component.ib_free_list_inc,
|
|
REGINT_GE_ONE));
|
|
CHECK(reg_string("mpool",
|
|
"Name of the memory pool to be used (it is unlikely that you will ever want to change this",
|
|
"rdma", &mca_btl_openib_component.ib_mpool_name,
|
|
0));
|
|
CHECK(reg_int("reg_mru_len",
|
|
"Length of the registration cache most recently used list "
|
|
"(must be >= 1)",
|
|
16, (int*) &mca_btl_openib_component.reg_mru_len,
|
|
REGINT_GE_ONE));
|
|
|
|
CHECK(reg_int("ib_cq_size", "Size of the IB completion "
|
|
"queue (will automatically be set to a minimum of "
|
|
"(2 * number_of_peers * btl_openib_rd_num))",
|
|
1000, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.ib_cq_size[BTL_OPENIB_LP_CQ] =
|
|
mca_btl_openib_component.ib_cq_size[BTL_OPENIB_HP_CQ] = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_sg_list_size", "Size of IB segment list "
|
|
"(must be >= 1)",
|
|
4, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.ib_sg_list_size = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_pkey_ix", "InfiniBand pkey index "
|
|
"(must be >= 0)",
|
|
0, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.ib_pkey_ix = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_pkey_val", "InfiniBand pkey value"
|
|
"(must be > 0 and < 0xffff)",
|
|
0, &ival, REGINT_GE_ZERO));
|
|
if (ival > 0xffff) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "invalid value for btl_openib_ib_pkey_val",
|
|
"btl_openib_ib_pkey_val ignored");
|
|
} else {
|
|
mca_btl_openib_component.ib_pkey_val = (uint32_t) ival;
|
|
}
|
|
|
|
CHECK(reg_int("ib_psn", "InfiniBand packet sequence starting number "
|
|
"(must be >= 0)",
|
|
0, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.ib_psn = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_qp_ous_rd_atom", "InfiniBand outstanding atomic reads "
|
|
"(must be >= 0)",
|
|
4, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.ib_qp_ous_rd_atom = (uint32_t) ival;
|
|
|
|
asprintf(&msg, "IB MTU, in bytes (if not specified in INI files). Valid values are: %d=256 bytes, %d=512 bytes, %d=1024 bytes, %d=2048 bytes, %d=4096 bytes",
|
|
IBV_MTU_256,
|
|
IBV_MTU_512,
|
|
IBV_MTU_1024,
|
|
IBV_MTU_2048,
|
|
IBV_MTU_4096);
|
|
if (NULL == msg) {
|
|
/* Don't try to recover from this */
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
}
|
|
CHECK(reg_int("ib_mtu", msg, IBV_MTU_1024, &ival, 0));
|
|
free(msg);
|
|
if (ival < IBV_MTU_1024 || ival > IBV_MTU_4096) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "invalid value for btl_openib_ib_mtu",
|
|
"btl_openib_ib_mtu reset to 1024");
|
|
mca_btl_openib_component.ib_mtu = IBV_MTU_1024;
|
|
} else {
|
|
mca_btl_openib_component.ib_mtu = (uint32_t) ival;
|
|
}
|
|
|
|
CHECK(reg_int("ib_min_rnr_timer", "InfiniBand minimum "
|
|
"\"receiver not ready\" timer, in seconds "
|
|
"(must be >= 0 and <= 31)",
|
|
5, &ival, 0));
|
|
if (ival > 31) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_min_rnr_timer > 31",
|
|
"btl_openib_ib_min_rnr_timer reset to 31");
|
|
ival = 31;
|
|
} else if (ival < 0){
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_min_rnr_timer < 0",
|
|
"btl_openib_ib_min_rnr_timer reset to 0");
|
|
ival = 0;
|
|
}
|
|
mca_btl_openib_component.ib_min_rnr_timer = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_timeout", "InfiniBand transmit timeout, plugged into formula: 4.096 microseconds * (2^btl_openib_ib_timeout)"
|
|
"(must be >= 0 and <= 31)",
|
|
10, &ival, 0));
|
|
if (ival > 31) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_timeout > 31",
|
|
"btl_openib_ib_timeout reset to 31");
|
|
ival = 31;
|
|
} else if (ival < 0) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_timeout < 0",
|
|
"btl_openib_ib_timeout reset to 0");
|
|
ival = 0;
|
|
}
|
|
mca_btl_openib_component.ib_timeout = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_retry_count", "InfiniBand transmit retry count "
|
|
"(must be >= 0 and <= 7)",
|
|
7, &ival, 0));
|
|
if (ival > 7) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_retry_count > 7",
|
|
"btl_openib_ib_retry_count reset to 7");
|
|
ival = 7;
|
|
} else if (ival < 0) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_retry_count < 0",
|
|
"btl_openib_ib_retry_count reset to 0");
|
|
ival = 0;
|
|
}
|
|
mca_btl_openib_component.ib_retry_count = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_rnr_retry", "InfiniBand \"receiver not ready\" "
|
|
"retry count "
|
|
"(must be >= 0 and <= 7)",
|
|
7, &ival, 0));
|
|
if (ival > 7) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_rnr_retry > 7",
|
|
"btl_openib_ib_rnr_retry reset to 7");
|
|
ival = 7;
|
|
} else if (ival < 0) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_rnr_retry < 0",
|
|
"btl_openib_ib_rnr_retry reset to 0");
|
|
ival = 0;
|
|
}
|
|
mca_btl_openib_component.ib_rnr_retry = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_max_rdma_dst_ops", "InfiniBand maximum pending RDMA "
|
|
"destination operations "
|
|
"(must be >= 0)",
|
|
4, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.ib_max_rdma_dst_ops = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("ib_service_level", "InfiniBand service level "
|
|
"(must be >= 0 and <= 15)",
|
|
0, &ival, 0));
|
|
if (ival > 15) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_service_level > 15",
|
|
"btl_openib_ib_service_level reset to 15");
|
|
ival = 15;
|
|
} else if (ival < 0) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
|
|
true, "btl_openib_ib_service_level < 0",
|
|
"btl_openib_ib_service_level reset to 0");
|
|
ival = 0;
|
|
}
|
|
mca_btl_openib_component.ib_service_level = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("use_eager_rdma", "Use RDMA for eager messages",
|
|
1, &ival, 0));
|
|
mca_btl_openib_component.use_eager_rdma = (uint32_t) (ival != 0);
|
|
#if OMPI_ENABLE_PROGRESS_THREADS == 1
|
|
/* Fast rdma path isn't supported by PROGRESS_THREAD */
|
|
mca_btl_openib_component.use_eager_rdma = 0;
|
|
#endif
|
|
|
|
CHECK(reg_int("eager_rdma_threshold",
|
|
"Use RDMA for short messages after this number of "
|
|
"messages are received from a given peer "
|
|
"(must be >= 1)",
|
|
16, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.eager_rdma_threshold = (int32_t) ival;
|
|
|
|
CHECK(reg_int("max_eager_rdma", "Maximum number of peers allowed to use "
|
|
"RDMA for short messages (RDMA is used for all long "
|
|
"messages, except if explicitly disabled, such as "
|
|
"with the \"dr\" pml) "
|
|
"(must be >= 0)",
|
|
16, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.max_eager_rdma = (int32_t) ival;
|
|
|
|
CHECK(reg_int("eager_rdma_num", "Number of RDMA buffers to allocate "
|
|
"for small messages"
|
|
"(must be >= 1)",
|
|
16, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.eager_rdma_num = (int32_t) (ival + 1);
|
|
|
|
CHECK(reg_int("btls_per_lid", "Number of BTLs to create for each "
|
|
"InfiniBand LID "
|
|
"(must be >= 1)",
|
|
1, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.btls_per_lid = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("max_lmc", "Maximum number of LIDs to use for each HCA port "
|
|
"(must be >= 0, where 0 = use all available)",
|
|
0, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.max_lmc = (uint32_t) ival;
|
|
|
|
#if OMPI_HAVE_THREADS
|
|
CHECK(reg_int("enable_apm_over_lmc", "Maximum number of alterative paths for each HCA port "
|
|
"(must be >= -1, where 0 = disable apm, -1 = all availible alternative paths )",
|
|
0, &ival, REGINT_NEG_ONE_OK|REGINT_GE_ZERO));
|
|
mca_btl_openib_component.apm_lmc = (uint32_t) ival;
|
|
CHECK(reg_int("enable_apm_over_ports", "Enable alterative path migration over different ports of the same HCA"
|
|
"(must be >= 0, where 0 = disable apm over ports , 1 = enable apm over ports of the same hca )",
|
|
0, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.apm_ports = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("enable_apm_over_lmc", "Maximum number of alterative paths for each HCA port "
|
|
"(must be >= -1, where 0 = disable apm, -1 = all availible alternative paths )",
|
|
0, &ival, REGINT_NEG_ONE_OK|REGINT_GE_ZERO));
|
|
mca_btl_openib_component.apm_lmc = (uint32_t) ival;
|
|
CHECK(reg_int("enable_apm_over_ports", "Enable alterative path migration over different ports of the same HCA"
|
|
"(must be >= 0, where 0 = disable apm over ports , 1 = enable apm over ports of the same hca )",
|
|
0, &ival, REGINT_GE_ZERO));
|
|
mca_btl_openib_component.apm_ports = (uint32_t) ival;
|
|
|
|
CHECK(reg_int("use_async_event_thread",
|
|
"If nonzero, use the thread that will handle InfiniBand asyncihronous events ",
|
|
1, &ival, 0));
|
|
mca_btl_openib_component.use_async_event_thread = (0 != ival);
|
|
#endif
|
|
|
|
CHECK(reg_int("buffer_alignment",
|
|
"Prefered communication buffer alignment, in bytes "
|
|
"(must be > 0 and power of two)",
|
|
64, &ival, REGINT_GE_ZERO));
|
|
if(ival <= 1 || (ival & (ival - 1))) {
|
|
orte_show_help("help-mpi-btl-openib.txt", "wrong buffer alignment",
|
|
true, ival, orte_process_info.nodename, 64);
|
|
mca_btl_openib_component.buffer_alignment = 64;
|
|
} else {
|
|
mca_btl_openib_component.buffer_alignment = (uint32_t) ival;
|
|
}
|
|
|
|
CHECK(reg_int("use_message_coalescing",
|
|
"Use message coalescing", 1, &ival, 0));
|
|
mca_btl_openib_component.use_message_coalescing = (0 != ival);
|
|
|
|
CHECK(reg_int("cq_poll_ratio",
|
|
"how often poll high priority CQ versus low priority CQ",
|
|
100, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.cq_poll_ratio = (uint32_t)ival;
|
|
CHECK(reg_int("eager_rdma_poll_ratio",
|
|
"how often poll eager RDMA channel versus CQ",
|
|
100, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.eager_rdma_poll_ratio = (uint32_t)ival;
|
|
CHECK(reg_int("hp_cq_poll_per_progress",
|
|
"max number of completion events to process for each call "
|
|
"of BTL progress engine",
|
|
10, &ival, REGINT_GE_ONE));
|
|
mca_btl_openib_component.cq_poll_progress = (uint32_t)ival;
|
|
|
|
/* Info only */
|
|
mca_base_param_reg_int(&mca_btl_openib_component.super.btl_version,
|
|
"have_fork_support",
|
|
"Whether the OpenFabrics stack supports applications that invoke the \"fork()\" system call or not (0 = no, 1 = yes). Note that this value does NOT indicate whether the system being run on supports \"fork()\" with OpenFabrics applications or not.",
|
|
false, true,
|
|
#ifdef HAVE_IBV_FORK_INIT
|
|
1,
|
|
#else
|
|
0,
|
|
#endif
|
|
NULL);
|
|
|
|
mca_btl_openib_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
|
|
|
|
mca_btl_openib_module.super.btl_eager_limit = 12 * 1024;
|
|
mca_btl_openib_module.super.btl_rndv_eager_limit = 12 * 1024;
|
|
mca_btl_openib_module.super.btl_max_send_size = 64 * 1024;
|
|
mca_btl_openib_module.super.btl_rdma_pipeline_send_length = 1024 * 1024;
|
|
mca_btl_openib_module.super.btl_rdma_pipeline_frag_size = 1024 * 1024;
|
|
mca_btl_openib_module.super.btl_min_rdma_pipeline_size = 256 * 1024;
|
|
mca_btl_openib_module.super.btl_flags = MCA_BTL_FLAGS_RDMA |
|
|
MCA_BTL_FLAGS_NEED_ACK | MCA_BTL_FLAGS_NEED_CSUM | MCA_BTL_FLAGS_HETEROGENEOUS_RDMA;
|
|
mca_btl_openib_module.super.btl_bandwidth = 800;
|
|
mca_btl_openib_module.super.btl_latency = 10;
|
|
CHECK(mca_btl_base_param_register(
|
|
&mca_btl_openib_component.super.btl_version,
|
|
&mca_btl_openib_module.super));
|
|
|
|
/* setup all the qp stuff */
|
|
mid_qp_size = mca_btl_openib_module.super.btl_eager_limit / 4;
|
|
/* round mid_qp_size to smallest power of two */
|
|
for(i = 31; i > 0; i--) {
|
|
if(!(mid_qp_size & (1<<i))) {
|
|
continue;
|
|
}
|
|
mid_qp_size = (1<<i);
|
|
break;
|
|
}
|
|
|
|
if(mid_qp_size <= 128) {
|
|
mid_qp_size = 1024;
|
|
}
|
|
|
|
snprintf(default_qps, 100,
|
|
"P,128,256,192,128:S,%u,256,128,32:S,%u,256,128,32:S,%u,256,128,32",
|
|
mid_qp_size,
|
|
(uint32_t)mca_btl_openib_module.super.btl_eager_limit,
|
|
(uint32_t)mca_btl_openib_module.super.btl_max_send_size);
|
|
CHECK(reg_string("receive_queues",
|
|
"Colon-delimited, comma delimited list of receive queues: P,4096,8,6,4:P,32768,8,6,4",
|
|
default_qps, &mca_btl_openib_component.receive_queues,
|
|
0));
|
|
mca_btl_openib_component.receive_queues_source =
|
|
(0 == strcmp(default_qps,
|
|
mca_btl_openib_component.receive_queues)) ?
|
|
BTL_OPENIB_RQ_SOURCE_DEFAULT : BTL_OPENIB_RQ_SOURCE_MCA;
|
|
|
|
CHECK(reg_string("if_include",
|
|
"Comma-delimited list of HCAs/ports to be used (e.g. \"mthca0,mthca1:2\"; empty value means to use all ports found). Mutually exclusive with btl_openib_if_exclude.",
|
|
NULL, &mca_btl_openib_component.if_include,
|
|
0));
|
|
|
|
CHECK(reg_string("if_exclude",
|
|
"Comma-delimited list of HCAs/ports to be excluded (empty value means to not exclude any ports). Mutually exclusive with btl_openib_if_include.",
|
|
NULL, &mca_btl_openib_component.if_exclude,
|
|
0));
|
|
|
|
/* Register any MCA params for the connect pseudo-components */
|
|
if (OMPI_SUCCESS == ret) {
|
|
ret = ompi_btl_openib_connect_base_register();
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|