eb977de5e9
Switch to using the query/priority method for selecting MTLs. This switch was motivated by the fact that now on some platforms, its possible for multiple MTLs to be initializable, but only one MTL should be selected. In addition, there is a complication with the PSM and IFO (with PSM provider) MTLs owing to the fact that they cannot both intialize the underlying PSM context, i.e. only one call to psm_init is allowed per process. The mxm component has not been compiled as the author doesn't currently have access to a system with a recent enough mxm installed to allow for a compile. The portals4, ofi, and psm components have been checked for compilation. The ofi and psm components have been checked for runtime correctness on a intel/qlogic system with up to date PSM installed.
308 строки
10 KiB
C
308 строки
10 KiB
C
/*
|
|
* Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "ompi_config.h"
|
|
|
|
#include "opal/util/output.h"
|
|
#include "opal/util/show_help.h"
|
|
#include "ompi/proc/proc.h"
|
|
#include "opal/memoryhooks/memory.h"
|
|
#include "ompi/runtime/mpiruntime.h"
|
|
|
|
#include "mtl_mxm.h"
|
|
#include "mtl_mxm_types.h"
|
|
#include "mtl_mxm_request.h"
|
|
|
|
#include <sys/types.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
static int ompi_mtl_mxm_component_open(void);
|
|
static int ompi_mtl_mxm_component_query(mca_base_module_t **module, int *priority);
|
|
static int ompi_mtl_mxm_component_close(void);
|
|
static int ompi_mtl_mxm_component_register(void);
|
|
|
|
static int param_priority;
|
|
|
|
int mca_mtl_mxm_output = -1;
|
|
|
|
|
|
static mca_mtl_base_module_t
|
|
* ompi_mtl_mxm_component_init(bool enable_progress_threads,
|
|
bool enable_mpi_threads);
|
|
|
|
mca_mtl_mxm_component_t mca_mtl_mxm_component = {
|
|
{
|
|
/*
|
|
* First, the mca_base_component_t struct containing meta
|
|
* information about the component itself
|
|
*/
|
|
{
|
|
MCA_MTL_BASE_VERSION_2_0_0,
|
|
"mxm", /* MCA component name */
|
|
OMPI_MAJOR_VERSION, /* MCA component major version */
|
|
OMPI_MINOR_VERSION, /* MCA component minor version */
|
|
OMPI_RELEASE_VERSION, /* MCA component release version */
|
|
ompi_mtl_mxm_component_open, /* component open */
|
|
ompi_mtl_mxm_component_close, /* component close */
|
|
ompi_mtl_mxm_component_query, /* component query */
|
|
ompi_mtl_mxm_component_register
|
|
},
|
|
{
|
|
/* The component is not checkpoint ready */
|
|
MCA_BASE_METADATA_PARAM_NONE
|
|
},
|
|
ompi_mtl_mxm_component_init /* component init */
|
|
}
|
|
};
|
|
|
|
static int ompi_mtl_mxm_component_register(void)
|
|
{
|
|
mca_base_component_t*c;
|
|
|
|
#if MXM_API < MXM_VERSION(3,0)
|
|
unsigned long cur_ver;
|
|
long major, minor;
|
|
char* runtime_version;
|
|
#endif
|
|
|
|
c = &mca_mtl_mxm_component.super.mtl_version;
|
|
|
|
ompi_mtl_mxm.verbose = 0;
|
|
(void) mca_base_component_var_register(c, "verbose",
|
|
"Verbose level of the MXM component",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_LOCAL,
|
|
&ompi_mtl_mxm.verbose);
|
|
|
|
#if MXM_API > MXM_VERSION(2,0)
|
|
ompi_mtl_mxm.mxm_np = 0;
|
|
#else
|
|
ompi_mtl_mxm.mxm_np = 128;
|
|
#endif
|
|
(void) mca_base_component_var_register(c, "np",
|
|
"[integer] Minimal number of MPI processes in a single job "
|
|
"required to activate the MXM transport",
|
|
MCA_BASE_VAR_TYPE_INT, NULL,0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&ompi_mtl_mxm.mxm_np);
|
|
|
|
ompi_mtl_mxm.compiletime_version = MXM_VERNO_STRING;
|
|
(void) mca_base_component_var_register(c,
|
|
MCA_COMPILETIME_VER,
|
|
"Version of the libmxm library with which Open MPI was compiled",
|
|
MCA_BASE_VAR_TYPE_VERSION_STRING,
|
|
NULL, 0, 0,
|
|
OPAL_INFO_LVL_3,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&ompi_mtl_mxm.compiletime_version);
|
|
|
|
#if MXM_API >= MXM_VERSION(3,0)
|
|
ompi_mtl_mxm.runtime_version = (char *)mxm_get_version_string();
|
|
#else
|
|
cur_ver = mxm_get_version();
|
|
major = (cur_ver >> MXM_MAJOR_BIT) & 0xff;
|
|
minor = (cur_ver >> MXM_MINOR_BIT) & 0xff;
|
|
asprintf(&runtime_version, "%ld.%ld", major, minor);
|
|
ompi_mtl_mxm.runtime_version = runtime_version;
|
|
#endif
|
|
|
|
(void) mca_base_component_var_register(c,
|
|
MCA_RUNTIME_VER,
|
|
"Version of the libmxm library with which Open MPI is running",
|
|
MCA_BASE_VAR_TYPE_VERSION_STRING,
|
|
NULL, 0, 0,
|
|
OPAL_INFO_LVL_3,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&ompi_mtl_mxm.runtime_version);
|
|
|
|
#if MXM_API < MXM_VERSION(3,0)
|
|
free(runtime_version);
|
|
#endif
|
|
|
|
param_priority = 100;
|
|
(void) mca_base_component_var_register (c,
|
|
"priority", "Priority of the MXM MTL component",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
¶m_priority);
|
|
|
|
|
|
#if MXM_API >= MXM_VERSION(3,1)
|
|
{
|
|
unsigned long cur_ver = mxm_get_version();
|
|
|
|
ompi_mtl_mxm.bulk_connect = 0;
|
|
|
|
if (cur_ver < MXM_VERSION(3,2)) {
|
|
ompi_mtl_mxm.bulk_disconnect = 0;
|
|
} else {
|
|
ompi_mtl_mxm.bulk_disconnect = 1;
|
|
}
|
|
|
|
(void) mca_base_component_var_register(c, "bulk_connect",
|
|
"[integer] use bulk connect",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&ompi_mtl_mxm.bulk_connect);
|
|
|
|
(void) mca_base_component_var_register(c, "bulk_disconnect",
|
|
"[integer] use bulk disconnect",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&ompi_mtl_mxm.bulk_disconnect);
|
|
|
|
if (cur_ver < MXM_VERSION(3,2) &&
|
|
(ompi_mtl_mxm.bulk_connect || ompi_mtl_mxm.bulk_disconnect)) {
|
|
ompi_mtl_mxm.bulk_connect = 0;
|
|
ompi_mtl_mxm.bulk_disconnect = 0;
|
|
|
|
MXM_VERBOSE(1, "WARNING: OMPI runs with %s version of MXM that is less than 3.2, "
|
|
"so bulk connect/disconnect cannot work properly and will be turn off.",
|
|
ompi_mtl_mxm.runtime_version);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static int ompi_mtl_mxm_component_open(void)
|
|
{
|
|
mxm_error_t err;
|
|
unsigned long cur_ver;
|
|
int rc;
|
|
|
|
mca_mtl_mxm_output = opal_output_open(NULL);
|
|
opal_output_set_verbosity(mca_mtl_mxm_output, ompi_mtl_mxm.verbose);
|
|
cur_ver = mxm_get_version();
|
|
if (cur_ver != MXM_API) {
|
|
MXM_VERBOSE(1,
|
|
"WARNING: OMPI was compiled with MXM version %d.%d but version %ld.%ld detected.",
|
|
MXM_VERNO_MAJOR,
|
|
MXM_VERNO_MINOR,
|
|
(cur_ver >> MXM_MAJOR_BIT) & 0xff,
|
|
(cur_ver >> MXM_MINOR_BIT) & 0xff);
|
|
}
|
|
|
|
#if MXM_API >= MXM_VERSION(2,0)
|
|
/* Register memory hooks */
|
|
if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
|
|
((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
|
|
opal_mem_hooks_support_level()))
|
|
{
|
|
setenv("MXM_MPI_MEM_ON_DEMAND_MAP", "y", 0);
|
|
MXM_VERBOSE(1, "Enabling on-demand memory mapping");
|
|
ompi_mtl_mxm.using_mem_hooks = 1;
|
|
} else {
|
|
MXM_VERBOSE(1, "Disabling on-demand memory mapping");
|
|
ompi_mtl_mxm.using_mem_hooks = 0;
|
|
}
|
|
setenv("MXM_MPI_SINGLE_THREAD", ompi_mpi_thread_multiple ? "n" : "y" , 0);
|
|
#endif
|
|
|
|
#if MXM_API >= MXM_VERSION(2,1)
|
|
if (MXM_OK != mxm_config_read_opts(&ompi_mtl_mxm.mxm_ctx_opts,
|
|
&ompi_mtl_mxm.mxm_ep_opts,
|
|
"MPI", NULL, 0))
|
|
#else
|
|
if ((MXM_OK != mxm_config_read_context_opts(&ompi_mtl_mxm.mxm_ctx_opts)) ||
|
|
(MXM_OK != mxm_config_read_ep_opts(&ompi_mtl_mxm.mxm_ep_opts)))
|
|
#endif
|
|
{
|
|
MXM_ERROR("Failed to parse MXM configuration");
|
|
return OPAL_ERR_BAD_PARAM;
|
|
}
|
|
|
|
err = mxm_init(ompi_mtl_mxm.mxm_ctx_opts, &ompi_mtl_mxm.mxm_context);
|
|
MXM_VERBOSE(1, "mxm component open");
|
|
|
|
if (MXM_OK != err) {
|
|
if (MXM_ERR_NO_DEVICE == err) {
|
|
MXM_VERBOSE(1, "No supported device found, disqualifying mxm");
|
|
} else {
|
|
opal_show_help("help-mtl-mxm.txt", "mxm init", true,
|
|
mxm_error_string(err));
|
|
}
|
|
return OPAL_ERR_NOT_AVAILABLE;
|
|
}
|
|
|
|
OBJ_CONSTRUCT(&mca_mtl_mxm_component.mxm_messages, ompi_free_list_t);
|
|
rc = ompi_free_list_init_new(&mca_mtl_mxm_component.mxm_messages,
|
|
sizeof(ompi_mtl_mxm_message_t),
|
|
opal_cache_line_size,
|
|
OBJ_CLASS(ompi_mtl_mxm_message_t),
|
|
0, opal_cache_line_size,
|
|
32 /* free list num */,
|
|
-1 /* free list max */,
|
|
32 /* free list inc */,
|
|
NULL);
|
|
if (OMPI_SUCCESS != rc) {
|
|
opal_show_help("help-mtl-mxm.txt", "mxm init", true,
|
|
mxm_error_string(err));
|
|
return OPAL_ERR_NOT_AVAILABLE;
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static int ompi_mtl_mxm_component_query(mca_base_module_t **module, int *priority)
|
|
{
|
|
|
|
/*
|
|
* if we get here it means that mxm is available so give high priority
|
|
*/
|
|
|
|
*priority = param_priority;
|
|
*module = &ompi_mtl_mxm.super;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static int ompi_mtl_mxm_component_close(void)
|
|
{
|
|
if (ompi_mtl_mxm.mxm_context != NULL) {
|
|
mxm_cleanup(ompi_mtl_mxm.mxm_context);
|
|
ompi_mtl_mxm.mxm_context = NULL;
|
|
OBJ_DESTRUCT(&mca_mtl_mxm_component.mxm_messages);
|
|
#if MXM_API >= MXM_VERSION(2,0)
|
|
mxm_config_free_ep_opts(ompi_mtl_mxm.mxm_ep_opts);
|
|
mxm_config_free_context_opts(ompi_mtl_mxm.mxm_ctx_opts);
|
|
#else
|
|
mxm_config_free(ompi_mtl_mxm.mxm_ep_opts);
|
|
mxm_config_free(ompi_mtl_mxm.mxm_ctx_opts);
|
|
#endif
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static mca_mtl_base_module_t*
|
|
ompi_mtl_mxm_component_init(bool enable_progress_threads,
|
|
bool enable_mpi_threads)
|
|
{
|
|
int rc;
|
|
|
|
rc = ompi_mtl_mxm_module_init();
|
|
if (OMPI_SUCCESS != rc) {
|
|
return NULL;
|
|
}
|
|
|
|
/* Calculate MTL constraints according to MXM types */
|
|
ompi_mtl_mxm.super.mtl_max_contextid = 1UL << (sizeof(mxm_ctxid_t) * 8);
|
|
ompi_mtl_mxm.super.mtl_max_tag = 1UL << (sizeof(mxm_tag_t) * 8 - 2);
|
|
ompi_mtl_mxm.super.mtl_request_size =
|
|
sizeof(mca_mtl_mxm_request_t) - sizeof(struct mca_mtl_request_t);
|
|
return &ompi_mtl_mxm.super;
|
|
}
|