/* * Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "ompi_config.h" #include "opal/util/output.h" #include "opal/util/show_help.h" #include "ompi/proc/proc.h" #include "opal/memoryhooks/memory.h" #include "ompi/runtime/mpiruntime.h" #include "mtl_mxm.h" #include "mtl_mxm_types.h" #include "mtl_mxm_request.h" #include #include #include static int ompi_mtl_mxm_component_open(void); static int ompi_mtl_mxm_component_close(void); static int ompi_mtl_mxm_component_register(void); int mca_mtl_mxm_output = -1; static mca_mtl_base_module_t * ompi_mtl_mxm_component_init(bool enable_progress_threads, bool enable_mpi_threads); mca_mtl_mxm_component_t mca_mtl_mxm_component = { { /* * First, the mca_base_component_t struct containing meta * information about the component itself */ { MCA_MTL_BASE_VERSION_2_0_0, "mxm", /* MCA component name */ OMPI_MAJOR_VERSION, /* MCA component major version */ OMPI_MINOR_VERSION, /* MCA component minor version */ OMPI_RELEASE_VERSION, /* MCA component release version */ ompi_mtl_mxm_component_open, /* component open */ ompi_mtl_mxm_component_close, /* component close */ NULL, ompi_mtl_mxm_component_register }, { /* The component is not checkpoint ready */ MCA_BASE_METADATA_PARAM_NONE }, ompi_mtl_mxm_component_init /* component init */ } }; static int ompi_mtl_mxm_component_register(void) { mca_base_component_t*c; #if MXM_API < MXM_VERSION(3,0) unsigned long cur_ver; long major, minor; char* runtime_version; #endif c = &mca_mtl_mxm_component.super.mtl_version; ompi_mtl_mxm.verbose = 0; (void) mca_base_component_var_register(c, "verbose", "Verbose level of the MXM component", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &ompi_mtl_mxm.verbose); #if MXM_API > MXM_VERSION(2,0) ompi_mtl_mxm.mxm_np = 0; #else ompi_mtl_mxm.mxm_np = 128; #endif (void) mca_base_component_var_register(c, "np", "[integer] Minimal number of MPI processes in a single job " "required to activate the MXM transport", MCA_BASE_VAR_TYPE_INT, NULL,0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.mxm_np); ompi_mtl_mxm.compiletime_version = MXM_VERNO_STRING; (void) mca_base_component_var_register(c, MCA_COMPILETIME_VER, "Version of the libmxm library with which Open MPI was compiled", MCA_BASE_VAR_TYPE_VERSION_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.compiletime_version); #if MXM_API >= MXM_VERSION(3,0) ompi_mtl_mxm.runtime_version = (char *)mxm_get_version_string(); #else cur_ver = mxm_get_version(); major = (cur_ver >> MXM_MAJOR_BIT) & 0xff; minor = (cur_ver >> MXM_MINOR_BIT) & 0xff; asprintf(&runtime_version, "%ld.%ld", major, minor); ompi_mtl_mxm.runtime_version = runtime_version; #endif (void) mca_base_component_var_register(c, MCA_RUNTIME_VER, "Version of the libmxm library with which Open MPI is running", MCA_BASE_VAR_TYPE_VERSION_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.runtime_version); #if MXM_API < MXM_VERSION(3,0) free(runtime_version); #endif #if MXM_API >= MXM_VERSION(3,1) { unsigned long cur_ver = mxm_get_version(); ompi_mtl_mxm.bulk_connect = 0; if (cur_ver < MXM_VERSION(3,2)) { ompi_mtl_mxm.bulk_disconnect = 0; } else { ompi_mtl_mxm.bulk_disconnect = 1; } (void) mca_base_component_var_register(c, "bulk_connect", "[integer] use bulk connect", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.bulk_connect); (void) mca_base_component_var_register(c, "bulk_disconnect", "[integer] use bulk disconnect", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mtl_mxm.bulk_disconnect); if (cur_ver < MXM_VERSION(3,2) && (ompi_mtl_mxm.bulk_connect || ompi_mtl_mxm.bulk_disconnect)) { ompi_mtl_mxm.bulk_connect = 0; ompi_mtl_mxm.bulk_disconnect = 0; MXM_VERBOSE(1, "WARNING: OMPI runs with %s version of MXM that is less than 3.2, " "so bulk connect/disconnect cannot work properly and will be turn off.", ompi_mtl_mxm.runtime_version); } } #endif return OMPI_SUCCESS; } static int ompi_mtl_mxm_component_open(void) { mxm_error_t err; unsigned long cur_ver; int rc; mca_mtl_mxm_output = opal_output_open(NULL); opal_output_set_verbosity(mca_mtl_mxm_output, ompi_mtl_mxm.verbose); cur_ver = mxm_get_version(); if (cur_ver != MXM_API) { MXM_VERBOSE(1, "WARNING: OMPI was compiled with MXM version %d.%d but version %ld.%ld detected.", MXM_VERNO_MAJOR, MXM_VERNO_MINOR, (cur_ver >> MXM_MAJOR_BIT) & 0xff, (cur_ver >> MXM_MINOR_BIT) & 0xff); } #if MXM_API >= MXM_VERSION(2,0) /* Register memory hooks */ if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & opal_mem_hooks_support_level())) { setenv("MXM_MPI_MEM_ON_DEMAND_MAP", "y", 0); MXM_VERBOSE(1, "Enabling on-demand memory mapping"); ompi_mtl_mxm.using_mem_hooks = 1; } else { MXM_VERBOSE(1, "Disabling on-demand memory mapping"); ompi_mtl_mxm.using_mem_hooks = 0; } setenv("MXM_MPI_SINGLE_THREAD", ompi_mpi_thread_multiple ? "n" : "y" , 0); #endif #if MXM_API >= MXM_VERSION(2,1) if (MXM_OK != mxm_config_read_opts(&ompi_mtl_mxm.mxm_ctx_opts, &ompi_mtl_mxm.mxm_ep_opts, "MPI", NULL, 0)) #else if ((MXM_OK != mxm_config_read_context_opts(&ompi_mtl_mxm.mxm_ctx_opts)) || (MXM_OK != mxm_config_read_ep_opts(&ompi_mtl_mxm.mxm_ep_opts))) #endif { MXM_ERROR("Failed to parse MXM configuration"); return OPAL_ERR_BAD_PARAM; } err = mxm_init(ompi_mtl_mxm.mxm_ctx_opts, &ompi_mtl_mxm.mxm_context); MXM_VERBOSE(1, "mxm component open"); if (MXM_OK != err) { if (MXM_ERR_NO_DEVICE == err) { MXM_VERBOSE(1, "No supported device found, disqualifying mxm"); } else { opal_show_help("help-mtl-mxm.txt", "mxm init", true, mxm_error_string(err)); } return OPAL_ERR_NOT_AVAILABLE; } OBJ_CONSTRUCT(&mca_mtl_mxm_component.mxm_messages, ompi_free_list_t); rc = ompi_free_list_init_new(&mca_mtl_mxm_component.mxm_messages, sizeof(ompi_mtl_mxm_message_t), opal_cache_line_size, OBJ_CLASS(ompi_mtl_mxm_message_t), 0, opal_cache_line_size, 32 /* free list num */, -1 /* free list max */, 32 /* free list inc */, NULL); if (OMPI_SUCCESS != rc) { opal_show_help("help-mtl-mxm.txt", "mxm init", true, mxm_error_string(err)); return OPAL_ERR_NOT_AVAILABLE; } return OMPI_SUCCESS; } static int ompi_mtl_mxm_component_close(void) { if (ompi_mtl_mxm.mxm_context != NULL) { mxm_cleanup(ompi_mtl_mxm.mxm_context); ompi_mtl_mxm.mxm_context = NULL; OBJ_DESTRUCT(&mca_mtl_mxm_component.mxm_messages); #if MXM_API >= MXM_VERSION(2,0) mxm_config_free_ep_opts(ompi_mtl_mxm.mxm_ep_opts); mxm_config_free_context_opts(ompi_mtl_mxm.mxm_ctx_opts); #else mxm_config_free(ompi_mtl_mxm.mxm_ep_opts); mxm_config_free(ompi_mtl_mxm.mxm_ctx_opts); #endif } return OMPI_SUCCESS; } static mca_mtl_base_module_t* ompi_mtl_mxm_component_init(bool enable_progress_threads, bool enable_mpi_threads) { int rc; rc = ompi_mtl_mxm_module_init(); if (OMPI_SUCCESS != rc) { return NULL; } /* Calculate MTL constraints according to MXM types */ ompi_mtl_mxm.super.mtl_max_contextid = 1UL << (sizeof(mxm_ctxid_t) * 8); ompi_mtl_mxm.super.mtl_max_tag = 1UL << (sizeof(mxm_tag_t) * 8 - 2); ompi_mtl_mxm.super.mtl_request_size = sizeof(mca_mtl_mxm_request_t) - sizeof(struct mca_mtl_request_t); return &ompi_mtl_mxm.super; }