diff --git a/opal/mca/btl/openib/btl_openib_component.c b/opal/mca/btl/openib/btl_openib_component.c index 058f51f738..7172eab11b 100644 --- a/opal/mca/btl/openib/btl_openib_component.c +++ b/opal/mca/btl/openib/btl_openib_component.c @@ -16,7 +16,7 @@ * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011-2014 NVIDIA Corporation. All rights reserved. + * Copyright (c) 2011-2015 NVIDIA Corporation. All rights reserved. * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved * Copyright (c) 2013-2014 Intel, Inc. All rights reserved * Copyright (c) 2014-2015 Research Organization for Information Science @@ -200,6 +200,10 @@ static int btl_openib_component_register(void) return OPAL_ERR_NOT_AVAILABLE; } +#if OPAL_CUDA_SUPPORT + mca_common_cuda_register_mca_variables(); +#endif + return OPAL_SUCCESS; } diff --git a/opal/mca/btl/smcuda/btl_smcuda_component.c b/opal/mca/btl/smcuda/btl_smcuda_component.c index 8ddbd8ac16..dcbf0ec518 100644 --- a/opal/mca/btl/smcuda/btl_smcuda_component.c +++ b/opal/mca/btl/smcuda/btl_smcuda_component.c @@ -195,6 +195,7 @@ static int smcuda_register(void) if (0 == mca_btl_smcuda.super.btl_cuda_eager_limit) { mca_btl_smcuda.super.btl_cuda_eager_limit = SIZE_MAX; /* magic number */ } + mca_common_cuda_register_mca_variables(); #endif /* OPAL_CUDA_SUPPORT */ return mca_btl_smcuda_component_verify(); } diff --git a/opal/mca/common/cuda/common_cuda.c b/opal/mca/common/cuda/common_cuda.c index a82807a5f7..be03f962a3 100644 --- a/opal/mca/common/cuda/common_cuda.c +++ b/opal/mca/common/cuda/common_cuda.c @@ -115,6 +115,7 @@ cudaFunctionTable_t cuFunc; static int stage_one_init_ref_count = 0; static bool stage_three_init_complete = false; static bool common_cuda_initialized = false; +static bool common_cuda_mca_parames_registered = false; static int mca_common_cuda_verbose; static int mca_common_cuda_output = 0; bool mca_common_cuda_enabled = false; @@ -223,42 +224,14 @@ static void cuda_dump_memhandle(int, void *, char *) __opal_attribute_unused__ ; #endif /* OPAL_CUDA_SUPPORT_41 */ - -/** - * This is the first stage of initialization. This function is - * called explicitly by any BTLs that can support CUDA-aware. - * It is called during the component open phase of initialization. - * This function will register some mca variables and then open - * and load the symbols needed from the CUDA driver library. Look for - * the SONAME of the library which is libcuda.so.1. In most cases, - * this will result in the library found. However, there are some - * setups that require the extra steps for searching. Any failure - * will result in this initialization failing and status will be set - * showing that. - */ -int mca_common_cuda_stage_one_init(void) +/* This is a seperate function so we can see these variables with ompi_info and + * also set them with the tools interface */ +void mca_common_cuda_register_mca_variables(void) { - int retval, i, j; - char *cudalibs[] = {"libcuda.so.1", "libcuda.dylib", NULL}; - char *searchpaths[] = {"", "/usr/lib64", NULL}; - char **errmsgs = NULL; - char *errmsg = NULL; - int errsize; - bool stage_one_init_passed = false; - stage_one_init_ref_count++; - if (stage_one_init_ref_count > 1) { - opal_output_verbose(10, mca_common_cuda_output, - "CUDA: stage_one_init_ref_count is now %d, no need to init", - stage_one_init_ref_count); - return OPAL_SUCCESS; + if (false == common_cuda_mca_parames_registered) { + common_cuda_mca_parames_registered = true; } - - OBJ_CONSTRUCT(&common_cuda_init_lock, opal_mutex_t); - OBJ_CONSTRUCT(&common_cuda_htod_lock, opal_mutex_t); - OBJ_CONSTRUCT(&common_cuda_dtoh_lock, opal_mutex_t); - OBJ_CONSTRUCT(&common_cuda_ipc_lock, opal_mutex_t); - /* Set different levels of verbosity in the cuda related code. */ mca_common_cuda_verbose = 0; (void) mca_base_var_register("ompi", "mpi", "common_cuda", "verbose", @@ -327,6 +300,43 @@ int mca_common_cuda_stage_one_init(void) MCA_BASE_VAR_SCOPE_READONLY, &mca_common_cuda_cumemcpy_timing); #endif /* OPAL_ENABLE_DEBUG */ +} + +/** + * This is the first stage of initialization. This function is called + * explicitly by any BTLs that can support CUDA-aware. It is called during + * the component open phase of initialization. This fuction will look for + * the SONAME of the library which is libcuda.so.1. In most cases, this will + * result in the library found. However, there are some setups that require + * the extra steps for searching. This function will then load the symbols + * needed from the CUDA driver library. Any failure will result in this + * initialization failing and status will be set showing that. + */ +int mca_common_cuda_stage_one_init(void) +{ + int retval, i, j; + char *cudalibs[] = {"libcuda.so.1", "libcuda.dylib", NULL}; + char *searchpaths[] = {"", "/usr/lib64", NULL}; + char **errmsgs = NULL; + char *errmsg = NULL; + int errsize; + bool stage_one_init_passed = false; + + stage_one_init_ref_count++; + if (stage_one_init_ref_count > 1) { + opal_output_verbose(10, mca_common_cuda_output, + "CUDA: stage_one_init_ref_count is now %d, no need to init", + stage_one_init_ref_count); + return OPAL_SUCCESS; + } + + /* This is a no-op in most cases as the parameters were registered earlier */ + mca_common_cuda_register_mca_variables(); + + OBJ_CONSTRUCT(&common_cuda_init_lock, opal_mutex_t); + OBJ_CONSTRUCT(&common_cuda_htod_lock, opal_mutex_t); + OBJ_CONSTRUCT(&common_cuda_dtoh_lock, opal_mutex_t); + OBJ_CONSTRUCT(&common_cuda_ipc_lock, opal_mutex_t); mca_common_cuda_output = opal_output_open(NULL); opal_output_set_verbosity(mca_common_cuda_output, mca_common_cuda_verbose); diff --git a/opal/mca/common/cuda/common_cuda.h b/opal/mca/common/cuda/common_cuda.h index 7da817de51..c0cd59c359 100644 --- a/opal/mca/common/cuda/common_cuda.h +++ b/opal/mca/common/cuda/common_cuda.h @@ -44,7 +44,7 @@ struct mca_mpool_common_cuda_reg_t { typedef struct mca_mpool_common_cuda_reg_t mca_mpool_common_cuda_reg_t; extern bool mca_common_cuda_enabled; -OPAL_DECLSPEC int mca_common_cuda_register_mca_variables(void); +OPAL_DECLSPEC void mca_common_cuda_register_mca_variables(void); OPAL_DECLSPEC void mca_common_cuda_register(void *ptr, size_t amount, char *msg);