Merge pull request #702 from rolfv/pr/fix-cuda-mca-register

Make explicit call to initalize MCA parameters in common CUDA code.

opal/mca/btl/openib/btl_openib_component.c

@@ -16,7 +16,7 @@
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2009-2012 Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2011-2014 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Laboratory.  All rights reserved
  * Copyright (c) 2013-2014 Intel, Inc. All rights reserved
  * Copyright (c) 2014-2015 Research Organization for Information Science
@@ -200,6 +200,10 @@ static int btl_openib_component_register(void)
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
+#if OPAL_CUDA_SUPPORT
+    mca_common_cuda_register_mca_variables();
+#endif
+
     return OPAL_SUCCESS;
 }
 

opal/mca/btl/smcuda/btl_smcuda_component.c

@@ -195,6 +195,7 @@ static int smcuda_register(void)
     if (0 == mca_btl_smcuda.super.btl_cuda_eager_limit) {
         mca_btl_smcuda.super.btl_cuda_eager_limit = SIZE_MAX; /* magic number */
     }
+    mca_common_cuda_register_mca_variables();
 #endif /* OPAL_CUDA_SUPPORT */
     return mca_btl_smcuda_component_verify();
 }

opal/mca/common/cuda/common_cuda.c

@@ -115,6 +115,7 @@ cudaFunctionTable_t cuFunc;
 static int stage_one_init_ref_count = 0;
 static bool stage_three_init_complete = false;
 static bool common_cuda_initialized = false;
+static bool common_cuda_mca_parames_registered = false;
 static int mca_common_cuda_verbose;
 static int mca_common_cuda_output = 0;
 bool mca_common_cuda_enabled = false;
@@ -223,42 +224,14 @@ static void cuda_dump_memhandle(int, void *, char *) __opal_attribute_unused__ ;
 
 #endif /* OPAL_CUDA_SUPPORT_41 */
 
-
+/* This is a seperate function so we can see these variables with ompi_info and
-/**
+ * also set them with the tools interface */
- * This is the first stage of initialization.  This function is
+void mca_common_cuda_register_mca_variables(void)
- * called explicitly by any BTLs that can support CUDA-aware.
- * It is called during the component open phase of initialization.
- * This function will register some mca variables and then open
- * and load the symbols needed from the CUDA driver library. Look for
- * the SONAME of the library which is libcuda.so.1.  In most cases,
- * this will result in the library found.  However, there are some
- * setups that require the extra steps for searching.  Any failure
- * will result in this initialization failing and status will be set
- * showing that.
- */
-int mca_common_cuda_stage_one_init(void)
 {
-    int retval, i, j;
-    char *cudalibs[] = {"libcuda.so.1", "libcuda.dylib", NULL};
-    char *searchpaths[] = {"", "/usr/lib64", NULL};
-    char **errmsgs = NULL;
-    char *errmsg = NULL;
-    int errsize;
-    bool stage_one_init_passed = false;
 
-    stage_one_init_ref_count++;
+    if (false == common_cuda_mca_parames_registered) {
-    if (stage_one_init_ref_count > 1) {
+        common_cuda_mca_parames_registered = true;
-        opal_output_verbose(10, mca_common_cuda_output,
-                            "CUDA: stage_one_init_ref_count is now %d, no need to init",
-                            stage_one_init_ref_count);
-        return OPAL_SUCCESS;
     }
-
-    OBJ_CONSTRUCT(&common_cuda_init_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&common_cuda_htod_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&common_cuda_dtoh_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&common_cuda_ipc_lock, opal_mutex_t);
-
     /* Set different levels of verbosity in the cuda related code. */
     mca_common_cuda_verbose = 0;
     (void) mca_base_var_register("ompi", "mpi", "common_cuda", "verbose",
@@ -327,6 +300,43 @@ int mca_common_cuda_stage_one_init(void)
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &mca_common_cuda_cumemcpy_timing);
 #endif /* OPAL_ENABLE_DEBUG */
+}
+
+/**
+ * This is the first stage of initialization.  This function is called
+ * explicitly by any BTLs that can support CUDA-aware. It is called during
+ * the component open phase of initialization. This fuction will look for
+ * the SONAME of the library which is libcuda.so.1. In most cases, this will
+ * result in the library found.  However, there are some setups that require
+ * the extra steps for searching. This function will then load the symbols
+ * needed from the CUDA driver library. Any failure will result in this
+ * initialization failing and status will be set showing that.
+ */
+int mca_common_cuda_stage_one_init(void)
+{
+    int retval, i, j;
+    char *cudalibs[] = {"libcuda.so.1", "libcuda.dylib", NULL};
+    char *searchpaths[] = {"", "/usr/lib64", NULL};
+    char **errmsgs = NULL;
+    char *errmsg = NULL;
+    int errsize;
+    bool stage_one_init_passed = false;
+
+    stage_one_init_ref_count++;
+    if (stage_one_init_ref_count > 1) {
+        opal_output_verbose(10, mca_common_cuda_output,
+                            "CUDA: stage_one_init_ref_count is now %d, no need to init",
+                            stage_one_init_ref_count);
+        return OPAL_SUCCESS;
+    }
+
+    /* This is a no-op in most cases as the parameters were registered earlier */
+    mca_common_cuda_register_mca_variables();
+
+    OBJ_CONSTRUCT(&common_cuda_init_lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&common_cuda_htod_lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&common_cuda_dtoh_lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&common_cuda_ipc_lock, opal_mutex_t);
 
     mca_common_cuda_output = opal_output_open(NULL);
     opal_output_set_verbosity(mca_common_cuda_output, mca_common_cuda_verbose);

opal/mca/common/cuda/common_cuda.h

@@ -44,7 +44,7 @@ struct mca_mpool_common_cuda_reg_t {
 typedef struct mca_mpool_common_cuda_reg_t mca_mpool_common_cuda_reg_t;
 extern bool mca_common_cuda_enabled;
 
-OPAL_DECLSPEC int mca_common_cuda_register_mca_variables(void);
+OPAL_DECLSPEC void mca_common_cuda_register_mca_variables(void);
 
 OPAL_DECLSPEC void mca_common_cuda_register(void *ptr, size_t amount, char *msg);