Merge pull request #4286 from aravindksg/master

Use opal_show_help to warn about PSM2_CUDA envvar setting

ompi/mca/mtl/psm2/help-mtl-psm2.txt

@@ -45,3 +45,7 @@ Unknown path record query mechanism %s. Supported mechanisms are %s.
 #
 [message too big]
 Message size %llu bigger than supported by PSM2 API. Max = %llu
+#
+[no psm2 cuda env]
+Using CUDA enabled OpenMPI but PSM2_CUDA environment variable is %s.
+This is not a recommended combination. If the application uses %s.

ompi/mca/mtl/psm2/mtl_psm2.c

@@ -100,9 +100,6 @@ int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs) {
     char *generated_key;
     char env_string[256];
     int rc;
-#if OPAL_CUDA_SUPPORT
-    char *cuda_env;
-#endif
 
     generated_key = getenv(OPAL_MCA_PREFIX"orte_precondition_transports");
     memset(uu, 0, sizeof(psm2_uuid_t));
@@ -178,11 +175,6 @@ int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs) {
 
 #if OPAL_CUDA_SUPPORT
     ompi_mtl_psm2.super.mtl_flags |= MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE;
-
-    cuda_env = getenv("PSM2_CUDA");
-    if (!cuda_env || ( strcmp(cuda_env, "0") == 0) )
-        opal_output(0, "Warning: If running with device buffers, there is a"
-                    " chance the application might fail. Try setting PSM2_CUDA=1.\n");
 #endif
 
     return OMPI_SUCCESS;

ompi/mca/mtl/psm2/mtl_psm2_component.c

@@ -199,6 +199,9 @@ static int
 ompi_mtl_psm2_component_register(void)
 {
     int num_local_procs, num_total_procs;
+#if OPAL_CUDA_SUPPORT
+    char *cuda_env;
+#endif
 
     ompi_mtl_psm2.connect_timeout = 180;
     (void) mca_base_component_var_register(&mca_mtl_psm2_component.super.mtl_version,
@@ -223,6 +226,30 @@ ompi_mtl_psm2_component_register(void)
         param_priority = 40;
     }
 
+#if OPAL_CUDA_SUPPORT
+    /*
+     * If using CUDA enabled OpenMPI, the user likely intends to
+     * run with CUDA buffers. So, force-set the envvar here if user failed
+     * to set it.
+     */
+    cuda_env = getenv("PSM2_CUDA");
+    if (!cuda_env) {
+        opal_show_help("help-mtl-psm2.txt",
+                       "no psm2 cuda env", true,
+                       "not set",
+                       "Host buffers,\nthere will be a performance penalty"
+                       " due to OMPI force setting this variable now.\n"
+                       "Set environment variable to 0 if using Host buffers" );
+        setenv("PSM2_CUDA", "1", 0);
+    } else if (strcmp(cuda_env, "0") == 0) {
+        opal_show_help("help-mtl-psm2.txt",
+                       "no psm2 cuda env", true,
+                       "set to 0",
+                       "CUDA buffers,\nthe execution will SEGFAULT."
+                       " Set environment variable to 1 if using CUDA buffers");
+    }
+#endif
+
     (void) mca_base_component_var_register (&mca_mtl_psm2_component.super.mtl_version,
                                             "priority", "Priority of the PSM2 MTL component",
                                             MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,