Add run-time parameter to enable and disable CUDA GPU support.

This commit was SVN r27970.

ompi/mca/common/cuda/common_cuda.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2011-2012 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,6 +36,7 @@
 #include "opal/util/output.h"
 #include "ompi/mca/mpool/base/base.h"
 #include "ompi/mca/rte/rte.h"
+#include "ompi/runtime/params.h"
 #include "common_cuda.h"
 
 static bool common_cuda_initialized = false;
@@ -124,6 +125,10 @@ static int mca_common_cuda_init(void)
     CUcontext cuContext;
     common_cuda_mem_regs_t *mem_reg;
 
+    if (!ompi_mpi_cuda_support) {
+        return OMPI_ERROR;
+    }
+
     if (common_cuda_initialized) {
         return OMPI_SUCCESS;
     }

ompi/runtime/ompi_mpi_params.c

@@ -12,6 +12,7 @@
  * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007-2012 Los Alamos National Security, LLC.  All rights
  *                         reserved. 
+ * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -61,6 +62,7 @@ int ompi_mpi_leave_pinned = -1;
 bool ompi_mpi_leave_pinned_pipeline = false;
 bool ompi_have_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
 bool ompi_use_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
+bool ompi_mpi_cuda_support = OPAL_INT_TO_BOOL(OMPI_CUDA_SUPPORT);
 
 static bool show_default_mca_params = false;
 static bool show_file_mca_params = false;
@@ -287,6 +289,23 @@ int ompi_mpi_register_params(void)
         }
     }
 
+    mca_base_param_reg_int_name("mpi", "cuda_support", 
+                                "Whether CUDA GPU buffer support is enabled or not",
+                                false, false, OMPI_CUDA_SUPPORT, &value);
+    ompi_mpi_cuda_support = OPAL_INT_TO_BOOL(value);
+    if (ompi_mpi_cuda_support) {
+        value = 0;
+        if (OMPI_CUDA_SUPPORT) {
+            value = 1;
+        }
+        if (0 == value) {
+            ompi_show_help("help-mpi-runtime.txt", 
+                           "CUDA GPU buffer support requested but compiled out",
+                           true);
+            ompi_mpi_cuda_support = false;
+        }
+    }
+
     return OMPI_SUCCESS;
 }
 

ompi/runtime/params.h

@@ -12,6 +12,7 @@
  * Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
  *                         reserved. 
  * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -138,6 +139,11 @@ OMPI_DECLSPEC extern bool ompi_have_sparse_group_storage;
  */
 OMPI_DECLSPEC extern bool ompi_use_sparse_group_storage;
 
+/**
+ * Whether we want to enable CUDA GPU buffer send and receive support.
+ */
+OMPI_DECLSPEC extern bool ompi_mpi_cuda_support;
+
 /**
  * Register MCA parameters used by the MPI layer.
  *

opal/datatype/opal_datatype_cuda.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,6 +22,7 @@
 
 static bool initialized = false;
 static int opal_cuda_verbose;
+static int opal_cuda_enabled = 0; /* Starts out disabled */
 static int opal_cuda_output = 0;
 static void opal_cuda_support_init(void);
 static int (*common_cuda_initialization_function)(void) = NULL;
@@ -41,6 +42,16 @@ void mca_cuda_convertor_init(opal_convertor_t* convertor, const void *pUserBuf)
     CUmemorytype memType;
     CUdeviceptr dbuf = (CUdeviceptr)pUserBuf;
 
+    /* Only do the initialization on the first GPU access */
+    if (!initialized) {
+        opal_cuda_support_init();
+    }
+
+    /* If not enabled, then nothing else to do */
+    if (!opal_cuda_enabled) {
+        return;
+    }
+
     res = cuPointerGetAttribute(&memType,
                                 CU_POINTER_ATTRIBUTE_MEMORY_TYPE, dbuf);
     if (res != CUDA_SUCCESS) {
@@ -54,11 +65,6 @@ void mca_cuda_convertor_init(opal_convertor_t* convertor, const void *pUserBuf)
     /* Must be a device pointer */
     assert(memType == CU_MEMORYTYPE_DEVICE);
 
-    /* Only do the initialization on the first GPU access */
-    if (!initialized) {
-        opal_cuda_support_init();
-    }
-
     convertor->cbmemcpy = (memcpy_fct_t)&opal_cuda_memcpy;
     convertor->flags |= CONVERTOR_CUDA;
 }
@@ -176,7 +182,13 @@ static void opal_cuda_support_init(void)
     /* Callback into the common cuda initialization routine. This is only
      * set if some work had been done already in the common cuda code.*/
     if (NULL != common_cuda_initialization_function) {
-        common_cuda_initialization_function();
+        if (0 == common_cuda_initialization_function()) {
+            opal_cuda_enabled = 1;
+        } else {
+            return; /* Initialization failed - no support */
+        }
+    } else {
+        return;  /* No initialization function - no support */
     }
 
     /* Set different levels of verbosity in the cuda related code. */