diff --git a/ompi/mca/common/cuda/common_cuda.c b/ompi/mca/common/cuda/common_cuda.c index 412f246a1b..f64849dd2e 100644 --- a/ompi/mca/common/cuda/common_cuda.c +++ b/ompi/mca/common/cuda/common_cuda.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2011-2012 NVIDIA Corporation. All rights reserved. + * Copyright (c) 2011-2013 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -36,6 +36,7 @@ #include "opal/util/output.h" #include "ompi/mca/mpool/base/base.h" #include "ompi/mca/rte/rte.h" +#include "ompi/runtime/params.h" #include "common_cuda.h" static bool common_cuda_initialized = false; @@ -124,6 +125,10 @@ static int mca_common_cuda_init(void) CUcontext cuContext; common_cuda_mem_regs_t *mem_reg; + if (!ompi_mpi_cuda_support) { + return OMPI_ERROR; + } + if (common_cuda_initialized) { return OMPI_SUCCESS; } diff --git a/ompi/runtime/ompi_mpi_params.c b/ompi/runtime/ompi_mpi_params.c index f7ec5b85f3..57c38d2aae 100644 --- a/ompi/runtime/ompi_mpi_params.c +++ b/ompi/runtime/ompi_mpi_params.c @@ -12,6 +12,7 @@ * Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2012 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2013 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -61,6 +62,7 @@ int ompi_mpi_leave_pinned = -1; bool ompi_mpi_leave_pinned_pipeline = false; bool ompi_have_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE); bool ompi_use_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE); +bool ompi_mpi_cuda_support = OPAL_INT_TO_BOOL(OMPI_CUDA_SUPPORT); static bool show_default_mca_params = false; static bool show_file_mca_params = false; @@ -287,6 +289,23 @@ int ompi_mpi_register_params(void) } } + mca_base_param_reg_int_name("mpi", "cuda_support", + "Whether CUDA GPU buffer support is enabled or not", + false, false, OMPI_CUDA_SUPPORT, &value); + ompi_mpi_cuda_support = OPAL_INT_TO_BOOL(value); + if (ompi_mpi_cuda_support) { + value = 0; + if (OMPI_CUDA_SUPPORT) { + value = 1; + } + if (0 == value) { + ompi_show_help("help-mpi-runtime.txt", + "CUDA GPU buffer support requested but compiled out", + true); + ompi_mpi_cuda_support = false; + } + } + return OMPI_SUCCESS; } diff --git a/ompi/runtime/params.h b/ompi/runtime/params.h index 5cc179abc8..24bb5b60f8 100644 --- a/ompi/runtime/params.h +++ b/ompi/runtime/params.h @@ -12,6 +12,7 @@ * Copyright (c) 2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2013 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -138,6 +139,11 @@ OMPI_DECLSPEC extern bool ompi_have_sparse_group_storage; */ OMPI_DECLSPEC extern bool ompi_use_sparse_group_storage; +/** + * Whether we want to enable CUDA GPU buffer send and receive support. + */ +OMPI_DECLSPEC extern bool ompi_mpi_cuda_support; + /** * Register MCA parameters used by the MPI layer. * diff --git a/opal/datatype/opal_datatype_cuda.c b/opal/datatype/opal_datatype_cuda.c index ea11c8fc7b..eafe8d2bdd 100755 --- a/opal/datatype/opal_datatype_cuda.c +++ b/opal/datatype/opal_datatype_cuda.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 NVIDIA Corporation. All rights reserved. + * Copyright (c) 2011-2013 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -22,6 +22,7 @@ static bool initialized = false; static int opal_cuda_verbose; +static int opal_cuda_enabled = 0; /* Starts out disabled */ static int opal_cuda_output = 0; static void opal_cuda_support_init(void); static int (*common_cuda_initialization_function)(void) = NULL; @@ -41,6 +42,16 @@ void mca_cuda_convertor_init(opal_convertor_t* convertor, const void *pUserBuf) CUmemorytype memType; CUdeviceptr dbuf = (CUdeviceptr)pUserBuf; + /* Only do the initialization on the first GPU access */ + if (!initialized) { + opal_cuda_support_init(); + } + + /* If not enabled, then nothing else to do */ + if (!opal_cuda_enabled) { + return; + } + res = cuPointerGetAttribute(&memType, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, dbuf); if (res != CUDA_SUCCESS) { @@ -54,11 +65,6 @@ void mca_cuda_convertor_init(opal_convertor_t* convertor, const void *pUserBuf) /* Must be a device pointer */ assert(memType == CU_MEMORYTYPE_DEVICE); - /* Only do the initialization on the first GPU access */ - if (!initialized) { - opal_cuda_support_init(); - } - convertor->cbmemcpy = (memcpy_fct_t)&opal_cuda_memcpy; convertor->flags |= CONVERTOR_CUDA; } @@ -176,7 +182,13 @@ static void opal_cuda_support_init(void) /* Callback into the common cuda initialization routine. This is only * set if some work had been done already in the common cuda code.*/ if (NULL != common_cuda_initialization_function) { - common_cuda_initialization_function(); + if (0 == common_cuda_initialization_function()) { + opal_cuda_enabled = 1; + } else { + return; /* Initialization failed - no support */ + } + } else { + return; /* No initialization function - no support */ } /* Set different levels of verbosity in the cuda related code. */