/* * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2014 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * This file contains various support functions for doing CUDA * operations. Some of the features are only available in CUDA 4.1 * and later, so some code is conditionalized around the * OPAL_CUDA_SUPPORT_41 macro. */ #include "ompi_config.h" #include #include #include #include "opal/align.h" #include "opal/datatype/opal_convertor.h" #include "opal/datatype/opal_datatype_cuda.h" #include "opal/util/output.h" #include "opal/util/lt_interface.h" #include "opal/util/show_help.h" #include "ompi/mca/mpool/base/base.h" #include "ompi/mca/rte/rte.h" #include "ompi/runtime/params.h" #include "common_cuda.h" /** * Since function names can get redefined in cuda.h file, we need to do this * stringifying to get the latest function name from the header file. For * example, cuda.h may have something like this: * #define cuMemFree cuMemFree_v2 * We want to make sure we find cuMemFree_v2, not cuMemFree. */ #define STRINGIFY2(x) #x #define STRINGIFY(x) STRINGIFY2(x) #define OMPI_CUDA_DLSYM(libhandle, funcName) \ do { \ *(void **)(&cuFunc.funcName) = opal_lt_dlsym(libhandle, STRINGIFY(funcName)); \ if (NULL == cuFunc.funcName) { \ opal_show_help("help-mpi-common-cuda.txt", "dlsym failed", true, \ STRINGIFY(funcName), opal_lt_dlerror()); \ return 1; \ } else { \ opal_output_verbose(15, mca_common_cuda_output, \ "CUDA: successful dlsym of %s", \ STRINGIFY(funcName)); \ } \ } while (0) /* Structure to hold CUDA function pointers that get dynamically loaded. */ struct cudaFunctionTable { int (*cuPointerGetAttribute)(void *, CUpointer_attribute, CUdeviceptr); int (*cuMemcpyAsync)(CUdeviceptr, CUdeviceptr, size_t, CUstream); int (*cuMemcpy)(CUdeviceptr, CUdeviceptr, size_t); int (*cuMemAlloc)(CUdeviceptr *, unsigned int); int (*cuMemFree)(CUdeviceptr buf); int (*cuCtxGetCurrent)(void *cuContext); int (*cuStreamCreate)(CUstream *, int); int (*cuEventCreate)(CUevent *, int); int (*cuEventRecord)(CUevent, CUstream); int (*cuMemHostRegister)(void *, size_t, unsigned int); int (*cuMemHostUnregister)(void *); int (*cuEventQuery)(CUevent); int (*cuEventDestroy)(CUevent); int (*cuStreamWaitEvent)(CUstream, CUevent, unsigned int); int (*cuMemGetAddressRange)(CUdeviceptr*, size_t*, CUdeviceptr); #if OPAL_CUDA_SUPPORT_41 int (*cuIpcGetEventHandle)(CUipcEventHandle*, CUevent); int (*cuIpcOpenEventHandle)(CUevent*, CUipcEventHandle); int (*cuIpcOpenMemHandle)(CUdeviceptr*, CUipcMemHandle, unsigned int); int (*cuIpcCloseMemHandle)(CUdeviceptr); int (*cuIpcGetMemHandle)(CUipcMemHandle*, CUdeviceptr); #endif /* OPAL_CUDA_SUPPORT_41 */ int (*cuCtxGetDevice)(CUdevice *); int (*cuDeviceCanAccessPeer)(int *, CUdevice, CUdevice); int (*cuDeviceGet)(CUdevice *, int); #if OPAL_CUDA_GDR_SUPPORT int (*cuPointerSetAttribute)(const void *, CUpointer_attribute, CUdeviceptr); #endif /* OPAL_CUDA_GDR_SUPPORT */ int (*cuCtxSetCurrent)(CUcontext); int (*cuEventSynchronize)(CUevent); } cudaFunctionTable; typedef struct cudaFunctionTable cudaFunctionTable_t; cudaFunctionTable_t cuFunc; static bool stage_one_init_complete = false; static bool stage_three_init_complete = false; static bool common_cuda_initialized = false; static int mca_common_cuda_verbose; static int mca_common_cuda_output = 0; bool mca_common_cuda_enabled = false; static bool mca_common_cuda_register_memory = true; static bool mca_common_cuda_warning = false; static opal_list_t common_cuda_memory_registrations; static CUstream ipcStream; static CUstream dtohStream; static CUstream htodStream; static CUstream memcpyStream; static CUevent memcpyEvent; /* Functions called by opal layer - plugged into opal function table */ static int mca_common_cuda_is_gpu_buffer(const void*); static int mca_common_cuda_memmove(void*, void*, size_t); static int mca_common_cuda_cu_memcpy_async(void*, const void*, size_t, opal_convertor_t*); static int mca_common_cuda_cu_memcpy(void*, const void*, size_t); /* Structure to hold memory registrations that are delayed until first * call to send or receive a GPU pointer */ struct common_cuda_mem_regs_t { opal_list_item_t super; void *ptr; size_t amount; char *msg; }; typedef struct common_cuda_mem_regs_t common_cuda_mem_regs_t; OBJ_CLASS_DECLARATION(common_cuda_mem_regs_t); OBJ_CLASS_INSTANCE(common_cuda_mem_regs_t, opal_list_item_t, NULL, NULL); #if OPAL_CUDA_SUPPORT_41 static int mca_common_cuda_async = 1; static int mca_common_cuda_cumemcpy_async; #if OPAL_ENABLE_DEBUG static int mca_common_cuda_cumemcpy_timing; #endif /* OPAL_ENABLE_DEBUG */ /* Array of CUDA events to be queried for IPC stream, sending side and * receiving side. */ CUevent *cuda_event_ipc_array; CUevent *cuda_event_dtoh_array; CUevent *cuda_event_htod_array; /* Array of fragments currently being moved by cuda async non-blocking * operations */ struct mca_btl_base_descriptor_t **cuda_event_ipc_frag_array; struct mca_btl_base_descriptor_t **cuda_event_dtoh_frag_array; struct mca_btl_base_descriptor_t **cuda_event_htod_frag_array; /* First free/available location in cuda_event_status_array */ int cuda_event_ipc_first_avail, cuda_event_dtoh_first_avail, cuda_event_htod_first_avail; /* First currently-being used location in the cuda_event_status_array */ int cuda_event_ipc_first_used, cuda_event_dtoh_first_used, cuda_event_htod_first_used; /* Number of status items currently in use */ int cuda_event_ipc_num_used, cuda_event_dtoh_num_used, cuda_event_htod_num_used; /* Size of array holding events */ int cuda_event_max = 400; static int cuda_event_ipc_most = 0; static int cuda_event_dtoh_most = 0; static int cuda_event_htod_most = 0; /* Handle to libcuda.so */ opal_lt_dlhandle libcuda_handle; #define CUDA_COMMON_TIMING 0 #if OPAL_ENABLE_DEBUG /* Some timing support structures. Enable this to help analyze * internal performance issues. */ static struct timespec ts_start; static struct timespec ts_end; static double accum; #define THOUSAND 1000L #define MILLION 1000000L static float mydifftime(struct timespec ts_start, struct timespec ts_end); #endif /* OPAL_ENABLE_DEBUG */ /* These functions are typically unused in the optimized builds. */ static void cuda_dump_evthandle(int, void *, char *) __opal_attribute_unused__ ; static void cuda_dump_memhandle(int, void *, char *) __opal_attribute_unused__ ; #if OPAL_ENABLE_DEBUG #define CUDA_DUMP_MEMHANDLE(a) cuda_dump_memhandle a #define CUDA_DUMP_EVTHANDLE(a) cuda_dump_evthandle a #else #define CUDA_DUMP_MEMHANDLE(a) #define CUDA_DUMP_EVTHANDLE(a) #endif /* OPAL_ENABLE_DEBUG */ #endif /* OPAL_CUDA_SUPPORT_41 */ /** * This function is registered with the OPAL CUDA support. In that way, * these function pointers will be loaded into the OPAL CUDA code when * the first convertor is initialized. This does not trigger any CUDA * specific initialization as this may just be a host buffer that is * triggering this call. */ static int mca_common_cuda_init(opal_common_cuda_function_table_t *ftable) { if (OPAL_UNLIKELY(!ompi_mpi_cuda_support)) { return OMPI_ERROR; } ftable->gpu_is_gpu_buffer = &mca_common_cuda_is_gpu_buffer; ftable->gpu_cu_memcpy_async = &mca_common_cuda_cu_memcpy_async; ftable->gpu_cu_memcpy = &mca_common_cuda_cu_memcpy; ftable->gpu_memmove = &mca_common_cuda_memmove; opal_output_verbose(30, mca_common_cuda_output, "CUDA: support functions initialized"); return OMPI_SUCCESS; } /** * This is the first stage of initialization. This function is * triggered when there are memory registration requests from various * BTLs. This function will register some mca variables and then open * and load the symbols needed from the CUDA driver library. Look for * the SONAME of the library which is libcuda.so.1. In most cases, * this will result in the library found. However, there are some * setups that require the extra steps for searching. Any failure * will result in this initialization failing and status will be set * showing that. */ int mca_common_cuda_stage_one_init(void) { opal_lt_dladvise advise; int retval, i, j; int advise_support = 1; char *cudalibs[] = {"libcuda.so.1", NULL}; char *searchpaths[] = {"", "/usr/lib64", NULL}; char **errmsgs = NULL; char *errmsg = NULL; int errsize; bool stage_one_init_passed = false; if (true == stage_one_init_complete) { return 0; } stage_one_init_complete = true; /* Set different levels of verbosity in the cuda related code. */ mca_common_cuda_verbose = 0; (void) mca_base_var_register("ompi", "mpi", "common_cuda", "verbose", "Set level of common cuda verbosity", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_common_cuda_verbose); /* Control whether system buffers get CUDA pinned or not. Allows for * performance analysis. */ mca_common_cuda_register_memory = true; (void) mca_base_var_register("ompi", "mpi", "common_cuda", "register_memory", "Whether to cuMemHostRegister preallocated BTL buffers", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_common_cuda_register_memory); /* Control whether we see warnings when CUDA memory registration fails. This is * useful when CUDA support is configured in, but we are running a regular MPI * application without CUDA. */ mca_common_cuda_warning = true; (void) mca_base_var_register("ompi", "mpi", "common_cuda", "warning", "Whether to print warnings when CUDA registration fails", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_common_cuda_warning); #if OPAL_CUDA_SUPPORT_41 /* Use this flag to test async vs sync copies */ mca_common_cuda_async = 1; (void) mca_base_var_register("ompi", "mpi", "common_cuda", "memcpy_async", "Set to 0 to force CUDA sync copy instead of async", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_common_cuda_async); /* Use this parameter to increase the number of outstanding events allows */ (void) mca_base_var_register("ompi", "mpi", "common_cuda", "event_max", "Set number of oustanding CUDA events", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &cuda_event_max); #endif /* OPAL_CUDA_SUPPORT_41 */ /* Use this flag to test cuMemcpyAsync vs cuMemcpy */ mca_common_cuda_cumemcpy_async = 0; (void) mca_base_var_register("ompi", "mpi", "common_cuda", "cumemcpy_async", "Set to 0 to force CUDA cuMemcpy instead of cuMemcpyAsync/cuEventRecord/cuEventSynchronize", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY, &mca_common_cuda_cumemcpy_async); #if OPAL_ENABLE_DEBUG /* Use this flag to dump out timing of cumempcy sync and async */ mca_common_cuda_cumemcpy_timing = 0; (void) mca_base_var_register("ompi", "mpi", "common_cuda", "cumemcpy_timing", "Set to 1 to dump timing of eager copies", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY, &mca_common_cuda_cumemcpy_timing); #endif /* OPAL_ENABLE_DEBUG */ mca_common_cuda_output = opal_output_open(NULL); opal_output_set_verbosity(mca_common_cuda_output, mca_common_cuda_verbose); /* First check if the support is enabled. In the case that the user has * turned it off, we do not need to continue with any CUDA specific * initialization. Do this after MCA parameter registration. */ if (!ompi_mpi_cuda_support) { return 1; } if (0 != (retval = opal_lt_dlinit())) { if (OPAL_ERR_NOT_SUPPORTED == retval) { opal_show_help("help-mpi-common-cuda.txt", "dlopen disabled", true); } else { opal_show_help("help-mpi-common-cuda.txt", "unknown ltdl error", true, "opal_lt_dlinit", retval, opal_lt_dlerror()); } return 1; } /* Initialize the lt_dladvise structure. If this does not work, we can * proceed without the support. Things should still work. */ if (0 != (retval = opal_lt_dladvise_init(&advise))) { if (OPAL_ERR_NOT_SUPPORTED == retval) { advise_support = 0; } else { opal_show_help("help-mpi-common-cuda.txt", "unknown ltdl error", true, "opal_lt_dladvise_init", retval, opal_lt_dlerror()); return 1; } } /* Now walk through all the potential names libcuda and find one * that works. If it does, all is good. If not, print out all * the messages about why things failed. This code was careful * to try and save away all error messages if the loading ultimately * failed to help with debugging. * NOTE: On the first loop we just utilize the default loading * paths from the system. For the second loop, set /usr/lib64 to * the search path and try again. This is done to handle the case * where we have both 32 and 64 bit libcuda.so libraries installed. * Even when running in 64-bit mode, the /usr/lib direcotry * is searched first and we may find a 32-bit libcuda.so.1 library. * Loading of this library will fail as libtool does not handle having * the wrong ABI in the search path (unlike ld or ld.so). Note that * we only set this search path after the original search. This is * so that LD_LIBRARY_PATH and run path settings are respected. * Setting this search path overrides them (rather then being appended). */ if (advise_support) { if (0 != (retval = opal_lt_dladvise_global(&advise))) { opal_show_help("help-mpi-common-cuda.txt", "unknown ltdl error", true, "opal_lt_dladvise_global", retval, opal_lt_dlerror()); opal_lt_dladvise_destroy(&advise); return 1; } j = 0; while (searchpaths[j] != NULL) { /* Set explicit search path if entry is not empty string */ if (strcmp("", searchpaths[j])) { opal_lt_dlsetsearchpath(searchpaths[j]); } i = 0; while (cudalibs[i] != NULL) { const char *str; libcuda_handle = opal_lt_dlopenadvise(cudalibs[i], advise); if (NULL == libcuda_handle) { str = opal_lt_dlerror(); if (NULL != str) { opal_argv_append(&errsize, &errmsgs, str); } else { opal_argv_append(&errsize, &errmsgs, "lt_dlerror() returned NULL."); } opal_output_verbose(10, mca_common_cuda_output, "CUDA: Library open error: %s", errmsgs[errsize-1]); } else { opal_output_verbose(10, mca_common_cuda_output, "CUDA: Library successfully opened %s", cudalibs[i]); stage_one_init_passed = true; break; } i++; } if (true == stage_one_init_passed) break; /* Break out of outer loop */ j++; } opal_lt_dladvise_destroy(&advise); } else { j = 0; /* No lt_dladvise support. This should rarely happen. */ while (searchpaths[j] != NULL) { /* Set explicit search path if entry is not empty string */ if (strcmp("", searchpaths[j])) { opal_lt_dlsetsearchpath(searchpaths[j]); } i = 0; while (cudalibs[i] != NULL) { const char *str; libcuda_handle = opal_lt_dlopen(cudalibs[i]); if (NULL == libcuda_handle) { str = opal_lt_dlerror(); if (NULL != str) { opal_argv_append(&errsize, &errmsgs, str); } else { opal_argv_append(&errsize, &errmsgs, "lt_dlerror() returned NULL."); } opal_output_verbose(10, mca_common_cuda_output, "CUDA: Library open error: %s", errmsgs[errsize-1]); } else { opal_output_verbose(10, mca_common_cuda_output, "CUDA: Library successfully opened %s", cudalibs[i]); stage_one_init_passed = true; break; } i++; } if (true == stage_one_init_passed) break; /* Break out of outer loop */ j++; } } if (true != stage_one_init_passed) { errmsg = opal_argv_join(errmsgs, '\n'); opal_show_help("help-mpi-common-cuda.txt", "dlopen failed", true, errmsg); } opal_argv_free(errmsgs); free(errmsg); if (true != stage_one_init_passed) { return 1; } opal_cuda_add_initialization_function(&mca_common_cuda_init); OBJ_CONSTRUCT(&common_cuda_memory_registrations, opal_list_t); /* Map in the functions that we need. Note that if there is an error * the macro OMPI_CUDA_DLSYM will print an error and call return. */ OMPI_CUDA_DLSYM(libcuda_handle, cuStreamCreate); OMPI_CUDA_DLSYM(libcuda_handle, cuCtxGetCurrent); OMPI_CUDA_DLSYM(libcuda_handle, cuEventCreate); OMPI_CUDA_DLSYM(libcuda_handle, cuEventRecord); OMPI_CUDA_DLSYM(libcuda_handle, cuMemHostRegister); OMPI_CUDA_DLSYM(libcuda_handle, cuMemHostUnregister); OMPI_CUDA_DLSYM(libcuda_handle, cuPointerGetAttribute); OMPI_CUDA_DLSYM(libcuda_handle, cuEventQuery); OMPI_CUDA_DLSYM(libcuda_handle, cuEventDestroy); OMPI_CUDA_DLSYM(libcuda_handle, cuStreamWaitEvent); OMPI_CUDA_DLSYM(libcuda_handle, cuMemcpyAsync); OMPI_CUDA_DLSYM(libcuda_handle, cuMemcpy); OMPI_CUDA_DLSYM(libcuda_handle, cuMemFree); OMPI_CUDA_DLSYM(libcuda_handle, cuMemAlloc); OMPI_CUDA_DLSYM(libcuda_handle, cuMemGetAddressRange); #if OPAL_CUDA_SUPPORT_41 OMPI_CUDA_DLSYM(libcuda_handle, cuIpcGetEventHandle); OMPI_CUDA_DLSYM(libcuda_handle, cuIpcOpenEventHandle); OMPI_CUDA_DLSYM(libcuda_handle, cuIpcOpenMemHandle); OMPI_CUDA_DLSYM(libcuda_handle, cuIpcCloseMemHandle); OMPI_CUDA_DLSYM(libcuda_handle, cuIpcGetMemHandle); #endif /* OPAL_CUDA_SUPPORT_41 */ OMPI_CUDA_DLSYM(libcuda_handle, cuCtxGetDevice); OMPI_CUDA_DLSYM(libcuda_handle, cuDeviceCanAccessPeer); OMPI_CUDA_DLSYM(libcuda_handle, cuDeviceGet); #if OPAL_CUDA_GDR_SUPPORT OMPI_CUDA_DLSYM(libcuda_handle, cuPointerSetAttribute); #endif /* OPAL_CUDA_GDR_SUPPORT */ OMPI_CUDA_DLSYM(libcuda_handle, cuCtxSetCurrent); OMPI_CUDA_DLSYM(libcuda_handle, cuEventSynchronize); return 0; } /** * This is the last phase of initialization. This is triggered when we examine * a buffer pointer and determine it is a GPU buffer. We then assume the user * has selected their GPU and we can go ahead with all the CUDA related * initializations. */ static int mca_common_cuda_stage_three_init(void) { int i, s; CUresult res; CUcontext cuContext; common_cuda_mem_regs_t *mem_reg; stage_three_init_complete = true; opal_output_verbose(20, mca_common_cuda_output, "CUDA: entering stage three init"); if (OPAL_UNLIKELY(!ompi_mpi_cuda_support)) { opal_output_verbose(20, mca_common_cuda_output, "CUDA: No mpi cuda support, exiting stage three init"); return OMPI_ERROR; } if (OPAL_LIKELY(common_cuda_initialized)) { opal_output_verbose(20, mca_common_cuda_output, "CUDA: Stage three already complete, exiting stage three init"); return OMPI_SUCCESS; } /* Check to see if this process is running in a CUDA context. If * so, all is good. If not, then disable registration of memory. */ res = cuFunc.cuCtxGetCurrent(&cuContext); if (CUDA_SUCCESS != res) { if (mca_common_cuda_warning) { /* Check for the not initialized error since we can make suggestions to * user for this error. */ if (CUDA_ERROR_NOT_INITIALIZED == res) { opal_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent failed not initialized", true); } else { opal_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent failed", true, res); } } mca_common_cuda_enabled = false; mca_common_cuda_register_memory = false; } else if ((CUDA_SUCCESS == res) && (NULL == cuContext)) { if (mca_common_cuda_warning) { opal_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent returned NULL", true); } mca_common_cuda_enabled = false; mca_common_cuda_register_memory = false; } else { /* All is good. mca_common_cuda_register_memory will retain its original * value. Normally, that is 1, but the user can override it to disable * registration of the internal buffers. */ mca_common_cuda_enabled = true; opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuCtxGetCurrent succeeded"); } /* No need to go on at this point. If we cannot create a context and we are at * the point where we are making MPI calls, it is time to fully disable * CUDA support. */ if (false == mca_common_cuda_enabled) { return OMPI_ERROR; } #if OPAL_CUDA_SUPPORT_41 if (true == mca_common_cuda_enabled) { /* Set up an array to store outstanding IPC async copy events */ cuda_event_ipc_array = NULL; cuda_event_ipc_frag_array = NULL; cuda_event_ipc_num_used = 0; cuda_event_ipc_first_avail = 0; cuda_event_ipc_first_used = 0; cuda_event_ipc_array = (CUevent *) malloc(sizeof(CUevent) * cuda_event_max); if (NULL == cuda_event_ipc_array) { opal_show_help("help-mpi-common-cuda.txt", "No memory", true, errno, strerror(errno)); return OMPI_ERROR; } /* Create the events since they can be reused. */ for (i = 0; i < cuda_event_max; i++) { res = cuFunc.cuEventCreate(&cuda_event_ipc_array[i], CU_EVENT_DISABLE_TIMING); if (CUDA_SUCCESS != res) { opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } } /* The first available status index is 0. Make an empty frag array. */ cuda_event_ipc_frag_array = (struct mca_btl_base_descriptor_t **) malloc(sizeof(struct mca_btl_base_descriptor_t *) * cuda_event_max); if (NULL == cuda_event_ipc_frag_array) { opal_show_help("help-mpi-common-cuda.txt", "No memory", true, errno, strerror(errno)); return OMPI_ERROR; } } #endif /* OPAL_CUDA_SUPPORT_41 */ if (true == mca_common_cuda_enabled) { /* Set up an array to store outstanding async dtoh events. Used on the * sending side for asynchronous copies. */ cuda_event_dtoh_array = NULL; cuda_event_dtoh_frag_array = NULL; cuda_event_dtoh_num_used = 0; cuda_event_dtoh_first_avail = 0; cuda_event_dtoh_first_used = 0; cuda_event_dtoh_array = (CUevent *) malloc(sizeof(CUevent) * cuda_event_max); if (NULL == cuda_event_dtoh_array) { opal_show_help("help-mpi-common-cuda.txt", "No memory", true, errno, strerror(errno)); return OMPI_ERROR; } /* Create the events since they can be reused. */ for (i = 0; i < cuda_event_max; i++) { res = cuFunc.cuEventCreate(&cuda_event_dtoh_array[i], CU_EVENT_DISABLE_TIMING); if (CUDA_SUCCESS != res) { opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } } /* The first available status index is 0. Make an empty frag array. */ cuda_event_dtoh_frag_array = (struct mca_btl_base_descriptor_t **) malloc(sizeof(struct mca_btl_base_descriptor_t *) * cuda_event_max); if (NULL == cuda_event_dtoh_frag_array) { opal_show_help("help-mpi-common-cuda.txt", "No memory", true, errno, strerror(errno)); return OMPI_ERROR; } /* Set up an array to store outstanding async htod events. Used on the * receiving side for asynchronous copies. */ cuda_event_htod_array = NULL; cuda_event_htod_frag_array = NULL; cuda_event_htod_num_used = 0; cuda_event_htod_first_avail = 0; cuda_event_htod_first_used = 0; cuda_event_htod_array = (CUevent *) malloc(sizeof(CUevent) * cuda_event_max); if (NULL == cuda_event_htod_array) { opal_show_help("help-mpi-common-cuda.txt", "No memory", true, errno, strerror(errno)); return OMPI_ERROR; } /* Create the events since they can be reused. */ for (i = 0; i < cuda_event_max; i++) { res = cuFunc.cuEventCreate(&cuda_event_htod_array[i], CU_EVENT_DISABLE_TIMING); if (CUDA_SUCCESS != res) { opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } } /* The first available status index is 0. Make an empty frag array. */ cuda_event_htod_frag_array = (struct mca_btl_base_descriptor_t **) malloc(sizeof(struct mca_btl_base_descriptor_t *) * cuda_event_max); if (NULL == cuda_event_htod_frag_array) { opal_show_help("help-mpi-common-cuda.txt", "No memory", true, errno, strerror(errno)); return OMPI_ERROR; } } s = opal_list_get_size(&common_cuda_memory_registrations); for(i = 0; i < s; i++) { mem_reg = (common_cuda_mem_regs_t *) opal_list_remove_first(&common_cuda_memory_registrations); if (mca_common_cuda_enabled && mca_common_cuda_register_memory) { res = cuFunc.cuMemHostRegister(mem_reg->ptr, mem_reg->amount, 0); if (res != CUDA_SUCCESS) { /* If registering the memory fails, print a message and continue. * This is not a fatal error. */ opal_show_help("help-mpi-common-cuda.txt", "cuMemHostRegister failed", true, mem_reg->ptr, mem_reg->amount, ompi_process_info.nodename, res, mem_reg->msg); } else { opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuMemHostRegister OK on mpool %s: " "address=%p, bufsize=%d", mem_reg->msg, mem_reg->ptr, (int)mem_reg->amount); } } free(mem_reg->msg); OBJ_RELEASE(mem_reg); } /* Create stream for use in ipc asynchronous copies */ res = cuFunc.cuStreamCreate(&ipcStream, 0); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } /* Create stream for use in dtoh asynchronous copies */ res = cuFunc.cuStreamCreate(&dtohStream, 0); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } /* Create stream for use in htod asynchronous copies */ res = cuFunc.cuStreamCreate(&htodStream, 0); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } if (mca_common_cuda_cumemcpy_async) { /* Create stream for use in cuMemcpyAsync synchronous copies */ res = cuFunc.cuStreamCreate(&memcpyStream, 0); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } /* Create event for use in cuMemcpyAsync synchronous copies */ res = cuFunc.cuEventCreate(&memcpyEvent, 0); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed", true, ompi_process_info.nodename, res); return OMPI_ERROR; } } opal_output_verbose(30, mca_common_cuda_output, "CUDA: initialized"); common_cuda_initialized = true; return OMPI_SUCCESS; } /** * Call the CUDA register function so we pin the memory in the CUDA * space. */ void mca_common_cuda_register(void *ptr, size_t amount, char *msg) { int res; /* Always first check if the support is enabled. If not, just return */ if (!ompi_mpi_cuda_support) return; /* Registering memory during BTL initialization will be the first call * into the cuda common code, so this is where we do the first * initialization function. If the first stage fails, then disable * support and return. */ if (!stage_one_init_complete) { if (0 != mca_common_cuda_stage_one_init()) { ompi_mpi_cuda_support = 0; return; } } if (!common_cuda_initialized) { common_cuda_mem_regs_t *regptr; regptr = OBJ_NEW(common_cuda_mem_regs_t); regptr->ptr = ptr; regptr->amount = amount; regptr->msg = strdup(msg); opal_list_append(&common_cuda_memory_registrations, (opal_list_item_t*)regptr); return; } if (mca_common_cuda_enabled && mca_common_cuda_register_memory) { res = cuFunc.cuMemHostRegister(ptr, amount, 0); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { /* If registering the memory fails, print a message and continue. * This is not a fatal error. */ opal_show_help("help-mpi-common-cuda.txt", "cuMemHostRegister failed", true, ptr, amount, ompi_process_info.nodename, res, msg); } else { opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuMemHostRegister OK on mpool %s: " "address=%p, bufsize=%d", msg, ptr, (int)amount); } } } /** * Call the CUDA unregister function so we unpin the memory in the CUDA * space. */ void mca_common_cuda_unregister(void *ptr, char *msg) { int res, i, s; common_cuda_mem_regs_t *mem_reg; /* This can happen if memory was queued up to be registered, but * no CUDA operations happened, so it never was registered. * Therefore, just release any of the resources. */ if (!common_cuda_initialized) { s = opal_list_get_size(&common_cuda_memory_registrations); for(i = 0; i < s; i++) { mem_reg = (common_cuda_mem_regs_t *) opal_list_remove_first(&common_cuda_memory_registrations); free(mem_reg->msg); OBJ_RELEASE(mem_reg); } return; } if (mca_common_cuda_enabled && mca_common_cuda_register_memory) { res = cuFunc.cuMemHostUnregister(ptr); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { /* If unregistering the memory fails, print a message and continue. * This is not a fatal error. */ opal_show_help("help-mpi-common-cuda.txt", "cuMemHostUnregister failed", true, ptr, ompi_process_info.nodename, res, msg); } else { opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuMemHostUnregister OK on mpool %s: " "address=%p", msg, ptr); } } } #if OPAL_CUDA_SUPPORT_41 /* * Get the memory handle of a local section of memory that can be sent * to the remote size so it can access the memory. This is the * registration function for the sending side of a message transfer. */ int cuda_getmemhandle(void *base, size_t size, mca_mpool_base_registration_t *newreg, mca_mpool_base_registration_t *hdrreg) { CUmemorytype memType; CUresult result; CUipcMemHandle memHandle; CUdeviceptr pbase; size_t psize; mca_mpool_common_cuda_reg_t *cuda_reg = (mca_mpool_common_cuda_reg_t*)newreg; /* We should only be there if this is a CUDA device pointer */ result = cuFunc.cuPointerGetAttribute(&memType, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, (CUdeviceptr)base); assert(CUDA_SUCCESS == result); assert(CU_MEMORYTYPE_DEVICE == memType); /* Get the memory handle so we can send it to the remote process. */ result = cuFunc.cuIpcGetMemHandle(&memHandle, (CUdeviceptr)base); CUDA_DUMP_MEMHANDLE((100, &memHandle, "GetMemHandle-After")); if (CUDA_SUCCESS != result) { opal_show_help("help-mpi-common-cuda.txt", "cuIpcGetMemHandle failed", true, result, base); return OMPI_ERROR; } else { opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuIpcGetMemHandle passed: base=%p size=%d", base, (int)size); } /* Need to get the real base and size of the memory handle. This is * how the remote side saves the handles in a cache. */ result = cuFunc.cuMemGetAddressRange(&pbase, &psize, (CUdeviceptr)base); if (CUDA_SUCCESS != result) { opal_show_help("help-mpi-common-cuda.txt", "cuMemGetAddressRange failed", true, result, base); return OMPI_ERROR; } else { opal_output_verbose(10, mca_common_cuda_output, "CUDA: cuMemGetAddressRange passed: addr=%p, size=%d, pbase=%p, psize=%d ", base, (int)size, (void *)pbase, (int)psize); } /* Store all the information in the registration */ cuda_reg->base.base = (void *)pbase; cuda_reg->base.bound = (unsigned char *)pbase + psize - 1; memcpy(&cuda_reg->memHandle, &memHandle, sizeof(memHandle)); #if OPAL_CUDA_SYNC_MEMOPS /* With CUDA 6.0, we can set an attribute on the memory pointer that will * ensure any synchronous copies are completed prior to any other access * of the memory region. This means we do not need to record an event * and send to the remote side. */ memType = 1; /* Just use this variable since we already have it */ result = cuFunc.cuPointerSetAttribute(&memType, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, (CUdeviceptr)base); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuPointerSetAttribute failed", true, ompi_process_info.nodename, result, base); return OMPI_ERROR; } #else /* Need to record the event to ensure that any memcopies into the * device memory have completed. The event handle associated with * this event is sent to the remote process so that it will wait * on this event prior to copying data out of the device memory. * Note that this needs to be the NULL stream to make since it is * unknown what stream any copies into the device memory were done * with. */ result = cuFunc.cuEventRecord((CUevent)cuda_reg->event, 0); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, result, base); return OMPI_ERROR; } #endif /* OPAL_CUDA_SYNC_MEMOPS */ return OMPI_SUCCESS; } /* * This function is called by the local side that called the cuda_getmemhandle. * There is nothing to be done so just return. */ int cuda_ungetmemhandle(void *reg_data, mca_mpool_base_registration_t *reg) { CUDA_DUMP_EVTHANDLE((100, ((mca_mpool_common_cuda_reg_t *)reg)->evtHandle, "cuda_ungetmemhandle")); opal_output_verbose(10, mca_common_cuda_output, "CUDA: cuda_ungetmemhandle (no-op): base=%p", reg->base); return OMPI_SUCCESS; } /* * Open a memory handle that refers to remote memory so we can get an address * that works on the local side. This is the registration function for the * remote side of a transfer. newreg contains the new handle. hddrreg contains * the memory handle that was received from the remote side. */ int cuda_openmemhandle(void *base, size_t size, mca_mpool_base_registration_t *newreg, mca_mpool_base_registration_t *hdrreg) { CUresult result; CUipcMemHandle memHandle; mca_mpool_common_cuda_reg_t *cuda_newreg = (mca_mpool_common_cuda_reg_t*)newreg; /* Need to copy into memory handle for call into CUDA library. */ memcpy(&memHandle, cuda_newreg->memHandle, sizeof(memHandle)); CUDA_DUMP_MEMHANDLE((100, &memHandle, "Before call to cuIpcOpenMemHandle")); /* Open the memory handle and store it into the registration structure. */ result = cuFunc.cuIpcOpenMemHandle((CUdeviceptr *)&newreg->alloc_base, memHandle, CU_IPC_MEM_LAZY_ENABLE_PEER_ACCESS); /* If there are some stale entries in the cache, they can cause other * registrations to fail. Let the caller know that so that can attempt * to clear them out. */ if (CUDA_ERROR_ALREADY_MAPPED == result) { opal_output_verbose(10, mca_common_cuda_output, "CUDA: cuIpcOpenMemHandle returned CUDA_ERROR_ALREADY_MAPPED for " "p=%p,size=%d: notify memory pool\n", base, (int)size); return OMPI_ERR_WOULD_BLOCK; } if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuIpcOpenMemHandle failed", true, result, base); /* Currently, this is a non-recoverable error */ return OMPI_ERROR; } else { opal_output_verbose(10, mca_common_cuda_output, "CUDA: cuIpcOpenMemHandle passed: base=%p (remote base=%p,size=%d)", newreg->alloc_base, base, (int)size); CUDA_DUMP_MEMHANDLE((200, &memHandle, "cuIpcOpenMemHandle")); } return OMPI_SUCCESS; } /* * Close a memory handle that refers to remote memory. */ int cuda_closememhandle(void *reg_data, mca_mpool_base_registration_t *reg) { CUresult result; mca_mpool_common_cuda_reg_t *cuda_reg = (mca_mpool_common_cuda_reg_t*)reg; result = cuFunc.cuIpcCloseMemHandle((CUdeviceptr)cuda_reg->base.alloc_base); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuIpcCloseMemHandle failed", true, result, cuda_reg->base.alloc_base); /* We will just continue on and hope things continue to work. */ } else { opal_output_verbose(10, mca_common_cuda_output, "CUDA: cuIpcCloseMemHandle passed: base=%p", cuda_reg->base.alloc_base); CUDA_DUMP_MEMHANDLE((100, cuda_reg->memHandle, "cuIpcCloseMemHandle")); } return OMPI_SUCCESS; } void mca_common_cuda_construct_event_and_handle(uint64_t **event, void **handle) { CUresult result; result = cuFunc.cuEventCreate((CUevent *)event, CU_EVENT_INTERPROCESS | CU_EVENT_DISABLE_TIMING); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed", true, ompi_process_info.nodename, result); } result = cuFunc.cuIpcGetEventHandle((CUipcEventHandle *)handle, (CUevent)*event); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuIpcGetEventHandle failed", true, result); } CUDA_DUMP_EVTHANDLE((10, handle, "construct_event_and_handle")); } void mca_common_cuda_destruct_event(uint64_t *event) { CUresult result; result = cuFunc.cuEventDestroy((CUevent)event); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventDestroy failed", true, result); } } /* * Put remote event on stream to ensure that the the start of the * copy does not start until the completion of the event. */ void mca_common_wait_stream_synchronize(mca_mpool_common_cuda_reg_t *rget_reg) { #if OPAL_CUDA_SYNC_MEMOPS /* No need for any of this with SYNC_MEMOPS feature */ return; #else /* OPAL_CUDA_SYNC_MEMOPS */ CUipcEventHandle evtHandle; CUevent event; CUresult result; memcpy(&evtHandle, rget_reg->evtHandle, sizeof(evtHandle)); CUDA_DUMP_EVTHANDLE((100, &evtHandle, "stream_synchronize")); result = cuFunc.cuIpcOpenEventHandle(&event, evtHandle); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuIpcOpenEventHandle failed", true, result); } /* BEGIN of Workaround - There is a bug in CUDA 4.1 RC2 and earlier * versions. Need to record an event on the stream, even though * it is not used, to make sure we do not short circuit our way * out of the cuStreamWaitEvent test. */ result = cuFunc.cuEventRecord(event, 0); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, ompi_process_info.nodename, result); } /* END of Workaround */ result = cuFunc.cuStreamWaitEvent(0, event, 0); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuStreamWaitEvent failed", true, result); } /* All done with this event. */ result = cuFunc.cuEventDestroy(event); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventDestroy failed", true, result); } #endif /* OPAL_CUDA_SYNC_MEMOPS */ } /* * Start the asynchronous copy. Then record and save away an event that will * be queried to indicate the copy has completed. */ int mca_common_cuda_memcpy(void *dst, void *src, size_t amount, char *msg, struct mca_btl_base_descriptor_t *frag, int *done) { CUresult result; int iter; /* First make sure there is room to store the event. If not, then * return an error. The error message will tell the user to try and * run again, but with a larger array for storing events. */ if (cuda_event_ipc_num_used == cuda_event_max) { opal_show_help("help-mpi-common-cuda.txt", "Out of cuEvent handles", true, cuda_event_max, cuda_event_max+100, cuda_event_max+100); return OMPI_ERR_OUT_OF_RESOURCE; } if (cuda_event_ipc_num_used > cuda_event_ipc_most) { cuda_event_ipc_most = cuda_event_ipc_num_used; /* Just print multiples of 10 */ if (0 == (cuda_event_ipc_most % 10)) { opal_output_verbose(20, mca_common_cuda_output, "Maximum ipc events used is now %d", cuda_event_ipc_most); } } /* This is the standard way to run. Running with synchronous copies is available * to measure the advantages of asynchronous copies. */ if (OPAL_LIKELY(mca_common_cuda_async)) { result = cuFunc.cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, amount, ipcStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed", true, dst, src, amount, result); return OMPI_ERROR; } else { opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuMemcpyAsync passed: dst=%p, src=%p, size=%d", dst, src, (int)amount); } result = cuFunc.cuEventRecord(cuda_event_ipc_array[cuda_event_ipc_first_avail], ipcStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, ompi_process_info.nodename, result); return OMPI_ERROR; } cuda_event_ipc_frag_array[cuda_event_ipc_first_avail] = frag; /* Bump up the first available slot and number used by 1 */ cuda_event_ipc_first_avail++; if (cuda_event_ipc_first_avail >= cuda_event_max) { cuda_event_ipc_first_avail = 0; } cuda_event_ipc_num_used++; *done = 0; } else { /* Mimic the async function so they use the same memcpy call. */ result = cuFunc.cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, amount, ipcStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed", true, dst, src, amount, result); return OMPI_ERROR; } else { opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuMemcpyAsync passed: dst=%p, src=%p, size=%d", dst, src, (int)amount); } /* Record an event, then wait for it to complete with calls to cuEventQuery */ result = cuFunc.cuEventRecord(cuda_event_ipc_array[cuda_event_ipc_first_avail], ipcStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, ompi_process_info.nodename, result); return OMPI_ERROR; } cuda_event_ipc_frag_array[cuda_event_ipc_first_avail] = frag; /* Bump up the first available slot and number used by 1 */ cuda_event_ipc_first_avail++; if (cuda_event_ipc_first_avail >= cuda_event_max) { cuda_event_ipc_first_avail = 0; } cuda_event_ipc_num_used++; result = cuFunc.cuEventQuery(cuda_event_ipc_array[cuda_event_ipc_first_used]); if ((CUDA_SUCCESS != result) && (CUDA_ERROR_NOT_READY != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed", true, result); return OMPI_ERROR; } iter = 0; while (CUDA_ERROR_NOT_READY == result) { if (0 == (iter % 10)) { opal_output(-1, "EVENT NOT DONE (iter=%d)", iter); } result = cuFunc.cuEventQuery(cuda_event_ipc_array[cuda_event_ipc_first_used]); if ((CUDA_SUCCESS != result) && (CUDA_ERROR_NOT_READY != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed", true, result); return OMPI_ERROR; } iter++; } --cuda_event_ipc_num_used; ++cuda_event_ipc_first_used; if (cuda_event_ipc_first_used >= cuda_event_max) { cuda_event_ipc_first_used = 0; } *done = 1; } return OMPI_SUCCESS; } /* * Record an event and save the frag. This is called by the sending side and * is used to queue an event when a htod copy has been initiated. */ int mca_common_cuda_record_dtoh_event(char *msg, struct mca_btl_base_descriptor_t *frag) { CUresult result; /* First make sure there is room to store the event. If not, then * return an error. The error message will tell the user to try and * run again, but with a larger array for storing events. */ if (cuda_event_dtoh_num_used == cuda_event_max) { opal_show_help("help-mpi-common-cuda.txt", "Out of cuEvent handles", true, cuda_event_max, cuda_event_max+100, cuda_event_max+100); return OMPI_ERR_OUT_OF_RESOURCE; } if (cuda_event_dtoh_num_used > cuda_event_dtoh_most) { cuda_event_dtoh_most = cuda_event_dtoh_num_used; /* Just print multiples of 10 */ if (0 == (cuda_event_dtoh_most % 10)) { opal_output_verbose(20, mca_common_cuda_output, "Maximum DtoH events used is now %d", cuda_event_dtoh_most); } } result = cuFunc.cuEventRecord(cuda_event_dtoh_array[cuda_event_dtoh_first_avail], dtohStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, ompi_process_info.nodename, result); return OMPI_ERROR; } cuda_event_dtoh_frag_array[cuda_event_dtoh_first_avail] = frag; /* Bump up the first available slot and number used by 1 */ cuda_event_dtoh_first_avail++; if (cuda_event_dtoh_first_avail >= cuda_event_max) { cuda_event_dtoh_first_avail = 0; } cuda_event_dtoh_num_used++; return OMPI_SUCCESS; } /* * Record an event and save the frag. This is called by the receiving side and * is used to queue an event when a dtoh copy has been initiated. */ int mca_common_cuda_record_htod_event(char *msg, struct mca_btl_base_descriptor_t *frag) { CUresult result; /* First make sure there is room to store the event. If not, then * return an error. The error message will tell the user to try and * run again, but with a larger array for storing events. */ if (cuda_event_htod_num_used == cuda_event_max) { opal_show_help("help-mpi-common-cuda.txt", "Out of cuEvent handles", true, cuda_event_max, cuda_event_max+100, cuda_event_max+100); return OMPI_ERR_OUT_OF_RESOURCE; } if (cuda_event_htod_num_used > cuda_event_htod_most) { cuda_event_htod_most = cuda_event_htod_num_used; /* Just print multiples of 10 */ if (0 == (cuda_event_htod_most % 10)) { opal_output_verbose(20, mca_common_cuda_output, "Maximum HtoD events used is now %d", cuda_event_htod_most); } } result = cuFunc.cuEventRecord(cuda_event_htod_array[cuda_event_htod_first_avail], htodStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, ompi_process_info.nodename, result); return OMPI_ERROR; } cuda_event_htod_frag_array[cuda_event_htod_first_avail] = frag; /* Bump up the first available slot and number used by 1 */ cuda_event_htod_first_avail++; if (cuda_event_htod_first_avail >= cuda_event_max) { cuda_event_htod_first_avail = 0; } cuda_event_htod_num_used++; return OMPI_SUCCESS; } /** * Used to get the dtoh stream for initiating asynchronous copies. */ void *mca_common_cuda_get_dtoh_stream(void) { return (void *)dtohStream; } /** * Used to get the htod stream for initiating asynchronous copies. */ void *mca_common_cuda_get_htod_stream(void) { return (void *)htodStream; } /* * Function is called every time progress is called with the sm BTL. If there * are outstanding events, check to see if one has completed. If so, hand * back the fragment for further processing. */ int progress_one_cuda_ipc_event(struct mca_btl_base_descriptor_t **frag) { CUresult result; if (cuda_event_ipc_num_used > 0) { opal_output_verbose(20, mca_common_cuda_output, "CUDA: progress_one_cuda_ipc_event, outstanding_events=%d", cuda_event_ipc_num_used); result = cuFunc.cuEventQuery(cuda_event_ipc_array[cuda_event_ipc_first_used]); /* We found an event that is not ready, so return. */ if (CUDA_ERROR_NOT_READY == result) { opal_output_verbose(20, mca_common_cuda_output, "CUDA: cuEventQuery returned CUDA_ERROR_NOT_READY"); *frag = NULL; return 0; } else if (CUDA_SUCCESS != result) { opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed", true, result); *frag = NULL; return OMPI_ERROR; } *frag = cuda_event_ipc_frag_array[cuda_event_ipc_first_used]; opal_output_verbose(10, mca_common_cuda_output, "CUDA: cuEventQuery returned %d", result); /* Bump counters, loop around the circular buffer if necessary */ --cuda_event_ipc_num_used; ++cuda_event_ipc_first_used; if (cuda_event_ipc_first_used >= cuda_event_max) { cuda_event_ipc_first_used = 0; } /* A return value of 1 indicates an event completed and a frag was returned */ return 1; } return 0; } /** * Progress any dtoh event completions. */ int progress_one_cuda_dtoh_event(struct mca_btl_base_descriptor_t **frag) { CUresult result; if (cuda_event_dtoh_num_used > 0) { opal_output_verbose(30, mca_common_cuda_output, "CUDA: progress_one_cuda_dtoh_event, outstanding_events=%d", cuda_event_dtoh_num_used); result = cuFunc.cuEventQuery(cuda_event_dtoh_array[cuda_event_dtoh_first_used]); /* We found an event that is not ready, so return. */ if (CUDA_ERROR_NOT_READY == result) { opal_output_verbose(30, mca_common_cuda_output, "CUDA: cuEventQuery returned CUDA_ERROR_NOT_READY"); *frag = NULL; return 0; } else if (CUDA_SUCCESS != result) { opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed", true, result); *frag = NULL; return OMPI_ERROR; } *frag = cuda_event_dtoh_frag_array[cuda_event_dtoh_first_used]; opal_output_verbose(30, mca_common_cuda_output, "CUDA: cuEventQuery returned %d", result); /* Bump counters, loop around the circular buffer if necessary */ --cuda_event_dtoh_num_used; ++cuda_event_dtoh_first_used; if (cuda_event_dtoh_first_used >= cuda_event_max) { cuda_event_dtoh_first_used = 0; } /* A return value of 1 indicates an event completed and a frag was returned */ return 1; } return 0; } /** * Progress any dtoh event completions. */ int progress_one_cuda_htod_event(struct mca_btl_base_descriptor_t **frag) { CUresult result; if (cuda_event_htod_num_used > 0) { opal_output_verbose(30, mca_common_cuda_output, "CUDA: progress_one_cuda_htod_event, outstanding_events=%d", cuda_event_htod_num_used); result = cuFunc.cuEventQuery(cuda_event_htod_array[cuda_event_htod_first_used]); /* We found an event that is not ready, so return. */ if (CUDA_ERROR_NOT_READY == result) { opal_output_verbose(30, mca_common_cuda_output, "CUDA: cuEventQuery returned CUDA_ERROR_NOT_READY"); *frag = NULL; return 0; } else if (CUDA_SUCCESS != result) { opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed", true, result); *frag = NULL; return OMPI_ERROR; } *frag = cuda_event_htod_frag_array[cuda_event_htod_first_used]; opal_output_verbose(30, mca_common_cuda_output, "CUDA: cuEventQuery returned %d", result); /* Bump counters, loop around the circular buffer if necessary */ --cuda_event_htod_num_used; ++cuda_event_htod_first_used; if (cuda_event_htod_first_used >= cuda_event_max) { cuda_event_htod_first_used = 0; } /* A return value of 1 indicates an event completed and a frag was returned */ return 1; } return 0; } /** * Need to make sure the handle we are retrieving from the cache is still * valid. Compare the cached handle to the one received. */ int mca_common_cuda_memhandle_matches(mca_mpool_common_cuda_reg_t *new_reg, mca_mpool_common_cuda_reg_t *old_reg) { if (0 == memcmp(new_reg->memHandle, old_reg->memHandle, sizeof(new_reg->memHandle))) { return 1; } else { return 0; } } /* * Function to dump memory handle information. This is based on * definitions from cuiinterprocess_private.h. */ static void cuda_dump_memhandle(int verbose, void *memHandle, char *str) { struct InterprocessMemHandleInternal { /* The first two entries are the CUinterprocessCtxHandle */ int64_t ctxId; /* unique (within a process) id of the sharing context */ int pid; /* pid of sharing context */ int64_t size; int64_t blocksize; int64_t offset; int gpuId; int subDeviceIndex; int64_t serial; } memH; if (NULL == str) { str = "CUDA"; } memcpy(&memH, memHandle, sizeof(memH)); opal_output_verbose(verbose, mca_common_cuda_output, "%s:ctxId=%d, pid=%d, size=%d, blocksize=%d, offset=%d, gpuId=%d, " "subDeviceIndex=%d, serial=%d", str, (int)memH.ctxId, memH.pid, (int)memH.size, (int)memH.blocksize, (int)memH.offset, memH.gpuId, memH.subDeviceIndex, (int)memH.serial); } /* * Function to dump memory handle information. This is based on * definitions from cuiinterprocess_private.h. */ static void cuda_dump_evthandle(int verbose, void *evtHandle, char *str) { struct InterprocessEventHandleInternal { /* The first two entries are the CUinterprocessCtxHandle */ int64_t ctxId; /* unique (within a process) id of the sharing context */ int pid; /* pid of sharing context */ int pad; /* pad to match the structure */ int index; } evtH; if (NULL == str) { str = "CUDA"; } memcpy(&evtH, evtHandle, sizeof(evtH)); opal_output_verbose(verbose, mca_common_cuda_output, "CUDA: %s:ctxId=%d, pid=%d, index=%d", str, (int)evtH.ctxId, evtH.pid, (int)evtH.index); } /* Return microseconds of elapsed time. Microseconds are relevant when * trying to understand the fixed overhead of the communication. Used * when trying to time various functions. * * Cut and past the following to get timings where wanted. * * clock_gettime(CLOCK_MONOTONIC, &ts_start); * FUNCTION OF INTEREST * clock_gettime(CLOCK_MONOTONIC, &ts_end); * accum = mydifftime(ts_start, ts_end); * opal_output(0, "Function took %7.2f usecs\n", accum); * */ #if OPAL_ENABLE_DEBUG static float mydifftime(struct timespec ts_start, struct timespec ts_end) { float seconds; float microseconds; float nanoseconds; /* If we did not rollover the seconds clock, then we just take * the difference between the nanoseconds clock for actual time */ if (0 == (ts_end.tv_sec - ts_start.tv_sec)) { nanoseconds = (float)(ts_end.tv_nsec - ts_start.tv_nsec); return nanoseconds / THOUSAND; } else { seconds = (float)(ts_end.tv_sec - ts_start.tv_sec); /* Note that this value can be negative or positive * which is fine. In the case that it is negative, it * just gets subtracted from the difference which is what * we want. */ nanoseconds = (float)(ts_end.tv_nsec - ts_start.tv_nsec); microseconds = (seconds * MILLION) + (nanoseconds/THOUSAND); return microseconds; } } #endif /* OPAL_ENABLE_DEBUG */ #endif /* OPAL_CUDA_SUPPORT_41 */ /* Routines that get plugged into the opal datatype code */ static int mca_common_cuda_is_gpu_buffer(const void *pUserBuf) { int res; CUmemorytype memType; CUdeviceptr dbuf = (CUdeviceptr)pUserBuf; CUcontext ctx = NULL; res = cuFunc.cuPointerGetAttribute(&memType, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, dbuf); if (res != CUDA_SUCCESS) { /* If we cannot determine it is device pointer, * just assume it is not. */ return 0; } else if (memType == CU_MEMORYTYPE_HOST) { /* Host memory, nothing to do here */ return 0; } /* Must be a device pointer */ assert(memType == CU_MEMORYTYPE_DEVICE); /* This piece of code was added in to handle in a case involving * OMP threads. The user had initialized CUDA and then spawned * two threads. The first thread had the CUDA context, but the * second thread did not. We therefore had no context to act upon * and future CUDA driver calls would fail. Therefore, if we have * GPU memory, but no context, get the context from the GPU memory * and set the current context to that. It is rare that we will not * have a context. */ res = cuFunc.cuCtxGetCurrent(&ctx); if (OPAL_UNLIKELY(NULL == ctx)) { if (CUDA_SUCCESS == res) { res = cuFunc.cuPointerGetAttribute(&ctx, CU_POINTER_ATTRIBUTE_CONTEXT, dbuf); if (res != CUDA_SUCCESS) { opal_output(0, "CUDA: error calling cuPointerGetAttribute: " "res=%d, ptr=%p aborting...", res, pUserBuf); ompi_rte_abort(1, NULL); } else { res = cuFunc.cuCtxSetCurrent(ctx); if (res != CUDA_SUCCESS) { opal_output(0, "CUDA: error calling cuCtxSetCurrent: " "res=%d, ptr=%p aborting...", res, pUserBuf); ompi_rte_abort(1, NULL); } else { opal_output_verbose(10, mca_common_cuda_output, "CUDA: cuCtxSetCurrent passed: ptr=%p", pUserBuf); } } } else { /* Print error and proceed */ opal_output(0, "CUDA: error calling cuCtxGetCurrent: " "res=%d, ptr=%p aborting...", res, pUserBuf); ompi_rte_abort(1, NULL); } } /* First access on a device pointer finalizes CUDA support initialization. * If initialization fails, disable support. */ if (!stage_three_init_complete) { if (0 != mca_common_cuda_stage_three_init()) { ompi_mpi_cuda_support = 0; } } return 1; } static int mca_common_cuda_cu_memcpy_async(void *dest, const void *src, size_t size, opal_convertor_t* convertor) { return cuFunc.cuMemcpyAsync((CUdeviceptr)dest, (CUdeviceptr)src, size, (CUstream)convertor->stream); } /** * This function is plugged into various areas where a cuMemcpy would be called. * This is a synchronous operation that will not return until the copy is complete. */ static int mca_common_cuda_cu_memcpy(void *dest, const void *src, size_t size) { CUresult result; #if OPAL_ENABLE_DEBUG CUmemorytype memTypeSrc, memTypeDst; if (OPAL_UNLIKELY(mca_common_cuda_cumemcpy_timing)) { /* Nice to know type of source and destination for timing output. Do * not care about return code as memory type will just be set to 0 */ result = cuFunc.cuPointerGetAttribute(&memTypeDst, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, (CUdeviceptr)dest); result = cuFunc.cuPointerGetAttribute(&memTypeSrc, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, (CUdeviceptr)src); clock_gettime(CLOCK_MONOTONIC, &ts_start); } #endif if (mca_common_cuda_cumemcpy_async) { result = cuFunc.cuMemcpyAsync((CUdeviceptr)dest, (CUdeviceptr)src, size, memcpyStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed", true, dest, src, size, result); return OMPI_ERROR; } result = cuFunc.cuEventRecord(memcpyEvent, memcpyStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, ompi_process_info.nodename, result); return 0; } result = cuFunc.cuEventSynchronize(memcpyEvent); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventSynchronize failed", true, ompi_process_info.nodename, result); return OMPI_ERROR; } } else { result = cuFunc.cuMemcpy((CUdeviceptr)dest, (CUdeviceptr)src, size); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuMemcpy failed", true, result); return OMPI_ERROR; } } #if OPAL_ENABLE_DEBUG if (OPAL_UNLIKELY(mca_common_cuda_cumemcpy_timing)) { clock_gettime(CLOCK_MONOTONIC, &ts_end); accum = mydifftime(ts_start, ts_end); if (mca_common_cuda_cumemcpy_async) { opal_output(0, "cuMemcpyAsync took %7.2f usecs (src=%p (%d), dst=%p (%d))\n", accum, src, memTypeSrc, dest, memTypeDst); } else { opal_output(0, "cuMemcpy took %7.2f usecs (src=%p (%d), dst=%p (%d))\n", accum, src, memTypeSrc, dest, memTypeDst); } } #endif return OMPI_SUCCESS; } static int mca_common_cuda_memmove(void *dest, void *src, size_t size) { CUdeviceptr tmp; int result; result = cuFunc.cuMemAlloc(&tmp,size); if (mca_common_cuda_cumemcpy_async) { result = cuFunc.cuMemcpyAsync(tmp, (CUdeviceptr)src, size, memcpyStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed", true, tmp, src, size, result); return OMPI_ERROR; } result = cuFunc.cuMemcpyAsync((CUdeviceptr)dest, tmp, size, memcpyStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed", true, dest, tmp, size, result); return OMPI_ERROR; } result = cuFunc.cuEventRecord(memcpyEvent, memcpyStream); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed", true, ompi_process_info.nodename, result); return OMPI_ERROR; } result = cuFunc.cuEventSynchronize(memcpyEvent); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuEventSynchronize failed", true, ompi_process_info.nodename, result); return OMPI_ERROR; } } else { result = cuFunc.cuMemcpy(tmp, (CUdeviceptr)src, size); if (OPAL_UNLIKELY(result != CUDA_SUCCESS)) { opal_output(0, "CUDA: memmove-Error in cuMemcpy: res=%d, dest=%p, src=%p, size=%d", result, (void *)tmp, src, (int)size); return OMPI_ERROR; } result = cuFunc.cuMemcpy((CUdeviceptr)dest, tmp, size); if (OPAL_UNLIKELY(result != CUDA_SUCCESS)) { opal_output(0, "CUDA: memmove-Error in cuMemcpy: res=%d, dest=%p, src=%p, size=%d", result, dest, (void *)tmp, (int)size); return OMPI_ERROR; } } cuFunc.cuMemFree(tmp); return OMPI_SUCCESS; } int mca_common_cuda_get_device(int *devicenum) { CUdevice cuDev; int res; res = cuFunc.cuCtxGetDevice(&cuDev); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_output(0, "CUDA: cuCtxGetDevice failed: res=%d", res); return res; } *devicenum = cuDev; return 0; } int mca_common_cuda_device_can_access_peer(int *access, int dev1, int dev2) { int res; res = cuFunc.cuDeviceCanAccessPeer(access, (CUdevice)dev1, (CUdevice)dev2); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_output(0, "CUDA: cuDeviceCanAccessPeer failed: res=%d", res); return res; } return 0; } int mca_common_cuda_get_address_range(void *pbase, size_t *psize, void *base) { CUresult result; result = cuFunc.cuMemGetAddressRange((CUdeviceptr *)pbase, psize, (CUdeviceptr)base); if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) { opal_show_help("help-mpi-common-cuda.txt", "cuMemGetAddressRange failed", true, result, base); return OMPI_ERROR; } else { opal_output_verbose(50, mca_common_cuda_output, "CUDA: cuMemGetAddressRange passed: addr=%p, pbase=%p, psize=%lu ", base, *(char **)pbase, *psize); } return 0; } #if OPAL_CUDA_GDR_SUPPORT /* Check to see if the memory was freed between the time it was stored in * the registration cache and now. Return true if the memory was previously * freed. This is indicated by the BUFFER_ID value in the registration cache * not matching the BUFFER_ID of the buffer we are checking. Return false * if the registration is still good. */ bool mca_common_cuda_previously_freed_memory(mca_mpool_base_registration_t *reg) { int res; unsigned long long bufID; unsigned char *dbuf = reg->base; res = cuFunc.cuPointerGetAttribute(&bufID, CU_POINTER_ATTRIBUTE_BUFFER_ID, (CUdeviceptr)dbuf); /* If we cannot determine the BUFFER_ID, then print a message and default * to forcing the registration to be kicked out. */ if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_show_help("help-mpi-common-cuda.txt", "bufferID failed", true, ompi_process_info.nodename, res); return true; } opal_output_verbose(50, mca_common_cuda_output, "CUDA: base=%p, bufID=%llu, reg->gpu_bufID=%llu, %s", dbuf, bufID, reg->gpu_bufID, (reg->gpu_bufID == bufID ? "BUFFER_ID match":"BUFFER_ID do not match")); if (bufID != reg->gpu_bufID) { return true; } else { return false; } } /* * Get the buffer ID from the memory and store it in the registration. * This is needed to ensure the cached registration is not stale. If * we fail to get buffer ID, print an error and set buffer ID to 0. * Also set SYNC_MEMOPS on any GPU registration to ensure that * synchronous copies complete before the buffer is accessed. */ void mca_common_cuda_get_buffer_id(mca_mpool_base_registration_t *reg) { int res; unsigned long long bufID = 0; unsigned char *dbuf = reg->base; int enable = 1; res = cuFunc.cuPointerGetAttribute(&bufID, CU_POINTER_ATTRIBUTE_BUFFER_ID, (CUdeviceptr)dbuf); if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) { opal_show_help("help-mpi-common-cuda.txt", "bufferID failed", true, res); } reg->gpu_bufID = bufID; res = cuFunc.cuPointerSetAttribute(&enable, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, (CUdeviceptr)dbuf); if (OPAL_UNLIKELY(CUDA_SUCCESS != res)) { opal_show_help("help-mpi-common-cuda.txt", "cuPointerSetAttribute failed", true, ompi_process_info.nodename, res, dbuf); } } #endif /* OPAL_CUDA_GDR_SUPPORT */