Change the way we register the sm memory pool with CUDA. Rather than just registering local free lists, register the entire pool as the local process does not know which memory the remote processes are using for free lists. Fixes performance problem we were seeing with copying out of memory (since host piece was not pinned).

opal/mca/btl/smcuda/btl_smcuda.c

@@ -14,7 +14,7 @@
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010-2014 Los Alamos National Security, LLC.
  *                         All rights reserved. 
- * Copyright (c) 2012-2013 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2012-2014 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -325,6 +325,15 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
         }
     }
 #if OPAL_CUDA_SUPPORT
+    /* Register the entire shared memory region with the CUDA library which will
+     * force it to be pinned.  This aproach was chosen as there is no way for this
+     * local process to know which parts of the memory are being utilized by a 
+     * remote process. */
+    opal_output_verbose(10, opal_btl_base_framework.framework_output,
+                        "btl:smcuda: CUDA cuMemHostRegister address=%p, size=%d",
+                        mca_btl_smcuda_component.sm_mpool_base, (int)res->size);
+    mca_common_cuda_register(mca_btl_smcuda_component.sm_mpool_base, res->size, "smcuda");
+
     /* Create a local memory pool that sends handles to the remote
      * side.  Note that the res argument is not really used, but
      * needed to satisfy function signature. */

opal/mca/mpool/sm/mpool_sm_module.c

@@ -96,13 +96,6 @@ void* mca_mpool_sm_alloc(
 #endif
     }
 
-#if OPAL_CUDA_SUPPORT
-    if ((flags & MCA_MPOOL_FLAGS_CUDA_REGISTER_MEM) && (NULL != mseg.mbs_start_addr)) {
-        mca_common_cuda_register(mseg.mbs_start_addr, size,
-                                 mpool->mpool_component->mpool_version.mca_component_name);
-    }
-#endif
-
     return mseg.mbs_start_addr;
 }