From cdffa4724d8e2e84a0bbbe2427d267a544d01bdc Mon Sep 17 00:00:00 2001 From: Rolf vandeVaart Date: Wed, 8 Jul 2015 17:11:25 -0400 Subject: [PATCH] Force smcuda BTL to use CUDA IPC path for all GPU buffers where possible --- opal/mca/btl/smcuda/btl_smcuda.c | 7 +++++++ opal/mca/btl/smcuda/btl_smcuda_component.c | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/opal/mca/btl/smcuda/btl_smcuda.c b/opal/mca/btl/smcuda/btl_smcuda.c index a617bffc33..29bac3b3b3 100644 --- a/opal/mca/btl/smcuda/btl_smcuda.c +++ b/opal/mca/btl/smcuda/btl_smcuda.c @@ -885,6 +885,13 @@ int mca_btl_smcuda_sendi( struct mca_btl_base_module_t* btl, if (mca_common_cuda_enabled && (IPC_INIT == endpoint->ipcstate) && mca_btl_smcuda_component.use_cuda_ipc) { mca_btl_smcuda_send_cuda_ipc_request(btl, endpoint); } + /* We do not want to use this path when we have CUDA IPC support */ + if ((convertor->flags & CONVERTOR_CUDA) && (IPC_ACKED == endpoint->ipcstate)) { + if (NULL != descriptor) { + *descriptor = mca_btl_smcuda_alloc(btl, endpoint, order, payload_size+header_size, flags); + } + return OPAL_ERR_RESOURCE_BUSY; + } #endif /* OPAL_CUDA_SUPPORT */ /* this check should be unnecessary... turn into an assertion? */ diff --git a/opal/mca/btl/smcuda/btl_smcuda_component.c b/opal/mca/btl/smcuda/btl_smcuda_component.c index f18e95e079..8ddbd8ac16 100644 --- a/opal/mca/btl/smcuda/btl_smcuda_component.c +++ b/opal/mca/btl/smcuda/btl_smcuda_component.c @@ -190,6 +190,11 @@ static int smcuda_register(void) if (0 == mca_btl_smcuda.super.btl_cuda_max_send_size) { mca_btl_smcuda.super.btl_cuda_max_send_size = 128*1024; } + /* If user has not set the value, then set to magic number which will be converted to the minimum + * size needed to fit the PML header (see pml_ob1.c) */ + if (0 == mca_btl_smcuda.super.btl_cuda_eager_limit) { + mca_btl_smcuda.super.btl_cuda_eager_limit = SIZE_MAX; /* magic number */ + } #endif /* OPAL_CUDA_SUPPORT */ return mca_btl_smcuda_component_verify(); }