From 3e7ef48c13f090fdf697e75ab44705df85dea1d1 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Tue, 14 Mar 2017 09:00:40 -0600 Subject: [PATCH] pml/ob1: do not cache leave_pinned This commit fixes a bug that disabled both the RDMA pipeline and RDMA protocols in ob1. ob1 was internally caching the values of opal_leave_pinned and opal_leave_pinned_pipeline at init time. This is no longer valid as opal_leave_pinned may be set by any call to a btl's add_procs. Signed-off-by: Nathan Hjelm --- ompi/mca/pml/ob1/pml_ob1.h | 4 +--- ompi/mca/pml/ob1/pml_ob1_component.c | 8 +------- ompi/mca/pml/ob1/pml_ob1_cuda.c | 5 +++-- ompi/mca/pml/ob1/pml_ob1_rdma.c | 7 ++++--- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/ompi/mca/pml/ob1/pml_ob1.h b/ompi/mca/pml/ob1/pml_ob1.h index 6c135faecf..10162916c6 100644 --- a/ompi/mca/pml/ob1/pml_ob1.h +++ b/ompi/mca/pml/ob1/pml_ob1.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -60,9 +60,7 @@ struct mca_pml_ob1_t { size_t rdma_retries_limit; int max_rdma_per_request; int max_send_per_range; - bool leave_pinned; bool use_all_rdma; - int leave_pinned_pipeline; /* lock queue access */ opal_mutex_t lock; diff --git a/ompi/mca/pml/ob1/pml_ob1_component.c b/ompi/mca/pml/ob1/pml_ob1_component.c index e922c18d8f..6557bc2037 100644 --- a/ompi/mca/pml/ob1/pml_ob1_component.c +++ b/ompi/mca/pml/ob1/pml_ob1_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved - * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2013-2017 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -295,12 +295,6 @@ mca_pml_ob1_component_init( int* priority, } - /* Set this here (vs in component_open()) because - opal_leave_pinned* may have been set after MCA params were - read (e.g., by the openib btl) */ - mca_pml_ob1.leave_pinned = (1 == opal_leave_pinned); - mca_pml_ob1.leave_pinned_pipeline = (int) opal_leave_pinned_pipeline; - return &mca_pml_ob1.super; } diff --git a/ompi/mca/pml/ob1/pml_ob1_cuda.c b/ompi/mca/pml/ob1/pml_ob1_cuda.c index 8f3985a0cb..84f6464a75 100644 --- a/ompi/mca/pml/ob1/pml_ob1_cuda.c +++ b/ompi/mca/pml/ob1/pml_ob1_cuda.c @@ -13,7 +13,7 @@ * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012-2015 NVIDIA Corporation. All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -25,6 +25,7 @@ #include "ompi_config.h" #include "opal/prefetch.h" +#include "opal/runtime/opal_params.h" #include "opal/mca/btl/btl.h" #include "opal/mca/mpool/mpool.h" #include "ompi/constants.h" @@ -143,7 +144,7 @@ size_t mca_pml_ob1_rdma_cuda_btls( /* if we don't use leave_pinned and all BTLs that already have this memory * registered amount to less then half of available bandwidth - fall back to * pipeline protocol */ - if(0 == num_btls_used || (!mca_pml_ob1.leave_pinned && weight_total < 0.5)) + if(0 == num_btls_used || (!opal_leave_pinned && weight_total < 0.5)) return 0; mca_pml_ob1_calc_weighted_length(rdma_btls, num_btls_used, size, diff --git a/ompi/mca/pml/ob1/pml_ob1_rdma.c b/ompi/mca/pml/ob1/pml_ob1_rdma.c index 7513b24bf1..8cae10b032 100644 --- a/ompi/mca/pml/ob1/pml_ob1_rdma.c +++ b/ompi/mca/pml/ob1/pml_ob1_rdma.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -27,6 +27,7 @@ #include "ompi/mca/pml/pml.h" #include "ompi/mca/bml/bml.h" #include "opal/mca/mpool/mpool.h" +#include "opal/runtime/opal_params.h" #include "pml_ob1.h" #include "pml_ob1_rdma.h" @@ -79,7 +80,7 @@ size_t mca_pml_ob1_rdma_btls( /* do not use the RDMA protocol with this btl if 1) leave pinned is disabled, * 2) the btl supports put, and 3) the fragment is larger than the minimum * pipeline size specified by the BTL */ - if (!mca_pml_ob1.leave_pinned && (btl->btl_flags & MCA_BTL_FLAGS_PUT) && + if (!opal_leave_pinned && (btl->btl_flags & MCA_BTL_FLAGS_PUT) && size > btl->btl_min_rdma_pipeline_size) { continue; } @@ -102,7 +103,7 @@ size_t mca_pml_ob1_rdma_btls( /* if we don't use leave_pinned and all BTLs that already have this memory * registered amount to less then half of available bandwidth - fall back to * pipeline protocol */ - if (0 == num_btls_used || (!mca_pml_ob1.leave_pinned && weight_total < 0.5)) + if (0 == num_btls_used || (!opal_leave_pinned && weight_total < 0.5)) return 0; mca_pml_ob1_calc_weighted_length(rdma_btls, num_btls_used, size,