From 6180386bea8a534ab008a917452274bd62aca6d1 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Thu, 21 Jan 2016 15:50:35 -0700 Subject: [PATCH] osc/rdma: disable put aggregation when using threads Optimizing put aggregation in the presence of threads will require a redesign of the code. For now just ensure that put aggregation is turned off when MPI_THREAD_MULTIPLE is enabled. Signed-off-by: Nathan Hjelm --- ompi/mca/osc/rdma/osc_rdma_comm.c | 4 ++-- ompi/mca/osc/rdma/osc_rdma_component.c | 25 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ompi/mca/osc/rdma/osc_rdma_comm.c b/ompi/mca/osc/rdma/osc_rdma_comm.c index 3b9f8f8de2..1a2b9dde77 100644 --- a/ompi/mca/osc/rdma/osc_rdma_comm.c +++ b/ompi/mca/osc/rdma/osc_rdma_comm.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -484,7 +484,7 @@ static int ompi_osc_rdma_aggregate_alloc (ompi_osc_rdma_sync_t *sync, ompi_osc_r ompi_osc_rdma_aggregate_append (aggregation, request, source_buffer, size); - OPAL_THREAD_SCOPED_LOCK(&sync->lock, opal_list_append (&sync->aggregations, (opal_list_item_t *) aggregation)); + opal_list_append (&sync->aggregations, (opal_list_item_t *) aggregation); return OMPI_SUCCESS; } diff --git a/ompi/mca/osc/rdma/osc_rdma_component.c b/ompi/mca/osc/rdma/osc_rdma_component.c index 390872f7c7..170c7457a8 100644 --- a/ompi/mca/osc/rdma/osc_rdma_component.c +++ b/ompi/mca/osc/rdma/osc_rdma_component.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2008 University of Houston. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. @@ -265,14 +265,21 @@ static int ompi_osc_rdma_component_init (bool enable_progress_threads, } OBJ_CONSTRUCT(&mca_osc_rdma_component.aggregate, opal_free_list_t); - ret = opal_free_list_init (&mca_osc_rdma_component.aggregate, - sizeof(ompi_osc_rdma_aggregation_t), 8, - OBJ_CLASS(ompi_osc_rdma_aggregation_t), 0, 0, - 32, 128, 32, NULL, 0, NULL, NULL, NULL); - if (OPAL_SUCCESS != ret) { - opal_output_verbose(1, ompi_osc_base_framework.framework_output, - "%s:%d: opal_free_list_init failed: %d\n", - __FILE__, __LINE__, ret); + + if (!enable_mpi_threads && mca_osc_rdma_component.aggregation_limit) { + ret = opal_free_list_init (&mca_osc_rdma_component.aggregate, + sizeof(ompi_osc_rdma_aggregation_t), 8, + OBJ_CLASS(ompi_osc_rdma_aggregation_t), 0, 0, + 32, 128, 32, NULL, 0, NULL, NULL, NULL); + + if (OPAL_SUCCESS != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: opal_free_list_init failed: %d\n", + __FILE__, __LINE__, ret); + } + } else { + /* only enable put aggregation when not using threads */ + mca_osc_rdma_component.aggregation_limit = 0; } return ret;