diff --git a/ompi/mca/mpool/base/mpool_base_open.c b/ompi/mca/mpool/base/mpool_base_open.c index b7a01e2d5d..6b8eca3257 100644 --- a/ompi/mca/mpool/base/mpool_base_open.c +++ b/ompi/mca/mpool/base/mpool_base_open.c @@ -9,6 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -22,6 +23,7 @@ #include "opal/mca/mca.h" #include "opal/mca/base/base.h" +#include "ompi/runtime/params.h" #include "ompi/mca/mpool/mpool.h" #include "ompi/mca/mpool/base/base.h" #include "ompi/constants.h" @@ -128,20 +130,10 @@ int mca_mpool_base_open(void) #else if(0 == mca_mpool_base_use_mem_hooks ) { #endif - int param; - mca_base_param_register_int("mpi", NULL, "leave_pinned", "leave_pinned", 0); - param = mca_base_param_find("mpi", NULL, "leave_pinned"); - mca_base_param_lookup_int(param, &mca_mpool_base_use_mem_hooks); - - if(0 == mca_mpool_base_use_mem_hooks) { - /* and now check leave_pinned_pipeline if necessary */ - mca_base_param_register_int("mpi", NULL, - "leave_pinned_pipeline", - "leave_pinned_pipeline", 0); - param = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline"); - mca_base_param_lookup_int(param, &mca_mpool_base_use_mem_hooks); - } - } + if (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline) { + mca_mpool_base_use_mem_hooks = 1; + } + } /* get the page size for this architecture*/ mca_mpool_base_page_size = sysconf(_SC_PAGESIZE); diff --git a/ompi/mca/mpool/rdma/mpool_rdma_component.c b/ompi/mca/mpool/rdma/mpool_rdma_component.c index e3c68190b5..f36b252f5c 100644 --- a/ompi/mca/mpool/rdma/mpool_rdma_component.c +++ b/ompi/mca/mpool/rdma/mpool_rdma_component.c @@ -10,6 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006 Voltaire. All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -25,6 +26,7 @@ #include "mpool_rdma.h" #include "orte/util/proc_info.h" #include "orte/util/sys_info.h" +#include "ompi/runtime/params.h" #include #ifdef HAVE_MALLOC_H #include @@ -72,7 +74,7 @@ mca_mpool_rdma_component_t mca_mpool_rdma_component = { */ static int mca_mpool_rdma_open(void) { - int param, val; + int val; mca_base_param_reg_string(&mca_mpool_rdma_component.super.mpool_version, "rcache_name", @@ -93,17 +95,8 @@ static int mca_mpool_rdma_open(void) mca_mpool_rdma_component.print_stats = val?true:false; - mca_base_param_register_int("mpi", NULL, "leave_pinned", "leave_pinned", 0); - param = mca_base_param_find("mpi", NULL, "leave_pinned"); - mca_base_param_lookup_int(param, (int*)&mca_mpool_rdma_component.leave_pinned); - - if(0 == mca_mpool_rdma_component.leave_pinned) { - /* and now check leave_pinned_pipeline if necessary */ - mca_base_param_register_int("mpi", NULL, "leave_pinned_pipeline", - "leave_pinned_pipeline", 0); - param = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline"); - mca_base_param_lookup_int(param, (int*)&mca_mpool_rdma_component.leave_pinned); - } + mca_mpool_rdma_component.leave_pinned = (int) + (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline); return OMPI_SUCCESS; } diff --git a/ompi/mca/pml/ob1/pml_ob1_component.c b/ompi/mca/pml/ob1/pml_ob1_component.c index 8549a594e3..7cf569af75 100644 --- a/ompi/mca/pml/ob1/pml_ob1_component.c +++ b/ompi/mca/pml/ob1/pml_ob1_component.c @@ -9,6 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -20,6 +21,7 @@ #include "opal/sys/cache.h" #include "opal/event/event.h" #include "mpi.h" +#include "ompi/runtime/params.h" #include "ompi/mca/pml/pml.h" #include "ompi/mca/btl/btl.h" #include "ompi/mca/btl/base/base.h" @@ -86,8 +88,6 @@ static inline int mca_pml_ob1_param_register_int( int mca_pml_ob1_component_open(void) { - int param, value; - mca_pml_ob1.free_list_num = mca_pml_ob1_param_register_int("free_list_num", 4); mca_pml_ob1.free_list_max = @@ -166,21 +166,10 @@ int mca_pml_ob1_component_open(void) OBJ_CONSTRUCT(&mca_pml_ob1.recv_pending, opal_list_t); OBJ_CONSTRUCT(&mca_pml_ob1.pckt_pending, opal_list_t); OBJ_CONSTRUCT(&mca_pml_ob1.rdma_pending, opal_list_t); - - mca_base_param_register_int("mpi", NULL, "leave_pinned", "leave_pinned", 0); - param = mca_base_param_find("mpi", NULL, "leave_pinned"); - mca_base_param_lookup_int(param, &value); - mca_pml_ob1.leave_pinned = OPAL_INT_TO_BOOL(value); - - mca_base_param_register_int("mpi", NULL, "leave_pinned_pipeline", "leave_pinned_pipeline", 0); - param = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline"); - mca_base_param_lookup_int(param, &value); - mca_pml_ob1.leave_pinned_pipeline = value; - if(mca_pml_ob1.leave_pinned_pipeline && mca_pml_ob1.leave_pinned) { - mca_pml_ob1.leave_pinned_pipeline = 0; - opal_output(0, "WARNING: Cannot set both mpi_leave_pinned and mpi_leave_pinned_pipeline, defaulting to mpi_leave_pinned ONLY\n"); - } + mca_pml_ob1.leave_pinned = ompi_mpi_leave_pinned; + mca_pml_ob1.leave_pinned_pipeline = (int) ompi_mpi_leave_pinned_pipeline; + mca_pml_ob1.enabled = false; return mca_bml_base_open(); diff --git a/ompi/runtime/help-mpi-runtime.txt b/ompi/runtime/help-mpi-runtime.txt index 325c44fa50..79feb0ac36 100644 --- a/ompi/runtime/help-mpi-runtime.txt +++ b/ompi/runtime/help-mpi-runtime.txt @@ -10,6 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. +# Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -32,6 +33,10 @@ developer): WARNING: The MCA parameter mpi_param_check has been set to true, but parameter checking has been compiled out of Open MPI. The mpi_param_check value has therefore been ignored. +[mpi-params:leave-pinned-and-pipeline-selected] +WARNING: Cannot set both the MCA parameters mpi_leave_pinned and +mpi_leave_pinned_pipeline to "true". Defaulting to mpi_leave_pinned +ONLY. [mpi_init:startup:paffinity-unavailable] The MCA parameter "mpi_paffinity_alone" was set to a nonzero value, but Open MPI was unable to bind MPI_COMM_WORLD rank %s to a processor. diff --git a/ompi/runtime/ompi_mpi_params.c b/ompi/runtime/ompi_mpi_params.c index 4f4481da12..f58ebf16d4 100644 --- a/ompi/runtime/ompi_mpi_params.c +++ b/ompi/runtime/ompi_mpi_params.c @@ -50,6 +50,8 @@ bool ompi_mpi_abort_print_stack = false; int ompi_mpi_abort_delay = 0; bool ompi_mpi_keep_peer_hostnames = true; bool ompi_mpi_preconnect_all = false; +bool ompi_mpi_leave_pinned = false; +bool ompi_mpi_leave_pinned_pipeline = false; int ompi_mpi_register_params(void) @@ -187,7 +189,28 @@ int ompi_mpi_register_params(void) (int) ompi_mpi_preconnect_all, &value); ompi_mpi_preconnect_all = OPAL_INT_TO_BOOL(value); + + /* Leave pinned parameter */ + + mca_base_param_reg_int_name("mpi", "leave_pinned", + "Whether to use the \"leave pinned\" protocol or not. Enabling this setting can help bandwidth performance when repeatedly sending and receiving large messages with the same buffers over RDMA-based networks.", + false, false, + (int) ompi_mpi_leave_pinned, &value); + ompi_mpi_leave_pinned = OPAL_INT_TO_BOOL(value); + + mca_base_param_reg_int_name("mpi", "leave_pinned_pipeline", + "Whether to use the \"leave pinned pipeline\" protocol or not.", + false, false, + (int) ompi_mpi_leave_pinned_pipeline, &value); + ompi_mpi_leave_pinned_pipeline = OPAL_INT_TO_BOOL(value); + if (ompi_mpi_leave_pinned && ompi_mpi_leave_pinned_pipeline) { + ompi_mpi_leave_pinned_pipeline = 0; + opal_show_help("help-mpi-runtime.txt", + "mpi-params:leave-pinned-and-pipeline-selected", + true); + } + /* The ddt engine has a few parameters */ return ompi_ddt_register_params(); diff --git a/ompi/runtime/params.h b/ompi/runtime/params.h index 0b52468062..4d43098320 100644 --- a/ompi/runtime/params.h +++ b/ompi/runtime/params.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -118,6 +118,16 @@ OMPI_DECLSPEC extern bool ompi_mpi_paffinity_alone; */ OMPI_DECLSPEC extern int ompi_mpi_abort_delay; + /** + * Whether to use the "leave pinned" protocol or not. + */ + OMPI_DECLSPEC extern bool ompi_mpi_leave_pinned; + + /** + * Whether to use the "leave pinned pipeline" protocol or not. + */ + OMPI_DECLSPEC extern bool ompi_mpi_leave_pinned_pipeline; + /** * Register MCA parameters used by the MPI layer. *