From 68697d9dab08f14b707a50afb5d914277ba0bbe2 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Sun, 1 May 2005 19:55:16 +0000 Subject: [PATCH] * add all the run-time tweakable parameters * start trying to figure out the sendfrag code * start of code to initialize recv buffer This commit was SVN r5569. --- src/mca/ptl/portals/Makefile.am | 4 +- src/mca/ptl/portals/configure.stub | 71 +++++++++++++++++++++ src/mca/ptl/portals/ptl_portals.c | 41 ++++++++++-- src/mca/ptl/portals/ptl_portals.h | 15 ++++- src/mca/ptl/portals/ptl_portals_component.c | 45 +++++++++++-- src/mca/ptl/portals/ptl_portals_sendfrag.c | 26 ++++++++ src/mca/ptl/portals/ptl_portals_sendfrag.h | 43 +++++++++++++ 7 files changed, 230 insertions(+), 15 deletions(-) create mode 100644 src/mca/ptl/portals/ptl_portals_sendfrag.c create mode 100644 src/mca/ptl/portals/ptl_portals_sendfrag.h diff --git a/src/mca/ptl/portals/Makefile.am b/src/mca/ptl/portals/Makefile.am index c94aa4f237..b605178412 100644 --- a/src/mca/ptl/portals/Makefile.am +++ b/src/mca/ptl/portals/Makefile.am @@ -37,10 +37,12 @@ EXTRA_DIST = \ portals_SOURCES = \ ptl_portals.h \ ptl_portals_compat.h \ + ptl_portals_sendfrag.h \ ptl_portals.c \ ptl_portals_component.c \ ptl_portals_stubs.c \ - ptl_portals_compat_utcp.c + ptl_portals_compat_utcp.c \ + ptl_portals_sendfrag.c mcacomponentdir = $(libdir)/openmpi diff --git a/src/mca/ptl/portals/configure.stub b/src/mca/ptl/portals/configure.stub index 22172e8749..973439169a 100644 --- a/src/mca/ptl/portals/configure.stub +++ b/src/mca/ptl/portals/configure.stub @@ -15,6 +15,37 @@ # $HEADER$ # +# +# quicky function to set #defines based on argument values +# +# ARGUMENTS: +# 1 configure name (first argument to ARG_WITH, minus the ptl-portals-) +# 2 define name +# 3 default value +# 4 description (used for both ARG_WITH and DEFINE) +AC_DEFUN([MCA_PTL_PORTALS_CONFIG_VAL], +[ + AC_ARG_WITH([ptl-portals-$1], AC_HELP_STRING([--with-ptl-portals-$1], + [$4 (default: $3)])) + AC_MSG_CHECKING([for $1 value]) + case "[$with_]m4_bpatsubst([ptl-portals-$1], -, _)" in + "") + $2=$3 + AC_MSG_RESULT([[$]$2 (default)]) + ;; + "no") + AC_MSG_RESULT([error]) + AC_MSG_ERROR([--without-ptl-portals-$1 is invalid argument]) + ;; + *) + $2="$with_m4_bpatsubst([ptl-portals-$1], -, _)" + AC_MSG_RESULT([[$]$2]) + ;; + esac + AC_DEFINE_UNQUOTED([$2], [[$]$2], [$4]) +])dnl + + # # Main function. This will be invoked in the middle of the templated # configure script. @@ -113,6 +144,46 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ [Use the Red Storm implementation or Portals]) AM_CONDITIONAL([PTL_PORTALS_REDSTORM], [test "$PTL_PORTALS_REDSTORM" = "1"]) + MCA_PTL_PORTALS_CONFIG_VAL([first-frag-table-id], + [PTL_PORTALS_FIRST_FRAG_TABLE_ID], [1], + [Portals table id to use for first fragment receive queue]) + + MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-table-id], + [PTL_PORTALS_RNDV_FRAG_TABLE_ID], [2], + [Portals table id to use for rndv fragment receive queue]) + + MCA_PTL_PORTALS_CONFIG_VAL([ack-table-id], + [PTL_PORTALS_ACK_TABLE_ID], [3], + [Portals table id to use for ack queue]) + + MCA_PTL_PORTALS_CONFIG_VAL([debug-level], + [PTL_PORTALS_DEFAULT_DEBUG_LEVEL], [1000], + [Default debugging level for portals ptl]) + + MCA_PTL_PORTALS_CONFIG_VAL([request-cache-size], + [PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE], [1], + [Default request cache size for portals ptl]) + + MCA_PTL_PORTALS_CONFIG_VAL([first-frag-size], + [PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE], [16384], + [Default first frag size for portals ptl]) + + MCA_PTL_PORTALS_CONFIG_VAL([first-frag-num-entries], + [PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES], [3], + [Default number of memory descriptors for first fragments]) + + MCA_PTL_PORTALS_CONFIG_VAL([first-frag-entry-size], + [PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE], [1048576], + [Default size of memory associeted with first fag md]) + + MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-min-size], + [PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE], [0], + [Default minimum size of rndv fragments]) + + MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-max-size], + [PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE], [32768], + [Default maximum size of rndv fragments]) + # # Save extra compiler/linker flags so that they can be added in # the wrapper compilers, if necessary diff --git a/src/mca/ptl/portals/ptl_portals.c b/src/mca/ptl/portals/ptl_portals.c index 86da8e1f4a..4becad8e39 100644 --- a/src/mca/ptl/portals/ptl_portals.c +++ b/src/mca/ptl/portals/ptl_portals.c @@ -27,17 +27,18 @@ #include "ptl_portals.h" #include "ptl_portals_compat.h" +#include "ptl_portals_sendfrag.h" mca_ptl_portals_module_t mca_ptl_portals_module = { { &mca_ptl_portals_component.super, - 1, /* max size of request cache */ - 1, /* byest required by ptl for a request */ - 128, /* max size of first frag */ + 0, /* max size of request cache */ + sizeof(mca_ptl_portals_send_frag_t), /* byes required by ptl for a request */ + 0, /* max size of first frag */ 0, /* min size of frag */ - 128, /* max size of frag */ + 0, /* max size of frag */ 0, /* exclusivity */ - 50, /* latency */ + 0, /* latency */ 0, /* bandwidth */ 0, /* ptl flags */ @@ -59,7 +60,9 @@ mca_ptl_portals_module_t mca_ptl_portals_module = { NULL /* PML use */ }, - 0, + 0, /* num first frag mds */ + 0, /* size first frag md */ + 0, /* ni handle */ }; @@ -75,6 +78,8 @@ mca_ptl_portals_add_procs(struct mca_ptl_base_module_t* ptl, struct ompi_proc_t *curr_proc; ptl_process_id_t *portals_procs; size_t i; + unsigned long distance; + struct mca_ptl_portals_module_t *myptl = (struct mca_ptl_portals_module_t*) ptl; /* make sure our environment is fully initialized. At end of this call, we have a working network handle on our module and @@ -90,7 +95,31 @@ mca_ptl_portals_add_procs(struct mca_ptl_base_module_t* ptl, /* BWB - do we want to send to self? No for now */ if (curr_proc == local_proc) continue; + /* make sure we can reach the process */ + ret = PtlNIDist(myptl->ni_handle, + portals_procs[i], + &distance); + if (ret != PTL_OK) { + ompi_output_verbose(100, mca_ptl_portals_component.portals_output, + "Could not find distance to process %d", i); + continue; + } + + /* set the peer as a pointer to the address */ + peers[i] = (struct mca_ptl_base_peer_t*) &(portals_procs[i]); + + /* and here we can reach */ + ompi_bitmap_set_bit(reachable, i); } return OMPI_SUCCESS; } + + +int +mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, + int value) +{ + /* need to do all the portals cleanup code here... */ + return OMPI_SUCCESS; +} diff --git a/src/mca/ptl/portals/ptl_portals.h b/src/mca/ptl/portals/ptl_portals.h index 91296cad7a..bd70c82c4e 100644 --- a/src/mca/ptl/portals/ptl_portals.h +++ b/src/mca/ptl/portals/ptl_portals.h @@ -49,14 +49,19 @@ struct mca_ptl_portals_component_t { */ int portals_output; +#if PTL_PORTALS_UTCP /** ethernet interface to use - only has meaning with utcp reference */ char *portals_ifname; +#endif /** Number of currently active portals modules */ uint32_t portals_num_modules; - /** List of currently running modules */ + /** List of currently available modules */ struct mca_ptl_portals_module_t **portals_modules; + + /** lock for accessing component */ + ompi_mutex_t portals_lock; }; struct mca_ptl_portals_recv_frag_t; @@ -142,6 +147,11 @@ extern int mca_ptl_portals_component_progress( struct mca_ptl_portals_module_t { mca_ptl_base_module_t super; /**< base PTL module interface */ + /* number of mds for first frags */ + int first_frag_num_mds; + /* size of each md for first frags */ + int first_frag_md_size; + /** our portals network interface */ ptl_handle_ni_t ni_handle; /** the limits returned from PtlNIInit for interface */ @@ -357,4 +367,7 @@ extern int mca_ptl_portals_send_continue( ); +extern int mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl, + int value); + #endif diff --git a/src/mca/ptl/portals/ptl_portals_component.c b/src/mca/ptl/portals/ptl_portals_component.c index 99150a1cfd..7eac75a0c8 100644 --- a/src/mca/ptl/portals/ptl_portals_component.c +++ b/src/mca/ptl/portals/ptl_portals_component.c @@ -115,6 +115,7 @@ mca_ptl_portals_component_open(void) mca_ptl_portals_component.portals_modules = NULL; /* initialize objects */ + OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_lock, ompi_mutex_t); /* register portals module parameters */ #if PTL_PORTALS_UTCP @@ -122,7 +123,28 @@ mca_ptl_portals_component_open(void) mca_ptl_portals_param_register_string("ifname", "eth0"); #endif portals_output_stream.lds_verbose_level = - mca_ptl_portals_param_register_int("debug_level", 1000); + mca_ptl_portals_param_register_int("debug_level", + PTL_PORTALS_DEFAULT_DEBUG_LEVEL); + + mca_ptl_portals_module.super.ptl_cache_size = + mca_ptl_portals_param_register_int("request_cache_size", + PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE); + mca_ptl_portals_module.super.ptl_first_frag_size = + mca_ptl_portals_param_register_int("first_frag_size", + PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE); + mca_ptl_portals_module.super.ptl_min_frag_size = + mca_ptl_portals_param_register_int("rndv_frag_min_size", + PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE); + mca_ptl_portals_module.super.ptl_max_frag_size = + mca_ptl_portals_param_register_int("rndv_frag_max_size", + PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE); + + mca_ptl_portals_module.first_frag_num_mds = + mca_ptl_portals_param_register_int("first_frag_num_entries", + PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES); + mca_ptl_portals_module.first_frag_md_size = + mca_ptl_portals_param_register_int("first_frag_entry_size", + PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE); /* finish with objects */ mca_ptl_portals_component.portals_output = @@ -149,6 +171,8 @@ mca_ptl_portals_component_close(void) /* print out debugging if anything is pending */ /* release resources */ + OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock); + if (NULL != mca_ptl_portals_component.portals_ifname) { free(mca_ptl_portals_component.portals_ifname); } @@ -201,20 +225,27 @@ mca_ptl_portals_component_init(int *num_ptls, int mca_ptl_portals_component_control(int param, void* value, size_t size) { + uint32_t i; + int ret = OMPI_SUCCESS; + ompi_output_verbose(100, mca_ptl_portals_component.portals_output, - "mca_ptl_portals_component_control(%d)", param); + "mca_ptl_portals_component_control(%d, %d)", + param, (*(int*) value)); switch(param) { case MCA_PTL_ENABLE: - if(*(int*)value) { - /* BWB - enable the ptl */ - } else - /* BWB - disable the ptl */ + OMPI_THREAD_LOCK(&mca_ptl_portals_component.portals_lock); + for (i = 0 ; i < mca_ptl_portals_component.portals_num_modules ; ++i) { + ret = mca_ptl_portals_module_enable(mca_ptl_portals_component.portals_modules[i], + *(int*)value); + if (ret != OMPI_SUCCESS) break; + } + OMPI_THREAD_UNLOCK(&mca_ptl_portals_component.portals_lock); break; default: break; } - return OMPI_SUCCESS; + return ret; } diff --git a/src/mca/ptl/portals/ptl_portals_sendfrag.c b/src/mca/ptl/portals/ptl_portals_sendfrag.c new file mode 100644 index 0000000000..28125dd609 --- /dev/null +++ b/src/mca/ptl/portals/ptl_portals_sendfrag.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "portals_config.h" + +#include "ptl_portals.h" +#include "ptl_portals_compat.h" +#include "ptl_portals_sendfrag.h" + +OBJ_CLASS_INSTANCE(mca_ptl_portals_send_frag_t, + mca_ptl_base_send_frag_t, + NULL, NULL); diff --git a/src/mca/ptl/portals/ptl_portals_sendfrag.h b/src/mca/ptl/portals/ptl_portals_sendfrag.h new file mode 100644 index 0000000000..02f649077c --- /dev/null +++ b/src/mca/ptl/portals/ptl_portals_sendfrag.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#ifndef MCA_PTL_PORTALS_SENDFRAG_H_ +#define MCA_PTL_PORTALS_SENDFRAG_H_ + +#include "mca/ptl/base/ptl_base_sendfrag.h" + + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + + struct mca_ptl_portals_send_frag_t { + mca_ptl_base_send_frag_t frag_send; + ptl_md_t memory_descriptor; /* BWB - not sure if we need this */ + ptl_handle_eq_t eq_handle; + int status; + }; + typedef struct mca_ptl_portals_send_frag_t mca_ptl_portals_send_frag_t; + + OBJ_CLASS_DECLARATION (mca_ptl_portals_send_frag_t); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + + +#endif /* MCA_PTL_PORTALS_SENDFRAG_H_ */