From d9fa62d1f2bc699e042b2372db9e70a75f25e5d9 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Tue, 5 Jul 2005 16:29:57 +0000 Subject: [PATCH] * checkpoint while it compiles This commit was SVN r6349. --- ompi/mca/btl/portals/Makefile.am | 8 + ompi/mca/btl/portals/configure.params | 2 +- ompi/mca/btl/portals/configure.stub | 84 +++++----- ompi/mca/btl/portals/src/btl_portals.c | 6 +- ompi/mca/btl/portals/src/btl_portals.h | 33 ++-- ompi/mca/btl/portals/src/btl_portals_compat.h | 10 +- .../btl/portals/src/btl_portals_compat_utcp.c | 7 +- .../btl/portals/src/btl_portals_component.c | 143 ++++++++++-------- ompi/mca/btl/portals/src/btl_portals_frag.c | 75 +++++++++ ompi/mca/btl/portals/src/btl_portals_frag.h | 106 +++++++++++++ ompi/mca/btl/portals/src/btl_portals_rdma.c | 32 ++++ ompi/mca/btl/portals/src/btl_portals_rdma.h | 24 +++ ompi/mca/btl/portals/src/btl_portals_recv.c | 32 ++++ ompi/mca/btl/portals/src/btl_portals_recv.h | 24 +++ ompi/mca/btl/portals/src/btl_portals_send.c | 32 ++++ ompi/mca/btl/portals/src/btl_portals_send.h | 24 +++ 16 files changed, 512 insertions(+), 130 deletions(-) create mode 100644 ompi/mca/btl/portals/src/btl_portals_frag.c create mode 100644 ompi/mca/btl/portals/src/btl_portals_frag.h create mode 100644 ompi/mca/btl/portals/src/btl_portals_rdma.c create mode 100644 ompi/mca/btl/portals/src/btl_portals_rdma.h create mode 100644 ompi/mca/btl/portals/src/btl_portals_recv.c create mode 100644 ompi/mca/btl/portals/src/btl_portals_recv.h create mode 100644 ompi/mca/btl/portals/src/btl_portals_send.c create mode 100644 ompi/mca/btl/portals/src/btl_portals_send.h diff --git a/ompi/mca/btl/portals/Makefile.am b/ompi/mca/btl/portals/Makefile.am index b469cb685e..774bec9aac 100644 --- a/ompi/mca/btl/portals/Makefile.am +++ b/ompi/mca/btl/portals/Makefile.am @@ -37,8 +37,16 @@ EXTRA_DIST = \ portals_SOURCES = \ src/btl_portals.h \ src/btl_portals_compat.h \ + src/btl_portals_frag.h \ + src/btl_portals_send.h \ + src/btl_portals_recv.h \ + src/btl_portals_rdma.h \ src/btl_portals.c \ src/btl_portals_component.c \ + src/btl_portals_frag.c \ + src/btl_portals_send.c \ + src/btl_portals_recv.c \ + src/btl_portals_rdma.c \ src/btl_portals_stubs.c \ src/btl_portals_compat_utcp.c diff --git a/ompi/mca/btl/portals/configure.params b/ompi/mca/btl/portals/configure.params index df410de9bb..535f6541f0 100644 --- a/ompi/mca/btl/portals/configure.params +++ b/ompi/mca/btl/portals/configure.params @@ -18,5 +18,5 @@ # Specific to this module PARAM_INIT_FILE=src/btl_portals.h -PARAM_CONFIG_HEADER_FILE="portals_config.h" +PARAM_CONFIG_HEADER_FILE="src/portals_config.h" PARAM_CONFIG_FILES="Makefile" diff --git a/ompi/mca/btl/portals/configure.stub b/ompi/mca/btl/portals/configure.stub index 4a830b1973..00b5222bbf 100644 --- a/ompi/mca/btl/portals/configure.stub +++ b/ompi/mca/btl/portals/configure.stub @@ -23,7 +23,7 @@ # 2 define name # 3 default value # 4 description (used for both ARG_WITH and DEFINE) -AC_DEFUN([MCA_BMI_PORTALS_CONFIG_VAL], +AC_DEFUN([MCA_BTL_PORTALS_CONFIG_VAL], [ AC_ARG_WITH([btl-portals-$1], AC_HELP_STRING([--with-btl-portals-$1], [$4 (default: $3)])) @@ -89,10 +89,10 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ # # Configure Portals for our local environment # - BMI_PORTALS_UTCP=0 - BMI_PORTALS_REDSTORM=0 - BMI_PORTALS_COMPAT="" - BMI_PORTALS_HAVE_EVENT_UNLINK=0 + BTL_PORTALS_UTCP=0 + BTL_PORTALS_REDSTORM=0 + BTL_PORTALS_COMPAT="" + BTL_PORTALS_HAVE_EVENT_UNLINK=0 AC_ARG_WITH([btl-portals-config], AC_HELP_STRING([--with-btl-portals-config], @@ -104,15 +104,15 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ fi case "$with_btl_portals_config" in "utcp") - BMI_PORTALS_UTCP=1 + BTL_PORTALS_UTCP=1 PORTALS_LIBS="-lutcpapi -lutcplib -lp3api -lp3lib -lp3rt" - BMI_PORTALS_HAVE_EVENT_UNLINK=1 + BTL_PORTALS_HAVE_EVENT_UNLINK=1 AC_MSG_RESULT([utcp]) ;; "redstorm") - BMI_PORTALS_REDSTORM=1 + BTL_PORTALS_REDSTORM=1 PORTALS_LIBS="-lp3api -lp3lib -lp3rt" - BMI_PORTALS_HAVE_EVENT_UNLINK=0 + BTL_PORTALS_HAVE_EVENT_UNLINK=0 AC_MSG_RESULT([red storm]) ;; *) @@ -139,66 +139,66 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[ [AC_MSG_RESULT([no]) AC_MSG_ERROR([Can not link with Portals libraries])]) - AC_DEFINE_UNQUOTED([BMI_PORTALS_HAVE_EVENT_UNLINK], - [$BMI_PORTALS_HAVE_EVENT_UNLINK], - [Does Portals send a BMI_EVENT_UNLINK event]) + AC_DEFINE_UNQUOTED([BTL_PORTALS_HAVE_EVENT_UNLINK], + [$BTL_PORTALS_HAVE_EVENT_UNLINK], + [Does Portals send a BTL_EVENT_UNLINK event]) - AC_DEFINE_UNQUOTED([BMI_PORTALS_UTCP], [$BMI_PORTALS_UTCP], + AC_DEFINE_UNQUOTED([BTL_PORTALS_UTCP], [$BTL_PORTALS_UTCP], [Use the UTCP reference implementation or Portals]) - AM_CONDITIONAL([BMI_PORTALS_UTCP], [test "$BMI_PORTALS_UTCP" = "1"]) + AM_CONDITIONAL([BTL_PORTALS_UTCP], [test "$BTL_PORTALS_UTCP" = "1"]) - AC_DEFINE_UNQUOTED([BMI_PORTALS_REDSTORM], [$BMI_PORTALS_REDSTORM], + AC_DEFINE_UNQUOTED([BTL_PORTALS_REDSTORM], [$BTL_PORTALS_REDSTORM], [Use the Red Storm implementation or Portals]) - AM_CONDITIONAL([BMI_PORTALS_REDSTORM], [test "$BMI_PORTALS_REDSTORM" = "1"]) + AM_CONDITIONAL([BTL_PORTALS_REDSTORM], [test "$BTL_PORTALS_REDSTORM" = "1"]) # # User configuration options # - MCA_BMI_PORTALS_CONFIG_VAL([send-table-id], - [BMI_PORTALS_SEND_TABLE_ID], [1], + MCA_BTL_PORTALS_CONFIG_VAL([send-table-id], + [BTL_PORTALS_SEND_TABLE_ID], [3], [Portals table id to use for send/recv ]) - MCA_BMI_PORTALS_CONFIG_VAL([rdma-table-id], - [BMI_PORTALS_RDMA_TABLE_ID], [2], + MCA_BTL_PORTALS_CONFIG_VAL([rdma-table-id], + [BTL_PORTALS_RDMA_TABLE_ID], [4], [Portals table id to use for RDMA request]) - MCA_BMI_PORTALS_CONFIG_VAL([debug-level], - [BMI_PORTALS_DEFAULT_DEBUG_LEVEL], [100], + MCA_BTL_PORTALS_CONFIG_VAL([debug-level], + [BTL_PORTALS_DEFAULT_DEBUG_LEVEL], [100], [debugging level for portals btl]) - MCA_BMI_PORTALS_CONFIG_VAL([eager-limit], - [BMI_PORTALS_DEFAULT_EAGER_LIMIT], [16384], + MCA_BTL_PORTALS_CONFIG_VAL([eager-limit], + [BTL_PORTALS_DEFAULT_EAGER_LIMIT], [16384], [max size for eager sends]) - MCA_BMI_PORTALS_CONFIG_VAL([min-send-size], - [BMI_PORTALS_DEFAULT_MIN_SEND_SIZE], [0], + MCA_BTL_PORTALS_CONFIG_VAL([min-send-size], + [BTL_PORTALS_DEFAULT_MIN_SEND_SIZE], [0], [min size for send fragments]) - MCA_BMI_PORTALS_CONFIG_VAL([max-send-size], - [BMI_PORTALS_DEFAULT_MAX_SEND_SIZE], [0], + MCA_BTL_PORTALS_CONFIG_VAL([max-send-size], + [BTL_PORTALS_DEFAULT_MAX_SEND_SIZE], [32768], [max size for send fragments]) - MCA_BMI_PORTALS_CONFIG_VAL([min-rdma-size], - [BMI_PORTALS_DEFAULT_MIN_RDMA_SIZE], [0], + MCA_BTL_PORTALS_CONFIG_VAL([min-rdma-size], + [BTL_PORTALS_DEFAULT_MIN_RDMA_SIZE], [0], [min size for rdma fragments]) - MCA_BMI_PORTALS_CONFIG_VAL([max-rdma-size], - [BMI_PORTALS_DEFAULT_MAX_RDMA_SIZE], [0], + MCA_BTL_PORTALS_CONFIG_VAL([max-rdma-size], + [BTL_PORTALS_DEFAULT_MAX_RDMA_SIZE], [2147483647], [max size for rdma fragments]) - MCA_BMI_PORTALS_CONFIG_VAL([max-sends-pending], - [BMI_PORTALS_MAX_SENDS_PENDING], [0], + MCA_BTL_PORTALS_CONFIG_VAL([max-sends-pending], + [BTL_PORTALS_MAX_SENDS_PENDING], [128], [max number of sends pending at any time]) - MCA_BMI_PORTALS_CONFIG_VAL([recv-queue-size], - [BMI_PORTALS_DEFAULT_RECV_QUEUE_SIZE], [512], + MCA_BTL_PORTALS_CONFIG_VAL([recv-queue-size], + [BTL_PORTALS_DEFAULT_RECV_QUEUE_SIZE], [512], [size of event queue for receiving frags]) - MCA_BMI_PORTALS_CONFIG_VAL([free-list-init-num], - [BMI_PORTALS_DEFAULT_FREE_LIST_INIT_NUM], [8], + MCA_BTL_PORTALS_CONFIG_VAL([free-list-init-num], + [BTL_PORTALS_DEFAULT_FREE_LIST_INIT_NUM], [8], [starting size of free lists]) - MCA_BMI_PORTALS_CONFIG_VAL([free-list-max-num], - [BMI_PORTALS_DEFAULT_FREE_LIST_MAX_NUM], [1024], + MCA_BTL_PORTALS_CONFIG_VAL([free-list-max-num], + [BTL_PORTALS_DEFAULT_FREE_LIST_MAX_NUM], [1024], [maximum size of free lists]) - MCA_BMI_PORTALS_CONFIG_VAL([free-list-inc-num], - [BMI_PORTALS_DEFAULT_FREE_LIST_INC_NUM], [32], + MCA_BTL_PORTALS_CONFIG_VAL([free-list-inc-num], + [BTL_PORTALS_DEFAULT_FREE_LIST_INC_NUM], [32], [grow size for freelists]) # diff --git a/ompi/mca/btl/portals/src/btl_portals.c b/ompi/mca/btl/portals/src/btl_portals.c index 757a1e7754..6d8fe0a549 100644 --- a/ompi/mca/btl/portals/src/btl_portals.c +++ b/ompi/mca/btl/portals/src/btl_portals.c @@ -94,7 +94,7 @@ mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl, if (curr_proc == local_proc) continue; /* make sure we can reach the process */ - ret = PtlNIDist(mybtl->ni_handle, + ret = PtlNIDist(mybtl->portals_ni_h, portals_procs[i], &distance); if (ret != PTL_OK) { @@ -134,8 +134,8 @@ mca_btl_portals_finalize(struct mca_btl_base_module_t *btl_base) (struct mca_btl_portals_module_t *) btl_base; int ret; - if (PTL_INVALID_HANDLE != btl->ni_handle) { - ret = PtlNIFini(btl->ni_handle); + if (PTL_INVALID_HANDLE != btl->portals_ni_h) { + ret = PtlNIFini(btl->portals_ni_h); if (PTL_OK != ret) { opal_output_verbose(20, mca_btl_portals_component.portals_output, "PtlNIFini returned %d", ret); diff --git a/ompi/mca/btl/portals/src/btl_portals.h b/ompi/mca/btl/portals/src/btl_portals.h index 5cb7ca068d..eb0e83ed17 100644 --- a/ompi/mca/btl/portals/src/btl_portals.h +++ b/ompi/mca/btl/portals/src/btl_portals.h @@ -16,23 +16,24 @@ /* * @file */ -#ifndef MCA_BMI_PORTALS_H -#define MCA_BMI_PORTALS_H +#ifndef MCA_BTL_PORTALS_H +#define MCA_BTL_PORTALS_H #include -#include "mca/pml/pml.h" -#include "mca/btl/btl.h" +#include "ompi/mca/pml/pml.h" +#include "ompi/mca/btl/btl.h" +#include "ompi/mca/btl/base/base.h" #include "class/ompi_bitmap.h" #include "class/ompi_free_list.h" #include "class/ompi_proc_table.h" /* - * Portals BMI component. + * Portals BTL component. */ struct mca_btl_portals_component_t { - /* base BMI component */ + /* base BTL component */ mca_btl_base_component_1_0_0_t super; /* output channel for debugging. Value settings when using @@ -47,7 +48,7 @@ struct mca_btl_portals_component_t { */ int portals_output; -#if BMI_PORTALS_UTCP +#if BTL_PORTALS_UTCP /* ethernet interface to use - only has meaning with utcp reference */ char *portals_ifname; @@ -73,17 +74,17 @@ struct mca_btl_portals_component_t { typedef struct mca_btl_portals_component_t mca_btl_portals_component_t; -#define MCA_BMI_PORTALS_EQ_RECV 0 -#define MCA_BMI_PORTALS_EQ_SEND 1 -#define MCA_BMI_PORTALS_EQ_RDMA 2 -#define MCA_BMI_PORTALS_EQ_SIZE 3 +#define MCA_BTL_PORTALS_EQ_RECV 0 +#define MCA_BTL_PORTALS_EQ_SEND 1 +#define MCA_BTL_PORTALS_EQ_RDMA 2 +#define MCA_BTL_PORTALS_EQ_SIZE 3 struct mca_btl_portals_module_t { - /* base BMI module interface */ + /* base BTL module interface */ mca_btl_base_module_t super; /* registered callbacks */ - mca_btl_base_recv_reg_t portals_reg[MCA_BMI_TAG_MAX]; + mca_btl_base_recv_reg_t portals_reg[MCA_BTL_TAG_MAX]; ompi_free_list_t portals_frag_eager; ompi_free_list_t portals_frag_max; @@ -95,9 +96,9 @@ struct mca_btl_portals_module_t { int portals_recv_mds_size; /* size for event queue */ - int portals_eq_sizes[MCA_BMI_PORTALS_EQ_SIZE]; + int portals_eq_sizes[MCA_BTL_PORTALS_EQ_SIZE]; /* frag receive event queue */ - ptl_handle_eq_t portals_eq_handles[MCA_BMI_PORTALS_EQ_SIZE]; + ptl_handle_eq_t portals_eq_handles[MCA_BTL_PORTALS_EQ_SIZE]; /* our portals network interface */ ptl_handle_ni_t portals_ni_h; @@ -125,7 +126,7 @@ int mca_btl_portals_component_progress(void); /* * Compatibility functions (btl_portals_compat_{}.c) * - * Not part of the BMI interface. Need to be implemented for every + * Not part of the BTL interface. Need to be implemented for every * version of Portals */ int mca_btl_portals_init(mca_btl_portals_component_t *comp); diff --git a/ompi/mca/btl/portals/src/btl_portals_compat.h b/ompi/mca/btl/portals/src/btl_portals_compat.h index a62f2e29db..ffefa5b147 100644 --- a/ompi/mca/btl/portals/src/btl_portals_compat.h +++ b/ompi/mca/btl/portals/src/btl_portals_compat.h @@ -15,16 +15,16 @@ */ -#ifndef BMI_PORTALS_COMPAT_H -#define BMI_PORTALS_COMPAT_H +#ifndef BTL_PORTALS_COMPAT_H +#define BTL_PORTALS_COMPAT_H -#if BMI_PORTALS_UTCP +#if BTL_PORTALS_UTCP #include #include #include -#elif BMI_PORTALS_REDSTORM +#elif BTL_PORTALS_REDSTORM #error "Red Storm Compatibility not implemented" @@ -34,4 +34,4 @@ #endif -#endif /* BMI_PORTALS_NAL_H */ +#endif /* BTL_PORTALS_NAL_H */ diff --git a/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c b/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c index 0ecfd9cf82..b150fc706d 100644 --- a/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c +++ b/ompi/mca/btl/portals/src/btl_portals_compat_utcp.c @@ -44,7 +44,8 @@ int mca_btl_portals_init(mca_btl_portals_component_t *comp) { ptl_process_id_t info; - int ret, i; + int ret; + uint32_t i; #if 0 FILE *output; char *tmp; @@ -90,7 +91,7 @@ mca_btl_portals_init(mca_btl_portals_component_t *comp) Portals-specific entries */ for (i = 0 ; i < comp->portals_num_modules ; ++i) { memcpy(&(comp->portals_modules[i]), - mca_btl_portals_module, + &mca_btl_portals_module, sizeof(mca_btl_portals_module_t)); /* the defaults are good enough for the rest */ } @@ -205,7 +206,7 @@ mca_btl_portals_add_procs_compat(struct mca_btl_portals_module_t* btl, ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ PTL_PID_ANY, /* let library assign our pid */ NULL, /* no desired limits */ - &(btl->portals_limits), /* save our limits somewhere */ + &(btl->portals_ni_limits), /* save our limits somewhere */ &(btl->portals_ni_h) /* our interface handle */ ); if (PTL_OK != ret) { diff --git a/ompi/mca/btl/portals/src/btl_portals_component.c b/ompi/mca/btl/portals/src/btl_portals_component.c index b2113fc3a4..03c4d3b760 100644 --- a/ompi/mca/btl/portals/src/btl_portals_component.c +++ b/ompi/mca/btl/portals/src/btl_portals_component.c @@ -27,6 +27,10 @@ #include "btl_portals.h" #include "btl_portals_compat.h" +#include "btl_portals_frag.h" +#include "btl_portals_send.h" +#include "btl_portals_recv.h" +#include "btl_portals_rdma.h" mca_btl_portals_component_t mca_btl_portals_component = { @@ -37,7 +41,7 @@ mca_btl_portals_component_t mca_btl_portals_component = { /* Indicate that we are a pml v1.0.0 module (which also implies a specific MCA version) */ - MCA_BMI_BASE_VERSION_1_0_0, + MCA_BTL_BASE_VERSION_1_0_0, "portals", /* MCA module name */ OMPI_MAJOR_VERSION, /* MCA module major version */ @@ -116,23 +120,23 @@ mca_btl_portals_component_open(void) opal_mutex_t); /* get configured state for component */ -#if PTL_PORTALS_UTCP - mca_ptl_portals_component.portals_ifname = +#if BTL_PORTALS_UTCP + mca_btl_portals_component.portals_ifname = param_register_string("ifname", "eth0"); #endif portals_output_stream.lds_verbose_level = param_register_int("debug_level", - PTL_PORTALS_DEFAULT_DEBUG_LEVEL); + BTL_PORTALS_DEFAULT_DEBUG_LEVEL); - mca_ptl_portals_component.portals_free_list_init_num = + mca_btl_portals_component.portals_free_list_init_num = param_register_int("free_list_init_num", - PTL_PORTALS_DEFAULT_FREE_LIST_INIT_NUM); - mca_ptl_portals_component.portals_free_list_max_num = + BTL_PORTALS_DEFAULT_FREE_LIST_INIT_NUM); + mca_btl_portals_component.portals_free_list_max_num = param_register_int("free_list_max_num", - PTL_PORTALS_DEFAULT_FREE_LIST_MAX_NUM); - mca_ptl_portals_component.portals_free_list_inc_num = + BTL_PORTALS_DEFAULT_FREE_LIST_MAX_NUM); + mca_btl_portals_component.portals_free_list_inc_num = param_register_int("free_list_inc_num", - PTL_PORTALS_DEFAULT_FREE_LIST_INC_NUM); + BTL_PORTALS_DEFAULT_FREE_LIST_INC_NUM); /* start up debugging output */ asprintf(&(portals_output_stream.lds_prefix), @@ -142,13 +146,9 @@ mca_btl_portals_component_open(void) opal_output_open(&portals_output_stream); /* fill default module state */ - mca_ptl_portals_module.super.btl_flags = MCA_BMI_FLAGS_RDMA; + mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_RDMA; - for (i = 0 ; i < MCA_BMI_TAG_MAX ; ++i) { - mca_btl_portals_module.portals_reg = NULL; - } - - for (i = 0 ; i < MCA_BMI_PORTALS_EQ_SIZE ; ++i) { + for (i = 0 ; i < MCA_BTL_PORTALS_EQ_SIZE ; ++i) { mca_btl_portals_module.portals_eq_sizes[i] = 0; mca_btl_portals_module.portals_eq_handles[i] = PTL_EQ_NONE; } @@ -157,26 +157,26 @@ mca_btl_portals_component_open(void) mca_btl_portals_module.portals_sr_dropped = 0; /* get configured state for default module */ - mca_ptl_portals_module.super.btl_eager_limit = + mca_btl_portals_module.super.btl_eager_limit = param_register_int("eager_limit", - PTL_PORTALS_DEFAULT_EAGER_LIMIT); - mca_ptl_portals_module.super.btl_min_send_size = + BTL_PORTALS_DEFAULT_EAGER_LIMIT); + mca_btl_portals_module.super.btl_min_send_size = param_register_int("min_send_size", - PTL_PORTALS_DEFAULT_MIN_SEND_SIZE); - mca_ptl_portals_module.super.btl_max_send_size = + BTL_PORTALS_DEFAULT_MIN_SEND_SIZE); + mca_btl_portals_module.super.btl_max_send_size = param_register_int("max_send_size", - PTL_PORTALS_DEFAULT_MAX_SEND_SIZE); - mca_ptl_portals_module.super.btl_min_rdma_size = + BTL_PORTALS_DEFAULT_MAX_SEND_SIZE); + mca_btl_portals_module.super.btl_min_rdma_size = param_register_int("min_rdma_size", - PTL_PORTALS_DEFAULT_MIN_RDMA_SIZE); - mca_ptl_portals_module.super.btl_max_rdma_size = + BTL_PORTALS_DEFAULT_MIN_RDMA_SIZE); + mca_btl_portals_module.super.btl_max_rdma_size = param_register_int("max_rdma_size", - PTL_PORTALS_DEFAULT_MAX_RDMA_SIZE); - mca_ptl_portals_module.super.btl_exclusivity = + BTL_PORTALS_DEFAULT_MAX_RDMA_SIZE); + mca_btl_portals_module.super.btl_exclusivity = param_register_int("exclusivity", 60); - mca_ptl_portals_module.super.btl_latency = + mca_btl_portals_module.super.btl_latency = param_register_int("latency", 0); - mca_ptl_portals_module.super.btl_bandwidth = + mca_btl_portals_module.super.btl_bandwidth = param_register_int("bandwidth", 1000); return OMPI_SUCCESS; @@ -214,9 +214,9 @@ mca_btl_portals_component_init(int *num_btls, bool enable_mpi_threads) { mca_btl_base_module_t** btls; + uint32_t i; + *num_btls = 0; - int i; - uint32_t length; if (enable_progress_threads) { opal_output_verbose(20, mca_btl_portals_component.portals_output, @@ -232,41 +232,50 @@ mca_btl_portals_component_init(int *num_btls, return NULL; } + /* create an array of btl* to return */ btls = malloc(mca_btl_portals_component.portals_num_modules * sizeof(mca_btl_portals_module_t*)); - for (i = 0 ; i < mca_btl_portals_component.portals_num_modules ; ++i) { - btls[i] = (mca_btl_base_module_t*) - (mca_btl_portals_component.portals_modules + i); - OBJ_CONSTRUCT(&btls[i]->portals_frag_eager, ompi_free_list_t); - OBJ_CONSTRUCT(&btls[i]->portals_frag_max, ompi_free_list_t); - OBJ_CONSTRUCT(&btls[i]->portals_frag_user, ompi_free_list_t); + /* fill in all the portable parts of the module structs - the + compat code filled in the other bits already */ + for (i = 0 ; i < mca_btl_portals_component.portals_num_modules ; ++i) { + mca_btl_portals_module_t* ptl_btl = + (mca_btl_portals_component.portals_modules + i); + btls[i] = (mca_btl_base_module_t*) ptl_btl; + + + OBJ_CONSTRUCT(&(ptl_btl->portals_frag_eager), ompi_free_list_t); + OBJ_CONSTRUCT(&(ptl_btl->portals_frag_max), ompi_free_list_t); + OBJ_CONSTRUCT(&(ptl_btl->portals_frag_user), ompi_free_list_t); /* eager frags */ - ompi_free_list_init(&(btls[i].send_free_eager), - sizeof(mca_btl_portals_btls[i].super.btl_eager_limit, - OBJ_CLASS(mca_btl_portals_send_frag_eager_t), + ompi_free_list_init(&(ptl_btl->portals_frag_eager), + sizeof(mca_btl_portals_frag_eager_t) + + ptl_btl->super.btl_eager_limit, + OBJ_CLASS(mca_btl_portals_frag_eager_t), mca_btl_portals_component.portals_free_list_init_num, mca_btl_portals_component.portals_free_list_max_num, - mca_btl_portals_component.portals_free_list_inc_num); + mca_btl_portals_component.portals_free_list_inc_num, + NULL); /* send frags */ - ompi_free_list_init(&(btls[i].send_free_eager), - btls[i].super.btl_max_sender_size, - OBJ_CLASS(mca_btl_portals_send_frag_eager_t), + ompi_free_list_init(&(ptl_btl->portals_frag_max), + sizeof(mca_btl_portals_frag_max_t) + + ptl_btl->super.btl_max_send_size, + OBJ_CLASS(mca_btl_portals_frag_max_t), mca_btl_portals_component.portals_free_list_init_num, mca_btl_portals_component.portals_free_list_max_num, - mca_btl_portals_component.portals_free_list_inc_num); + mca_btl_portals_component.portals_free_list_inc_num, + NULL); /* user frags */ - ompi_free_list_init(&(btls[i].send_free_eager), - btls[i].super.btl_max_sender_size, - OBJ_CLASS(mca_btl_portals_send_frag_eager_t), + ompi_free_list_init(&(ptl_btl->portals_frag_user), + sizeof(mca_btl_portals_frag_user_t), + OBJ_CLASS(mca_btl_portals_frag_user_t), mca_btl_portals_component.portals_free_list_init_num, mca_btl_portals_component.portals_free_list_max_num, - mca_btl_portals_component.portals_free_list_inc_num); - - + mca_btl_portals_component.portals_free_list_inc_num, + NULL); } *num_btls = mca_btl_portals_component.portals_num_modules; @@ -286,30 +295,30 @@ mca_btl_portals_component_progress(void) for (i = 0 ; i < mca_btl_portals_component.portals_num_modules ; ++i) { struct mca_btl_portals_module_t *module = - mca_btl_portals_component.portals_modules[i]; + &(mca_btl_portals_component.portals_modules)[i]; ptl_event_t ev; ptl_sr_value_t numdropped; int which; int ret; - if (module->eq_handles[MCA_BMI_PORTALS_EQ_SIZE - 1] == + if (module->portals_eq_handles[MCA_BTL_PORTALS_EQ_SIZE - 1] == PTL_EQ_NONE) continue; /* they are all initialized at once */ #if OMPI_ENABLE_DEBUG /* BWB - this is going to kill performance */ - PtlNIStatus(module->ni_handle, + PtlNIStatus(module->portals_ni_h, PTL_SR_DROP_COUNT, &numdropped); - if (numdropped != module->dropped) { + if (numdropped != module->portals_sr_dropped) { opal_output_verbose(30, mca_btl_portals_component.portals_output, "*** Dropped message count changed. %lld, %lld", - module->dropped, numdropped); - module->dropped = numdropped; + module->portals_sr_dropped, numdropped); + module->portals_sr_dropped = numdropped; } #endif - ret = PtlEQPoll(module->eq_handles, - MCA_BMI_PORTALS_EQ_SIZE, /* number of eq handles */ + ret = PtlEQPoll(module->portals_eq_handles, + MCA_BTL_PORTALS_EQ_SIZE, /* number of eq handles */ 10, &ev, &which); @@ -326,7 +335,7 @@ mca_btl_portals_component_progress(void) "*** Event queue entries were dropped"); } -#if BMI_PORTALS_HAVE_EVENT_UNLINK +#if BTL_PORTALS_HAVE_EVENT_UNLINK && OMPI_ENABLE_DEBUG /* not everyone has UNLINK. Use it only to print the event, so we can make sure we properly re-initialize the ones that need to be re-initialized */ @@ -337,6 +346,20 @@ mca_btl_portals_component_progress(void) } #endif + switch (which) { + case MCA_BTL_PORTALS_EQ_RECV: + mca_btl_portals_process_recv(module, &ev); + break; + case MCA_BTL_PORTALS_EQ_SEND: + mca_btl_portals_process_send(module, &ev); + break; + case MCA_BTL_PORTALS_EQ_RDMA: + mca_btl_portals_process_rdma(module, &ev); + break; + default: + abort(); + } + num_progressed++; } diff --git a/ompi/mca/btl/portals/src/btl_portals_frag.c b/ompi/mca/btl/portals/src/btl_portals_frag.c new file mode 100644 index 0000000000..1d4ba01ae7 --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_frag.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "portals_config.h" + +#include "btl_portals.h" +#include "btl_portals_frag.h" + + +static void mca_btl_portals_frag_common_constructor(mca_btl_portals_frag_t* frag) +{ + mca_btl_portals_frag_common_constructor(frag); + frag->base.des_src = NULL; + frag->base.des_src_cnt = 0; + frag->base.des_dst = NULL; + frag->base.des_dst_cnt = 0; +} + +static void mca_btl_portals_frag_eager_constructor(mca_btl_portals_frag_t* frag) +{ + frag->size = mca_btl_portals_module.super.btl_eager_limit; + mca_btl_portals_frag_common_constructor(frag); +} + +static void mca_btl_portals_frag_max_constructor(mca_btl_portals_frag_t* frag) +{ + frag->size = mca_btl_portals_module.super.btl_max_send_size; + mca_btl_portals_frag_common_constructor(frag); +} + +static void mca_btl_portals_frag_user_constructor(mca_btl_portals_frag_t* frag) +{ + frag->size = 0; + mca_btl_portals_frag_common_constructor(frag); +} + + +OBJ_CLASS_INSTANCE( + mca_btl_portals_frag_t, + mca_btl_base_descriptor_t, + NULL, + NULL); + +OBJ_CLASS_INSTANCE( + mca_btl_portals_frag_eager_t, + mca_btl_base_descriptor_t, + mca_btl_portals_frag_eager_constructor, + NULL); + +OBJ_CLASS_INSTANCE( + mca_btl_portals_frag_max_t, + mca_btl_base_descriptor_t, + mca_btl_portals_frag_max_constructor, + NULL); + +OBJ_CLASS_INSTANCE( + mca_btl_portals_frag_user_t, + mca_btl_base_descriptor_t, + mca_btl_portals_frag_user_constructor, + NULL); + diff --git a/ompi/mca/btl/portals/src/btl_portals_frag.h b/ompi/mca/btl/portals/src/btl_portals_frag.h new file mode 100644 index 0000000000..3cf2bfe670 --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_frag.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_BTL_PORTALS_FRAG_H +#define MCA_BTL_PORTALS_FRAG_H + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif +OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_portals_frag_t); + + + +/** + * PORTALS send fraportalsent derived type. + */ +struct mca_btl_portals_frag_t { + mca_btl_base_descriptor_t base; + mca_btl_base_segment_t segment; + struct mca_btl_base_endpoint_t *endpoint; + mca_btl_base_header_t *hdr; + size_t size; +}; +typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_t; +OBJ_CLASS_DECLARATION(mca_btl_portals_frag_t); + + +typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_eager_t; + +OBJ_CLASS_DECLARATION(mca_btl_portals_frag_eager_t); + +typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_max_t; + +OBJ_CLASS_DECLARATION(mca_btl_portals_frag_max_t); + +typedef struct mca_btl_portals_frag_t mca_btl_portals_frag_user_t; + +OBJ_CLASS_DECLARATION(mca_btl_portals_frag_user_t); + + +/* + * Macros to allocate/return descriptors from module specific + * free list(s). + */ + +#define MCA_BTL_PORTALS_FRAG_ALLOC_EAGER(btl, frag, rc) \ +{ \ + \ + opal_list_item_t *item; \ + OMPI_FREE_LIST_WAIT(&((mca_btl_portals_module_t*)btl)->portals_frag_eager, item, rc); \ + frag = (mca_btl_portals_frag_t*) item; \ +} + +#define MCA_BTL_PORTALS_FRAG_RETURN_EAGER(btl, frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&((mca_btl_portals_module_t*)btl)->portals_frag_eager, \ + (opal_list_item_t*)(frag)); \ +} + +#define MCA_BTL_PORTALS_FRAG_ALLOC_MAX(btl, frag, rc) \ +{ \ + \ + opal_list_item_t *item; \ + OMPI_FREE_LIST_WAIT(&((mca_btl_portals_module_t*)btl)->portals_frag_max, item, rc); \ + frag = (mca_btl_portals_frag_t*) item; \ +} + +#define MCA_BTL_PORTALS_FRAG_RETURN_MAX(btl, frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&((mca_btl_portals_module_t*)btl)->portals_frag_max, \ + (opal_list_item_t*)(frag)); \ +} + + +#define MCA_BTL_PORTALS_FRAG_ALLOC_USER(btl, frag, rc) \ +{ \ + opal_list_item_t *item; \ + OMPI_FREE_LIST_WAIT(&((mca_btl_portals_module_t*)btl)->portals_frag_user, item, rc); \ + frag = (mca_btl_portals_frag_t*) item; \ +} + +#define MCA_BTL_PORTALS_FRAG_RETURN_USER(btl, frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&((mca_btl_portals_module_t*)btl)->portals_frag_user, \ + (opal_list_item_t*)(frag)); \ +} + + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif +#endif diff --git a/ompi/mca/btl/portals/src/btl_portals_rdma.c b/ompi/mca/btl/portals/src/btl_portals_rdma.c new file mode 100644 index 0000000000..040ea6a7e9 --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_rdma.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "portals_config.h" + +#include "include/constants.h" + +#include "btl_portals.h" +#include "btl_portals_rdma.h" + + +int +mca_btl_portals_process_rdma(mca_btl_portals_module_t *module, + ptl_event_t *ev) +{ + return OMPI_SUCCESS; +} diff --git a/ompi/mca/btl/portals/src/btl_portals_rdma.h b/ompi/mca/btl/portals/src/btl_portals_rdma.h new file mode 100644 index 0000000000..f68b074bcc --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_rdma.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_BTL_PORTALS_RDMA_H +#define MCA_BTL_PORTALS_RDMA_H + +int mca_btl_portals_process_rdma(mca_btl_portals_module_t *module, + ptl_event_t *ev); + + +#endif /* MCA_BTL_PORTALS_RDMA_H */ diff --git a/ompi/mca/btl/portals/src/btl_portals_recv.c b/ompi/mca/btl/portals/src/btl_portals_recv.c new file mode 100644 index 0000000000..4de9905089 --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_recv.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "portals_config.h" + +#include "include/constants.h" + +#include "btl_portals.h" +#include "btl_portals_recv.h" + + +int +mca_btl_portals_process_recv(mca_btl_portals_module_t *module, + ptl_event_t *ev) +{ + return OMPI_SUCCESS; +} diff --git a/ompi/mca/btl/portals/src/btl_portals_recv.h b/ompi/mca/btl/portals/src/btl_portals_recv.h new file mode 100644 index 0000000000..7dee6933fb --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_recv.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_BTL_PORTALS_RECV_H +#define MCA_BTL_PORTALS_RECV_H + +int mca_btl_portals_process_recv(mca_btl_portals_module_t *module, + ptl_event_t *ev); + + +#endif /* MCA_BTL_PORTALS_RECV_H */ diff --git a/ompi/mca/btl/portals/src/btl_portals_send.c b/ompi/mca/btl/portals/src/btl_portals_send.c new file mode 100644 index 0000000000..fea44a6be0 --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_send.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "portals_config.h" + +#include "include/constants.h" + +#include "btl_portals.h" +#include "btl_portals_send.h" + + +int +mca_btl_portals_process_send(mca_btl_portals_module_t *module, + ptl_event_t *ev) +{ + return OMPI_SUCCESS; +} diff --git a/ompi/mca/btl/portals/src/btl_portals_send.h b/ompi/mca/btl/portals/src/btl_portals_send.h new file mode 100644 index 0000000000..a15e63da9c --- /dev/null +++ b/ompi/mca/btl/portals/src/btl_portals_send.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_BTL_PORTALS_SEND_H +#define MCA_BTL_PORTALS_SEND_H + +int mca_btl_portals_process_send(mca_btl_portals_module_t *module, + ptl_event_t *ev); + + +#endif /* MCA_BTL_PORTALS_SEND_H */