diff --git a/src/mca/bmi/portals/.ompi_ignore b/src/mca/bmi/portals/.ompi_ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/mca/bmi/portals/.ompi_unignore b/src/mca/bmi/portals/.ompi_unignore new file mode 100644 index 0000000000..e4f6cc316d --- /dev/null +++ b/src/mca/bmi/portals/.ompi_unignore @@ -0,0 +1,2 @@ +brbarret +bbarrett diff --git a/src/mca/bmi/portals/Makefile.am b/src/mca/bmi/portals/Makefile.am new file mode 100644 index 0000000000..9992d1b53d --- /dev/null +++ b/src/mca/bmi/portals/Makefile.am @@ -0,0 +1,55 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University. +# All rights reserved. +# Copyright (c) 2004-2005 The Trustees of the University of Tennessee. +# All rights reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Use the top-level Makefile.options + +include $(top_ompi_srcdir)/config/Makefile.options + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if OMPI_BUILD_bmi_portals_DSO +component_noinst = +component_install = mca_bmi_portals.la +else +component_noinst = libmca_bmi_portals.la +component_install = +endif + +EXTRA_DIST = \ + src/bmi_portals_compat_utcp.c \ + src/bmi_portals_compat_redstorm.c + +portals_SOURCES = \ + src/bmi_portals.h \ + src/bmi_portals_compat.h \ + src/bmi_portals.c \ + src/bmi_portals_component.c \ + src/bmi_portals_stubs.c \ + src/bmi_portals_compat_utcp.c + + +mcacomponentdir = $(libdir)/openmpi +mcacomponent_LTLIBRARIES = $(component_install) +mca_bmi_portals_la_SOURCES = $(portals_SOURCES) +mca_bmi_portals_la_LIBADD = +mca_bmi_portals_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_bmi_portals_la_SOURCES = $(portals_SOURCES) +libmca_bmi_portals_la_LIBADD = +libmca_bmi_portals_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/bmi/portals/configure.params b/src/mca/bmi/portals/configure.params new file mode 100644 index 0000000000..4f0b39e38d --- /dev/null +++ b/src/mca/bmi/portals/configure.params @@ -0,0 +1,22 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University. +# All rights reserved. +# Copyright (c) 2004-2005 The Trustees of the University of Tennessee. +# All rights reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Specific to this module + +PARAM_INIT_FILE=src/bmi_portals.h +PARAM_CONFIG_HEADER_FILE="portals_config.h" +PARAM_CONFIG_FILES="Makefile" diff --git a/src/mca/bmi/portals/configure.stub b/src/mca/bmi/portals/configure.stub new file mode 100644 index 0000000000..94a6988df5 --- /dev/null +++ b/src/mca/bmi/portals/configure.stub @@ -0,0 +1,215 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University. +# All rights reserved. +# Copyright (c) 2004-2005 The Trustees of the University of Tennessee. +# All rights reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# +# quicky function to set #defines based on argument values +# +# ARGUMENTS: +# 1 configure name (first argument to ARG_WITH, minus the bmi-portals-) +# 2 define name +# 3 default value +# 4 description (used for both ARG_WITH and DEFINE) +AC_DEFUN([MCA_BMI_PORTALS_CONFIG_VAL], +[ + AC_ARG_WITH([bmi-portals-$1], AC_HELP_STRING([--with-bmi-portals-$1], + [$4 (default: $3)])) + AC_MSG_CHECKING([for $1 value]) + case "[$with_]m4_bpatsubst([bmi-portals-$1], -, _)" in + "") + $2=$3 + AC_MSG_RESULT([[$]$2 (default)]) + ;; + "no") + AC_MSG_RESULT([error]) + AC_MSG_ERROR([--without-bmi-portals-$1 is invalid argument]) + ;; + *) + $2="$with_m4_bpatsubst([bmi-portals-$1], -, _)" + AC_MSG_RESULT([[$]$2]) + ;; + esac + AC_DEFINE_UNQUOTED([$2], [[$]$2], [$4]) +])dnl + + +# +# Main function. This will be invoked in the middle of the templated +# configure script. +# +AC_DEFUN([MCA_CONFIGURE_STUB],[ + + # Additional --with flags that can be specified + + AC_ARG_WITH(bmi-portals, + AC_HELP_STRING([--with-bmi-portals=DIR], + [Specify the installation directory of PORTALS])) + + # Add to CPPFLAGS if necessary + EXTRA_CPPFLAGS= + if test -n "$with_bmi_portals"; then + if test -d "$with_bmi_portals/include"; then + EXTRA_CPPFLAGS="-I$with_bmi_portals/include" + else + AC_MSG_WARN([*** Warning: cannot find $with_bmi_portals/include]) + AC_MSG_WARN([*** Will still try to configure portals bmi anyway...]) + fi + fi + + # See if we can find portals.h + CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS" + AC_CHECK_HEADERS(portals3.h,, + AC_MSG_ERROR([*** Cannot find working portals3.h.])) + + # Add to LDFLAGS if necessary + EXTRA_LDFLAGS= + if test -n "$with_bmi_portals"; then + if test -d "$with_bmi_portals/lib"; then + EXTRA_LDFLAGS="-L$with_bmi_portals/lib" + else + AC_MSG_WARN([*** Warning: cannot find $with_bmi_portals/lib]) + AC_MSG_WARN([*** Will still try to configure portals bmi anyway...]) + fi + fi + + + # + # Configure Portals for our local environment + # + BMI_PORTALS_UTCP=0 + BMI_PORTALS_REDSTORM=0 + BMI_PORTALS_COMPAT="" + BMI_PORTALS_HAVE_EVENT_UNLINK=0 + + AC_ARG_WITH([bmi-portals-config], + AC_HELP_STRING([--with-bmi-portals-config], + [configuration to use for Portals support. + One of "utcp", "redstorm". (default: utcp)])) + AC_MSG_CHECKING([for Portals configuration]) + if test "$with_bmi_portals_config" = "" ; then + with_bmi_portals_config="utcp" + fi + case "$with_bmi_portals_config" in + "utcp") + BMI_PORTALS_UTCP=1 + PORTALS_LIBS="-lutcpapi -lutcplib -lp3api -lp3lib -lp3rt" + BMI_PORTALS_HAVE_EVENT_UNLINK=1 + AC_MSG_RESULT([utcp]) + ;; + "redstorm") + BMI_PORTALS_REDSTORM=1 + PORTALS_LIBS="-lp3api -lp3lib -lp3rt" + BMI_PORTALS_HAVE_EVENT_UNLINK=0 + AC_MSG_RESULT([red storm]) + ;; + *) + AC_MSG_ERROR([unknown Portals configuration. Can not continue]) + ;; + esac + + # Try to find all the portals libraries (this is not fun!) + AC_ARG_WITH(bmi-portals-libs, + AC_HELP_STRING([--with-bmi-portals-libs=LIBS], + [Libraries to link with for portals])) + if test -n "$with_bmi_portals_libs" ; then + PORTALS_LIBS="" + for lib in $with_bmi_portals_libs ; do + PORTALS_LIBS="$PORTALS_LIBS -l$lib" + done + fi + + AC_MSG_CHECKING([if possible to link Portals application]) + LIBS="$LIBS $PORTALS_LIBS" + LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], [int i; PtlInit(&i);])], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_ERROR([Can not link with Portals libraries])]) + + AC_DEFINE_UNQUOTED([BMI_PORTALS_HAVE_EVENT_UNLINK], + [$BMI_PORTALS_HAVE_EVENT_UNLINK], + [Does Portals send a BMI_EVENT_UNLINK event]) + + AC_DEFINE_UNQUOTED([BMI_PORTALS_UTCP], [$BMI_PORTALS_UTCP], + [Use the UTCP reference implementation or Portals]) + AM_CONDITIONAL([BMI_PORTALS_UTCP], [test "$BMI_PORTALS_UTCP" = "1"]) + + AC_DEFINE_UNQUOTED([BMI_PORTALS_REDSTORM], [$BMI_PORTALS_REDSTORM], + [Use the Red Storm implementation or Portals]) + AM_CONDITIONAL([BMI_PORTALS_REDSTORM], [test "$BMI_PORTALS_REDSTORM" = "1"]) + + MCA_BMI_PORTALS_CONFIG_VAL([frag-table-id], + [BMI_PORTALS_FRAG_TABLE_ID], [1], + [Portals table id to use for fragment receive queue]) + + MCA_BMI_PORTALS_CONFIG_VAL([retrans-table-id], + [BMI_PORTALS_RETRANS_TABLE_ID], [2], + [Portals table id to use for retransmit request queue]) + + MCA_BMI_PORTALS_CONFIG_VAL([debug-level], + [BMI_PORTALS_DEFAULT_DEBUG_LEVEL], [100], + [debugging level for portals bmi]) + + MCA_BMI_PORTALS_CONFIG_VAL([request-cache-size], + [BMI_PORTALS_DEFAULT_REQUEST_CACHE_SIZE], [1], + [request cache size for portals bmi]) + + MCA_BMI_PORTALS_CONFIG_VAL([first-frag-size], + [BMI_PORTALS_DEFAULT_FIRST_FRAG_SIZE], [16384], + [first frag size for portals bmi]) + + MCA_BMI_PORTALS_CONFIG_VAL([first-frag-num-entries], + [BMI_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES], [3], + [number of memory descriptors for first fragments]) + + MCA_BMI_PORTALS_CONFIG_VAL([first-frag-entry-size], + [BMI_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE], [2098152], + [size of memory associeted with first fag md]) + + MCA_BMI_PORTALS_CONFIG_VAL([recv-queue-size], + [BMI_PORTALS_DEFAULT_RECV_QUEUE_SIZE], [512], + [size of event queue for receiving frags]) + + MCA_BMI_PORTALS_CONFIG_VAL([send-queue-size], + [BMI_PORTALS_DEFAULT_SEND_QUEUE_SIZE], [128], + [Max number of send fragmenst pending]) + + MCA_BMI_PORTALS_CONFIG_VAL([rndv-frag-min-size], + [BMI_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE], [0], + [minimum size of rndv fragments]) + + MCA_BMI_PORTALS_CONFIG_VAL([rndv-frag-max-size], + [BMI_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE], [16384], + [maximum size of rndv fragments]) + + MCA_BMI_PORTALS_CONFIG_VAL([free-list-init-num], + [BMI_PORTALS_DEFAULT_FREE_LIST_INIT_NUM], [256], + [starting size of free lists]) + MCA_BMI_PORTALS_CONFIG_VAL([free-list-max-num], + [BMI_PORTALS_DEFAULT_FREE_LIST_MAX_NUM], [-1], + [maximum size of free lists]) + MCA_BMI_PORTALS_CONFIG_VAL([free-list-inc-num], + [BMI_PORTALS_DEFAULT_FREE_LIST_inc_NUM], [256], + [grow size for freelists]) + + # + # Save extra compiler/linker flags so that they can be added in + # the wrapper compilers, if necessary + # + + WRAPPER_EXTRA_LDFLAGS="$EXTRA_LDFLAGS" + WRAPPER_EXTRA_LIBS="$PORTALS_LIBS" +])dnl diff --git a/src/mca/bmi/portals/src/bmi_portals.c b/src/mca/bmi/portals/src/bmi_portals.c new file mode 100644 index 0000000000..b76e46fb16 --- /dev/null +++ b/src/mca/bmi/portals/src/bmi_portals.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "portals_config.h" + +#include +#include +#include +#include + +#include "include/constants.h" +#include "util/output.h" +#include "mca/pml/pml.h" +#include "mca/bmi/bmi.h" + +#include "bmi_portals.h" +#include "bmi_portals_compat.h" + +mca_bmi_portals_module_t mca_bmi_portals_module = { + { + &mca_bmi_portals_component.super, + 0, /* max size of first frag */ + 0, /* min send size */ + 0, /* max send size */ + 0, /* min rdma size */ + 0, /* max rdma size */ + 60, /* exclusivity - higher than sm, lower than self */ + 0, /* latency */ + 0, /* bandwidth */ + 0, /* bmi flags */ + + mca_bmi_portals_add_procs, + mca_bmi_portals_del_procs, + mca_bmi_portals_register, + mca_bmi_portals_finalize, + + mca_bmi_portals_alloc, + mca_bmi_portals_free, + mca_bmi_portals_prepare_src, + mca_bmi_portals_prepare_dst, + mca_bmi_portals_send, + mca_bmi_portals_put, + mca_bmi_portals_get, + }, +}; + + + +int +mca_bmi_portals_add_procs(struct mca_bmi_base_module_t* bmi, + size_t nprocs, struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t** peers, + ompi_bitmap_t* reachable) +{ + int ret; + struct ompi_proc_t *local_proc = ompi_proc_local(); + struct ompi_proc_t *curr_proc; + ptl_process_id_t *portals_procs; + size_t i; + unsigned long distance; + struct mca_bmi_portals_module_t *mybmi = + (struct mca_bmi_portals_module_t*) bmi; + + /* make sure our environment is fully initialized. At end of this + call, we have a working network handle on our module and + portals_procs will have the portals process identifier for each + proc (ordered, in theory) */ + ret = mca_bmi_portals_add_procs_compat(mybmi, nprocs, procs, + &portals_procs); + if (OMPI_SUCCESS != ret) return ret; + + /* loop through all procs, setting our reachable flag */ + for (i= 0; i < nprocs ; ++i) { + curr_proc = procs[i]; + /* BWB - do we want to send to self? No for now */ + if (curr_proc == local_proc) continue; + + /* make sure we can reach the process */ + ret = PtlNIDist(mybmi->ni_handle, + portals_procs[i], + &distance); + if (ret != PTL_OK) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "Could not find distance to process %d", i); + continue; + } + + /* set the peer as a pointer to the address */ + peers[i] = (struct mca_bmi_base_endpoint_t*) &(portals_procs[i]); + + /* and here we can reach */ + ompi_bitmap_set_bit(reachable, i); + } + + return OMPI_SUCCESS; +} + + +int +mca_bmi_portals_del_procs(struct mca_bmi_base_module_t *bmi, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t **peers) +{ + /* yeah, I have no idea what to do here */ + + return OMPI_SUCCESS; +} + + + +int +mca_bmi_portals_finalize(struct mca_bmi_base_module_t *bmi_base) +{ + struct mca_bmi_portals_module_t *bmi = + (struct mca_bmi_portals_module_t *) bmi_base; + int ret; + + ret = PtlNIFini(bmi->ni_handle); + if (PTL_OK != ret) { + ompi_output_verbose(20, mca_bmi_portals_component.portals_output, + "PtlNIFini returned %d", ret); + return OMPI_ERROR; + } + ompi_output_verbose(20, mca_bmi_portals_component.portals_output, + "successfully finalized module"); + + return OMPI_SUCCESS; +} diff --git a/src/mca/bmi/portals/src/bmi_portals.h b/src/mca/bmi/portals/src/bmi_portals.h new file mode 100644 index 0000000000..d250b77dda --- /dev/null +++ b/src/mca/bmi/portals/src/bmi_portals.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/* + * @file + */ +#ifndef MCA_BMI_PORTALS_H +#define MCA_BMI_PORTALS_H + +#include + +#include "mca/pml/pml.h" +#include "mca/bmi/bmi.h" +#include "class/ompi_bitmap.h" +#include "class/ompi_free_list.h" +#include "class/ompi_proc_table.h" + + +/* + * Portals BMI component. + */ +struct mca_bmi_portals_component_t { + /* base BMI component */ + mca_bmi_base_component_1_0_0_t super; + + /* output channel for debugging. Value settings when using + * output_verbose: + * + * - 0 : critical user information + * - 10: general execution diagnostic information + * - 20: initialization / shutdown diagnostic information + * - 30: basic debugging information + * - 90: useful only to developers + * - 100: lots and lots of performance impacting output + */ + int portals_output; + +#if BMI_PORTALS_UTCP + /* ethernet interface to use - only has meaning with utcp + reference */ + char *portals_ifname; +#endif + + /* Number of currently active portals modules. We assume these + never change between init and finalize, so these aren't thread + locked */ + uint32_t portals_num_modules; + /* List of currently available modules */ + struct mca_bmi_portals_module_t **portals_modules; + + /* initial size of free lists */ + int portals_free_list_init_num; + /* max size of free lists */ + int portals_free_list_max_num; + /* numer of elements to grow free lists */ + int portals_free_list_inc_num; + + /* lock for accessing component */ + ompi_mutex_t portals_lock; +}; +typedef struct mca_bmi_portals_component_t mca_bmi_portals_component_t; + + +#define MCA_BMI_PORTALS_EQ_RECV 0 +#define MCA_BMI_PORTALS_EQ_SEND 1 +#define MCA_BMI_PORTALS_EQ_SIZE 2 + +struct mca_bmi_portals_module_t { + /* base BMI module interface */ + mca_bmi_base_module_t super; + + /* number of mds for first frags */ + int first_frag_num_entries; + /* size of each md for first frags */ + int first_frag_entry_size; + + /* size for event queue */ + int eq_sizes[MCA_BMI_PORTALS_EQ_SIZE]; + /* frag receive event queue */ + ptl_handle_eq_t eq_handles[MCA_BMI_PORTALS_EQ_SIZE]; + + /* our portals network interface */ + ptl_handle_ni_t ni_handle; + /* the limits returned from PtlNIInit for interface */ + ptl_ni_limits_t limits; + + /* number of dropped messages */ + ptl_sr_value_t dropped; +}; +typedef struct mca_bmi_portals_module_t mca_bmi_portals_module_t; + +struct mca_bmi_portals_recv_frag_t; +struct mca_bmi_portals_send_frag_t; + +/* + * Component functions (bmi_portals_component.c) + */ +int mca_bmi_portals_component_open(void); +int mca_bmi_portals_component_close(void); + + +mca_bmi_base_module_t** mca_bmi_portals_component_init(int *num_bmis, + bool has_progress_threads, + bool has_mpi_threads); + +int mca_bmi_portals_component_progress(void); + +/* + * Compatibility functions (bmi_portals_compat_{}.c) + * + * Not part of the BMI interface. Need to be implemented for every + * version of Portals + */ +int mca_bmi_portals_init(mca_bmi_portals_component_t *comp); + +int mca_bmi_portals_add_procs_compat(mca_bmi_portals_module_t* bmi, + size_t nprocs, struct ompi_proc_t **procs, + ptl_process_id_t **portals_procs); + +/* + * Module configuration functions (bmi_portals.c) + */ +int mca_bmi_portals_finalize(struct mca_bmi_base_module_t* bmi); + +int mca_bmi_portals_add_procs(struct mca_bmi_base_module_t* bmi, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t** peers, + ompi_bitmap_t* reachable); + +int mca_bmi_portals_del_procs(struct mca_bmi_base_module_t* bmi, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t** peers); +/* + * stubbed functions + */ +int mca_bmi_portals_register(struct mca_bmi_base_module_t* bmi, + mca_bmi_base_tag_t tag, + mca_bmi_base_module_recv_cb_fn_t cbfunc, + void* cbdata); + +mca_bmi_base_descriptor_t* +mca_bmi_portals_alloc(struct mca_bmi_base_module_t* bmi, + size_t size); + +int mca_bmi_portals_free(struct mca_bmi_base_module_t* bmi, + mca_bmi_base_descriptor_t* des); + +mca_bmi_base_descriptor_t* +mca_bmi_portals_prepare_src(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* peer, + mca_mpool_base_registration_t* registration, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size); + +mca_bmi_base_descriptor_t* +mca_bmi_portals_prepare_dst(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* peer, + mca_mpool_base_registration_t* registration, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size); + +int mca_bmi_portals_send(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* bmi_peer, + struct mca_bmi_base_descriptor_t* descriptor, + mca_bmi_base_tag_t tag); + +int mca_bmi_portals_put(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* bmi_peer, + struct mca_bmi_base_descriptor_t* decriptor); + + +int mca_bmi_portals_get(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* bmi_peer, + struct mca_bmi_base_descriptor_t* decriptor); + + +/* + * global structures + */ +extern mca_bmi_portals_component_t mca_bmi_portals_component; +/* don't use, except as base for creating module instances */ +extern mca_bmi_portals_module_t mca_bmi_portals_module; + +#endif diff --git a/src/mca/bmi/portals/src/bmi_portals_compat.h b/src/mca/bmi/portals/src/bmi_portals_compat.h new file mode 100644 index 0000000000..a62f2e29db --- /dev/null +++ b/src/mca/bmi/portals/src/bmi_portals_compat.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#ifndef BMI_PORTALS_COMPAT_H +#define BMI_PORTALS_COMPAT_H + +#if BMI_PORTALS_UTCP + +#include +#include +#include + +#elif BMI_PORTALS_REDSTORM + +#error "Red Storm Compatibility not implemented" + +#else + +#error "Unknown Portals library configuration" + +#endif + +#endif /* BMI_PORTALS_NAL_H */ diff --git a/src/mca/bmi/portals/src/bmi_portals_compat_redstorm.c b/src/mca/bmi/portals/src/bmi_portals_compat_redstorm.c new file mode 100644 index 0000000000..ccb19e7ac4 --- /dev/null +++ b/src/mca/bmi/portals/src/bmi_portals_compat_redstorm.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "portals_config.h" + +#include "include/constants.h" +#include "util/output.h" + +#include "bmi_portals.h" +#include "bmi_portals_compat.h" + + +int +mca_bmi_portals_init(mca_bmi_portals_component_t *comp) +{ + int ret, max_interfaces; + struct mca_bmi_portals_module_t *bmi; + + /* + * Initialize Portals interface + */ + ret = PtlInit(&max_interfaces); + if (PTL_OK != ret) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "PtlInit failed, returning %d\n", ret); + return OMPI_ERR_FATAL; + } + + /* + * create module - only ever one "NIC" on red storm + */ + comp->portals_num_modules = 1; + comp->portals_modules = calloc(comp->portals_num_modules, + sizeof(mca_bmi_portals_module_t *)); + if (NULL == comp->portals_modules) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "malloc failed in mca_bmi_portals_init"); + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + comp->portals_modules[0] = malloc(sizeof(mca_bmi_portals_module_t)); + if (NULL == comp->portals_modules) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "malloc failed in mca_bmi_portals_init"); + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + bmi = comp->portals_modules[0]; + + *bmi = = mca_bmi_portals_module; + + /* + * Initialize a network device + */ + ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ + PTL_PID_ANY, /* let library assign our pid */ + NULL, /* no desired limits */ + &(bmi->limits), /* save our limits somewhere */ + &(bmi->ni_handle) /* our interface handle */ + ); + if (PTL_OK != ret) { + ompi_output_verbose(10, mca_ptl_portals_component.portals_output, + "PtlNIInit failed, returning %d\n", ret); + return OMPI_ERR_FATAL; + } + + + return OMPI_SUCCESS; +} + + +int +mca_bmi_portals_add_procs_compat(struct mca_bmi_portals_module_t* bmi, + size_t nprocs, struct ompi_proc_t **procs, + bmi_process_id_t **portals_procs) +{ + int nptl_procs = 0; + + /* + * FIXME - XXX - FIXME + * BWB - implicit assumption that cnos procs list will match our + * procs list. Don't know what to do about that... + */ + + nptl_procs = cnos_get_nidpid_map(portals_procs); + if (nptl_procs <= 0) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "cnos_get_nidpid_map() returned %d", nptl_procs); + return OMPI_ERR_FATAL; + } else if (nptl_procs != nprocs) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "nptl_procs != nprocs (%d, %d)", nptl_procs, + nprocs); + return OMPI_ERR_FATAL; + } + + return OMPI_ERR_NOT_IMPLEMENTED; +} diff --git a/src/mca/bmi/portals/src/bmi_portals_compat_utcp.c b/src/mca/bmi/portals/src/bmi_portals_compat_utcp.c new file mode 100644 index 0000000000..46cfea67a6 --- /dev/null +++ b/src/mca/bmi/portals/src/bmi_portals_compat_utcp.c @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "portals_config.h" + +#include +#include +#include +#include +#include + +#include "include/constants.h" +#include "util/output.h" + +#include "bmi_portals.h" +#include "bmi_portals_compat.h" + +#include + +/* how's this for source code diving? - find private method for + getting interface */ +extern unsigned int utcp_my_nid(const char *if_str); + +/* these need to be defined, or things get "unhappy" */ +FILE* utcp_api_out; +FILE* utcp_lib_out; + +int +mca_bmi_portals_init(mca_bmi_portals_component_t *comp) +{ + ptl_process_id_t info; + int ret; +#if 0 + FILE *output; + char *tmp; + + asprintf(&tmp, "portals.%d", getpid()); + output = fopen(tmp, "w"); + free(tmp); + + utcp_lib_out = output; + utcp_api_out = output; +#else + utcp_lib_out = stderr; + utcp_api_out = stderr; +#endif + + info.nid = htonl(utcp_my_nid(mca_bmi_portals_component.portals_ifname)); + info.pid = htonl((ptl_pid_t) getpid()); + ompi_output_verbose(100, mca_bmi_portals_component.portals_output, + "contact info: %u, %u", ntohl(info.nid), + ntohl(info.pid)); + + ret = mca_base_modex_send(&mca_bmi_portals_component.super.bmi_version, + &info, sizeof(ptl_process_id_t)); + if (OMPI_SUCCESS != ret) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "mca_base_modex_send failed: %d", ret); + return ret; + } + + /* with the utcp interface, only ever one "NIC" */ + comp->portals_num_modules = 1; + comp->portals_modules = calloc(comp->portals_num_modules, + sizeof(mca_bmi_portals_module_t *)); + if (NULL == comp->portals_modules) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "malloc failed in mca_bmi_portals_init"); + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + comp->portals_modules[0] = malloc(sizeof(mca_bmi_portals_module_t)); + if (NULL == comp->portals_modules) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "malloc failed in mca_bmi_portals_init"); + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + *(comp->portals_modules[0]) = mca_bmi_portals_module; + + return OMPI_SUCCESS; +} + + +int +mca_bmi_portals_add_procs_compat(struct mca_bmi_portals_module_t* bmi, + size_t nprocs, struct ompi_proc_t **procs, + ptl_process_id_t **portals_procs) +{ + int ret, my_rid; + ptl_process_id_t *info; + char *nidmap = NULL; + char *pidmap = NULL; + char *nid_str; + char *pid_str; + const size_t map_size = nprocs * 12 + 1; /* 12 is max length of long in decimal */ + size_t size, i; + char *tmp; + ompi_proc_t* proc_self = ompi_proc_local(); + int max_interfaces; + + /* + * Do all the NID/PID map setup + */ + /* each nid is a int, so need 10 there, plus the : */ + nidmap = malloc(map_size); + pidmap = malloc(map_size); + nid_str = malloc(12 + 1); + pid_str = malloc(12 + 1); + if (NULL == nidmap || NULL == pidmap || NULL == nid_str || NULL == pid_str) + return OMPI_ERROR; + + /* get space for the portals procs list */ + *portals_procs = calloc(nprocs, sizeof(ptl_process_id_t)); + if (NULL == *portals_procs) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "calloc(nprocs, sizeof(ptl_process_id_t)) failed"); + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + + for (i = 0 ; i < nprocs ; ++i) { + if (proc_self == procs[i]) my_rid = i; + + ret = mca_base_modex_recv(&mca_bmi_portals_component.super.bmi_version, + procs[i], (void**) &info, &size); + if (OMPI_SUCCESS != ret) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "mca_base_modex_recv failed: %d", ret); + return ret; + } else if (sizeof(ptl_process_id_t) != size) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "mca_base_modex_recv returned size %d, expected %d", + size, sizeof(ptl_process_id_t)); + return OMPI_ERROR; + } + + if (i == 0) { + snprintf(nidmap, map_size, "%u", ntohl(info->nid)); + snprintf(pidmap, map_size, "%u", ntohl(info->pid)); + } else { + snprintf(nid_str, 12 + 1, ":%u", ntohl(info->nid)); + snprintf(pid_str, 12 + 1, ":%u", ntohl(info->pid)); + strncat(nidmap, nid_str, 12); + strncat(pidmap, pid_str, 12); + } + + /* update my local array of proc structs */ + (*portals_procs)[i].nid = ntohl(info->nid); + (*portals_procs)[i].pid = ntohl(info->pid); + + free(info); + } + + ompi_output_verbose(100, mca_bmi_portals_component.portals_output, + "my rid: %u", my_rid); + ompi_output_verbose(100, mca_bmi_portals_component.portals_output, + "nid map: %s", nidmap); + ompi_output_verbose(100, mca_bmi_portals_component.portals_output, + "pid map: %s", pidmap); + ompi_output_verbose(100, mca_bmi_portals_component.portals_output, + "iface: %s", + mca_bmi_portals_component.portals_ifname); + + asprintf(&tmp, "PTL_MY_RID=%u", my_rid); + putenv(tmp); + asprintf(&tmp, "PTL_NIDMAP=%s", nidmap); + putenv(tmp); + asprintf(&tmp, "PTL_PIDMAP=%s", pidmap); + putenv(tmp); + asprintf(&tmp, "PTL_IFACE=%s", mca_bmi_portals_component.portals_ifname); + putenv(tmp); + + free(pidmap); + free(nidmap); + free(pid_str); + free(nid_str); + + /* + * Initialize Portals + */ + ret = PtlInit(&max_interfaces); + if (PTL_OK != ret) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "PtlInit failed, returning %d\n", ret); + return OMPI_ERR_FATAL; + } + + ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ + PTL_PID_ANY, /* let library assign our pid */ + NULL, /* no desired limits */ + &(bmi->limits), /* save our limits somewhere */ + &(bmi->ni_handle) /* our interface handle */ + ); + if (PTL_OK != ret) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "PtlNIInit failed, returning %d\n", ret); + return OMPI_ERR_FATAL; + } + +#if 0 + PtlNIDebug(bmi->ni_handle, PTL_DBG_ALL | PTL_DBG_NI_ALL); +#endif + + return OMPI_SUCCESS; +} + diff --git a/src/mca/bmi/portals/src/bmi_portals_component.c b/src/mca/bmi/portals/src/bmi_portals_component.c new file mode 100644 index 0000000000..6570acbea0 --- /dev/null +++ b/src/mca/bmi/portals/src/bmi_portals_component.c @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "portals_config.h" + +#include +#include + +#include "include/constants.h" + +#include "util/output.h" +#include "threads/thread.h" + +#include "bmi_portals.h" +#include "bmi_portals_compat.h" + + +mca_bmi_portals_component_t mca_bmi_portals_component = { + { + /* First, the mca_base_module_t struct containing meta + information about the module itself */ + { + /* Indicate that we are a pml v1.0.0 module (which also + implies a specific MCA version) */ + + MCA_BMI_BASE_VERSION_1_0_0, + + "portals", /* MCA module name */ + 1, /* MCA module major version */ + 0, /* MCA module minor version */ + 0, /* MCA module release version */ + mca_bmi_portals_component_open, /* module open */ + mca_bmi_portals_component_close /* module close */ + }, + + /* Next the MCA v1.0.0 module meta data */ + + { + /* Whether the module is checkpointable or not */ + + false + }, + + mca_bmi_portals_component_init, + mca_bmi_portals_component_progress, + } +}; + + +static ompi_output_stream_t portals_output_stream = { + true, /* is debugging */ + 0, /* verbose level */ + 0, /* want syslog */ + 0, /* syslog priority */ + NULL, /* syslog ident */ + NULL, /* prefix */ + true, /* want stdout */ + false, /* want stderr */ + false, /* want file */ + false, /* file append */ + "bmi-portals" /* file suffix */ +}; + + +static inline char* +param_register_string(const char* param_name, + const char* default_value) +{ + char *param_value; + int id = mca_base_param_register_string("bmi", "portals", + param_name, NULL, + default_value); + mca_base_param_lookup_string(id, ¶m_value); + return param_value; +} + + +static inline int +param_register_int(const char* param_name, + int default_value) +{ + int id = mca_base_param_register_int("bmi", "portals", param_name, + NULL, default_value); + int param_value = default_value; + mca_base_param_lookup_int(id, ¶m_value); + return param_value; +} + + + +int +mca_bmi_portals_component_open(void) +{ + int i; + + /* initialize state */ + mca_bmi_portals_component.portals_num_modules = 0; + mca_bmi_portals_component.portals_modules = NULL; + + /* initialize objects */ +#if 0 + OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_send_frags, + ompi_free_list_t); + OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_recv_frags, + ompi_free_list_t); + OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_pending_acks, + ompi_list_t); +#endif + OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_lock, + ompi_mutex_t); + + /* register portals module parameters */ +#if BMI_PORTALS_UTCP + mca_bmi_portals_component.portals_ifname = + param_register_string("ifname", "eth0"); +#endif + portals_output_stream.lds_verbose_level = + param_register_int("debug_level", + BMI_PORTALS_DEFAULT_DEBUG_LEVEL); + + mca_bmi_portals_component.portals_free_list_init_num = + param_register_int("free_list_init_num", + BMI_PORTALS_DEFAULT_FREE_LIST_INIT_NUM); + mca_bmi_portals_component.portals_free_list_max_num = + param_register_int("free_list_max_num", + BMI_PORTALS_DEFAULT_FREE_LIST_MAX_NUM); + mca_bmi_portals_component.portals_free_list_inc_num = + param_register_int("free_list_inc_num", + BMI_PORTALS_DEFAULT_FREE_LIST_inc_NUM); + +#if 0 + mca_bmi_portals_module.super.bmi_cache_size = + param_register_int("request_cache_size", + BMI_PORTALS_DEFAULT_REQUEST_CACHE_SIZE); + mca_bmi_portals_module.super.bmi_first_frag_size = + param_register_int("first_frag_size", + BMI_PORTALS_DEFAULT_FIRST_FRAG_SIZE); + mca_bmi_portals_module.super.bmi_min_frag_size = + param_register_int("rndv_frag_min_size", + BMI_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE); + mca_bmi_portals_module.super.bmi_max_frag_size = + param_register_int("rndv_frag_max_size", + BMI_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE); +#endif + + mca_bmi_portals_module.first_frag_num_entries = + param_register_int("first_frag_num_entries", + BMI_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES); + mca_bmi_portals_module.first_frag_entry_size = + param_register_int("first_frag_entry_size", + BMI_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE); + + mca_bmi_portals_module.eq_sizes[MCA_BMI_PORTALS_EQ_RECV] = + param_register_int("recv_queue_size", + BMI_PORTALS_DEFAULT_RECV_QUEUE_SIZE); + mca_bmi_portals_module.eq_sizes[MCA_BMI_PORTALS_EQ_SEND] = + (param_register_int("send_queue_size", + BMI_PORTALS_DEFAULT_SEND_QUEUE_SIZE)) * 3; + + /* finish with objects */ + asprintf(&(portals_output_stream.lds_prefix), + "bmi: portals (%5d): ", getpid()); + + mca_bmi_portals_component.portals_output = + ompi_output_open(&portals_output_stream); + + /* fill in remaining defaults for module data */ + for (i = 0 ; i < MCA_BMI_PORTALS_EQ_SIZE ; ++i) { + mca_bmi_portals_module.eq_handles[i] = PTL_EQ_NONE; + } + + mca_bmi_portals_module.ni_handle = PTL_INVALID_HANDLE; + mca_bmi_portals_module.dropped = 0; + + return OMPI_SUCCESS; +} + + +int +mca_bmi_portals_component_close(void) +{ + /* print out debugging if anything is pending */ + /* BWB - implement me, if possible */ + + /* release resources */ + OBJ_DESTRUCT(&mca_bmi_portals_component.portals_lock); +#if 0 + OBJ_DESTRUCT(&mca_bmi_portals_component.portals_recv_frags); + OBJ_DESTRUCT(&mca_bmi_portals_component.portals_pending_acks); + OBJ_DESTRUCT(&mca_bmi_portals_component.portals_lock); +#endif + + if (NULL != mca_bmi_portals_component.portals_ifname) { + free(mca_bmi_portals_component.portals_ifname); + } + + if (NULL != portals_output_stream.lds_prefix) { + free(portals_output_stream.lds_prefix); + } + + ompi_output_close(mca_bmi_portals_component.portals_output); + mca_bmi_portals_component.portals_output = -1; + + return OMPI_SUCCESS; +} + + +mca_bmi_base_module_t** +mca_bmi_portals_component_init(int *num_bmis, + bool enable_progress_threads, + bool enable_mpi_threads) +{ + mca_bmi_base_module_t** bmis; + *num_bmis = 0; + + if (enable_progress_threads) { + ompi_output_verbose(20, mca_bmi_portals_component.portals_output, + "disabled because progress threads enabled"); + return NULL; + } + +#if 0 + ompi_free_list_init(&mca_bmi_portals_component.portals_send_frags, + sizeof(mca_bmi_portals_send_frag_t), + OBJ_CLASS(mca_bmi_portals_send_frag_t), + mca_bmi_portals_component.portals_free_list_init_num, + mca_bmi_portals_component.portals_free_list_max_num, + mca_bmi_portals_component.portals_free_list_inc_num, + NULL); /* use default allocator */ + + ompi_free_list_init(&mca_bmi_portals_component.portals_recv_frags, + sizeof(mca_bmi_portals_recv_frag_t), + OBJ_CLASS(mca_bmi_portals_recv_frag_t), + mca_bmi_portals_component.portals_free_list_init_num, + mca_bmi_portals_component.portals_free_list_max_num, + mca_bmi_portals_component.portals_free_list_inc_num, + NULL); /* use default allocator */ +#endif + + /* initialize portals bmi. note that this is in the compat code because + it's fairly non-portable between implementations */ + if (OMPI_SUCCESS != mca_bmi_portals_init(&mca_bmi_portals_component)) { + ompi_output_verbose(20, mca_bmi_portals_component.portals_output, + "disabled because compatibility init failed"); + return NULL; + } + + /* return array of bmis */ + bmis = malloc(mca_bmi_portals_component.portals_num_modules * + sizeof(mca_bmi_base_module_t*)); + if (NULL == bmis) return NULL; + + memcpy(bmis, + mca_bmi_portals_component.portals_modules, + mca_bmi_portals_component.portals_num_modules * + sizeof(mca_bmi_base_module_t*)); + *num_bmis = mca_bmi_portals_component.portals_num_modules; + + ompi_output_verbose(20, mca_bmi_portals_component.portals_output, + "initialized %d modules", + *num_bmis); + + return bmis; +} + + +int +mca_bmi_portals_component_progress(void) +{ + int num_progressed = 0; + size_t i; + + for (i = 0 ; i < mca_bmi_portals_component.portals_num_modules ; ++i) { + struct mca_bmi_portals_module_t *module = + mca_bmi_portals_component.portals_modules[i]; + ptl_event_t ev; + ptl_sr_value_t numdropped; + int which; + int ret; + + if (module->eq_handles[MCA_BMI_PORTALS_EQ_SIZE - 1] == + PTL_EQ_NONE) continue; /* they are all initialized at once */ + +#if OMPI_ENABLE_DEBUG + /* BWB - this is going to kill performance */ + PtlNIStatus(module->ni_handle, + PTL_SR_DROP_COUNT, + &numdropped); + if (numdropped != module->dropped) { + ompi_output_verbose(30, mca_bmi_portals_component.portals_output, + "*** Dropped message count changed. %lld, %lld", + module->dropped, numdropped); + module->dropped = numdropped; + } +#endif + + ret = PtlEQPoll(module->eq_handles, + MCA_BMI_PORTALS_EQ_SIZE, /* number of eq handles */ + 10, + &ev, + &which); + if (PTL_EQ_EMPTY == ret) { + /* nothing to see here - move along */ + continue; + } else if (!(PTL_OK == ret || PTL_EQ_DROPPED == ret)) { + /* BWB - how can we report errors? */ + ompi_output(mca_bmi_portals_component.portals_output, + "Error calling PtlEQGet: %d", ret); + continue; + } else if (PTL_EQ_DROPPED == ret) { + ompi_output_verbose(10, mca_bmi_portals_component.portals_output, + "*** Event queue entries were dropped"); + } + +#if BMI_PORTALS_HAVE_EVENT_UNLINK + /* not everyone has UNLINK. Use it only to print the event, + so we can make sure we properly re-initialize the ones that + need to be re-initialized */ + if (PTL_EVENT_UNLINK == ev.type) { + OMPI_OUTPUT_VERBOSE((100, mca_bmi_portals_component.portals_output, + "unlink event occurred")); + continue; + } +#endif + + num_progressed++; + } + + return num_progressed; +} + diff --git a/src/mca/bmi/portals/src/bmi_portals_stubs.c b/src/mca/bmi/portals/src/bmi_portals_stubs.c new file mode 100644 index 0000000000..804ea0ae3d --- /dev/null +++ b/src/mca/bmi/portals/src/bmi_portals_stubs.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "portals_config.h" + +#include "bmi_portals.h" + +/* BWB - README - BWB - README - BWB - README - BWB - README - BWB + * + * These are stub functions that return error so that the + * initialization code can be developed and the whole thing will + * link. This file will disappear once all functions are + * implemented. Do not implement any functions in this file. + * + * BWB - README - BWB - README - BWB - README - BWB - README - BWB */ + + +int +mca_bmi_portals_register(struct mca_bmi_base_module_t* bmi, + mca_bmi_base_tag_t tag, + mca_bmi_base_module_recv_cb_fn_t cbfunc, + void* cbdata) +{ + return OMPI_SUCCESS; +} + + +mca_bmi_base_descriptor_t* +mca_bmi_portals_alloc(struct mca_bmi_base_module_t* bmi, + size_t size) +{ + return NULL; +} + +int +mca_bmi_portals_free(struct mca_bmi_base_module_t* bmi, + mca_bmi_base_descriptor_t* des) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +} + + +mca_bmi_base_descriptor_t* +mca_bmi_portals_prepare_src(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* peer, + mca_mpool_base_registration_t* registration, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size) +{ + return NULL; +} + + +mca_bmi_base_descriptor_t* +mca_bmi_portals_prepare_dst(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* peer, + mca_mpool_base_registration_t* registration, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size) +{ + return NULL; +} + + +int +mca_bmi_portals_send(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* bmi_peer, + struct mca_bmi_base_descriptor_t* descriptor, + mca_bmi_base_tag_t tag) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +} + + +int +mca_bmi_portals_put(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* bmi_peer, + struct mca_bmi_base_descriptor_t* decriptor) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +} + + +int +mca_bmi_portals_get(struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* bmi_peer, + struct mca_bmi_base_descriptor_t* decriptor) +{ + return OMPI_ERR_NOT_IMPLEMENTED; +}