From f402854a9610152065c821387c31026613cad1f9 Mon Sep 17 00:00:00 2001 From: Andrew Friedley Date: Mon, 19 Dec 2005 16:37:05 +0000 Subject: [PATCH] Initial commit of uDAPL BTL component. - Copied the template BTL and renamed everything - Compiles and shows up correctly in ompi_info, not tested past that - Should be ignored for everyone but me This commit was SVN r8544. --- ompi/mca/btl/udapl/.ompi_ignore | 0 ompi/mca/btl/udapl/.ompi_unignore | 1 + ompi/mca/btl/udapl/Makefile.am | 62 ++++ ompi/mca/btl/udapl/btl_udapl.c | 427 +++++++++++++++++++++++ ompi/mca/btl/udapl/btl_udapl.h | 313 +++++++++++++++++ ompi/mca/btl/udapl/btl_udapl_component.c | 171 +++++++++ ompi/mca/btl/udapl/btl_udapl_endpoint.c | 61 ++++ ompi/mca/btl/udapl/btl_udapl_endpoint.h | 58 +++ ompi/mca/btl/udapl/btl_udapl_frag.c | 58 +++ ompi/mca/btl/udapl/btl_udapl_frag.h | 116 ++++++ ompi/mca/btl/udapl/btl_udapl_proc.c | 162 +++++++++ ompi/mca/btl/udapl/btl_udapl_proc.h | 69 ++++ ompi/mca/btl/udapl/configure.params | 24 ++ 13 files changed, 1522 insertions(+) create mode 100644 ompi/mca/btl/udapl/.ompi_ignore create mode 100644 ompi/mca/btl/udapl/.ompi_unignore create mode 100644 ompi/mca/btl/udapl/Makefile.am create mode 100644 ompi/mca/btl/udapl/btl_udapl.c create mode 100644 ompi/mca/btl/udapl/btl_udapl.h create mode 100644 ompi/mca/btl/udapl/btl_udapl_component.c create mode 100644 ompi/mca/btl/udapl/btl_udapl_endpoint.c create mode 100644 ompi/mca/btl/udapl/btl_udapl_endpoint.h create mode 100644 ompi/mca/btl/udapl/btl_udapl_frag.c create mode 100644 ompi/mca/btl/udapl/btl_udapl_frag.h create mode 100644 ompi/mca/btl/udapl/btl_udapl_proc.c create mode 100644 ompi/mca/btl/udapl/btl_udapl_proc.h create mode 100644 ompi/mca/btl/udapl/configure.params diff --git a/ompi/mca/btl/udapl/.ompi_ignore b/ompi/mca/btl/udapl/.ompi_ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ompi/mca/btl/udapl/.ompi_unignore b/ompi/mca/btl/udapl/.ompi_unignore new file mode 100644 index 0000000000..534c909828 --- /dev/null +++ b/ompi/mca/btl/udapl/.ompi_unignore @@ -0,0 +1 @@ +afriedle diff --git a/ompi/mca/btl/udapl/Makefile.am b/ompi/mca/btl/udapl/Makefile.am new file mode 100644 index 0000000000..5d50ba75e4 --- /dev/null +++ b/ompi/mca/btl/udapl/Makefile.am @@ -0,0 +1,62 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Use the top-level Makefile.options + + + +sources = \ + btl_udapl.c \ + btl_udapl.h \ + btl_udapl_component.c \ + btl_udapl_endpoint.c \ + btl_udapl_endpoint.h \ + btl_udapl_frag.c \ + btl_udapl_frag.h \ + btl_udapl_proc.c \ + btl_udapl_proc.h \ + btl_udapl_error.h + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if OMPI_BUILD_btl_udapl_DSO +lib = +lib_sources = +component = mca_btl_udapl.la +component_sources = $(sources) +else +lib = libmca_btl_udapl.la +lib_sources = $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(libdir)/openmpi +mcacomponent_LTLIBRARIES = $(component) +mca_btl_udapl_la_SOURCES = $(component_sources) +mca_btl_udapl_la_LDFLAGS = -module -avoid-version +mca_btl_udapl_la_LIBADD = \ + $(top_ompi_builddir)/ompi/libmpi.la \ + $(top_ompi_builddir)/orte/liborte.la \ + $(top_ompi_builddir)/opal/libopal.la + +noinst_LTLIBRARIES = $(lib) +libmca_btl_udapl_la_SOURCES = $(lib_sources) +libmca_btl_udapl_la_LDFLAGS = -module -avoid-version diff --git a/ompi/mca/btl/udapl/btl_udapl.c b/ompi/mca/btl/udapl/btl_udapl.c new file mode 100644 index 0000000000..010a392618 --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl.c @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include +#include "opal/util/output.h" +#include "opal/util/if.h" +#include "mca/pml/pml.h" +#include "mca/btl/btl.h" + +#include "btl_udapl.h" +#include "btl_udapl_frag.h" +#include "btl_udapl_proc.h" +#include "btl_udapl_endpoint.h" +#include "datatype/convertor.h" +#include "mca/mpool/base/base.h" +#include "mca/mpool/mpool.h" + + +mca_btl_udapl_module_t mca_btl_udapl_module = { + { + &mca_btl_udapl_component.super, + 0, /* max size of first fragment */ + 0, /* min send fragment size */ + 0, /* max send fragment size */ + 0, /* min rdma fragment size */ + 0, /* max rdma fragment size */ + 0, /* exclusivity */ + 0, /* latency */ + 0, /* bandwidth */ + 0, /* flags */ + mca_btl_udapl_add_procs, + mca_btl_udapl_del_procs, + mca_btl_udapl_register, + mca_btl_udapl_finalize, + mca_btl_udapl_alloc, + mca_btl_udapl_free, + mca_btl_udapl_prepare_src, + mca_btl_udapl_prepare_dst, + mca_btl_udapl_send, + mca_btl_udapl_put, + NULL /* get */ + } +}; + +/** + * + */ + +int mca_btl_udapl_add_procs( + struct mca_btl_base_module_t* btl, + size_t nprocs, + struct ompi_proc_t **ompi_procs, + struct mca_btl_base_endpoint_t** peers, + ompi_bitmap_t* reachable) +{ + mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl; + int i, rc; + + for(i = 0; i < (int) nprocs; i++) { + + struct ompi_proc_t* ompi_proc = ompi_procs[i]; + mca_btl_udapl_proc_t* udapl_proc; + mca_btl_base_endpoint_t* udapl_endpoint; + + if(NULL == (udapl_proc = mca_btl_udapl_proc_create(ompi_proc))) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + /* + * Check to make sure that the peer has at least as many interface + * addresses exported as we are trying to use. If not, then + * don't bind this BTL instance to the proc. + */ + + OPAL_THREAD_LOCK(&udapl_proc->proc_lock); + + /* The btl_proc datastructure is shared by all uDAPL BTL + * instances that are trying to reach this destination. + * Cache the peer instance on the btl_proc. + */ + udapl_endpoint = OBJ_NEW(mca_btl_udapl_endpoint_t); + if(NULL == udapl_endpoint) { + OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock); + return OMPI_ERR_OUT_OF_RESOURCE; + } + + udapl_endpoint->endpoint_btl = udapl_btl; + rc = mca_btl_udapl_proc_insert(udapl_proc, udapl_endpoint); + if(rc != OMPI_SUCCESS) { + OBJ_RELEASE(udapl_endpoint); + OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock); + continue; + } + + ompi_bitmap_set_bit(reachable, i); + OPAL_THREAD_UNLOCK(&udapl_proc->proc_lock); + peers[i] = udapl_endpoint; + } + + return OMPI_SUCCESS; +} + +int mca_btl_udapl_del_procs(struct mca_btl_base_module_t* btl, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_btl_base_endpoint_t ** peers) +{ + /* TODO */ + return OMPI_SUCCESS; +} + + +/** + * Register callback function to support send/recv semantics + */ + +int mca_btl_udapl_register( + struct mca_btl_base_module_t* btl, + mca_btl_base_tag_t tag, + mca_btl_base_module_recv_cb_fn_t cbfunc, + void* cbdata) +{ + mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; + udapl_btl->udapl_reg[tag].cbfunc = cbfunc; + udapl_btl->udapl_reg[tag].cbdata = cbdata; + return OMPI_SUCCESS; +} + + +/** + * Allocate a segment. + * + * @param btl (IN) BTL module + * @param size (IN) Request segment size. + */ + +mca_btl_base_descriptor_t* mca_btl_udapl_alloc( + struct mca_btl_base_module_t* btl, + size_t size) +{ + mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; + mca_btl_udapl_frag_t* frag; + int rc; + + if(size <= btl->btl_eager_limit){ + MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(udapl_btl, frag, rc); + frag->segment.seg_len = + size <= btl->btl_eager_limit ? + size : btl->btl_eager_limit ; + } else { + MCA_BTL_UDAPL_FRAG_ALLOC_MAX(udapl_btl, frag, rc); + frag->segment.seg_len = + size <= btl->btl_max_send_size ? + size : btl->btl_max_send_size ; + } + + frag->base.des_flags = 0; + return (mca_btl_base_descriptor_t*)frag; +} + + +/** + * Return a segment + */ + +int mca_btl_udapl_free( + struct mca_btl_base_module_t* btl, + mca_btl_base_descriptor_t* des) +{ + mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des; + if(frag->size == 0) { +#if MCA_BTL_HAS_MPOOL + OBJ_RELEASE(frag->registration); +#endif + MCA_BTL_UDAPL_FRAG_RETURN_USER(btl, frag); + } else if(frag->size == btl->btl_eager_limit){ + MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag); + } else if(frag->size == btl->btl_max_send_size) { + MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag); + } else { + return OMPI_ERR_BAD_PARAM; + } + return OMPI_SUCCESS; +} + +/** + * Pack data and return a descriptor that can be + * used for send/put. + * + * @param btl (IN) BTL module + * @param peer (IN) BTL peer addressing + */ +mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* endpoint, + struct mca_mpool_base_registration_t* registration, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size +) +{ + mca_btl_udapl_frag_t* frag; + struct iovec iov; + uint32_t iov_count = 1; + size_t max_data = *size; + int32_t free_after; + int rc; + + + /* + * if we aren't pinning the data and the requested size is less + * than the eager limit pack into a fragment from the eager pool + */ + if (max_data+reserve <= btl->btl_eager_limit) { + + MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(btl, frag, rc); + if(NULL == frag) { + return NULL; + } + + iov.iov_len = max_data; + iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve; + + rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); + *size = max_data; + if( rc < 0 ) { + MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag); + return NULL; + } + frag->segment.seg_len = max_data + reserve; + } + + /* + * otherwise pack as much data as we can into a fragment + * that is the max send size. + */ + else { + + MCA_BTL_UDAPL_FRAG_ALLOC_MAX(btl, frag, rc); + if(NULL == frag) { + return NULL; + } + if(max_data + reserve > frag->size){ + max_data = frag->size - reserve; + } + iov.iov_len = max_data; + iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve; + + rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); + *size = max_data; + + if( rc < 0 ) { + MCA_BTL_UDAPL_FRAG_RETURN_MAX(btl, frag); + return NULL; + } + frag->segment.seg_len = max_data + reserve; + } + + frag->base.des_src = &frag->segment; + frag->base.des_src_cnt = 1; + frag->base.des_dst = NULL; + frag->base.des_dst_cnt = 0; + frag->base.des_flags = 0; + return &frag->base; +} + + +/** + * Prepare a descriptor for send/rdma using the supplied + * convertor. If the convertor references data that is contigous, + * the descriptor may simply point to the user buffer. Otherwise, + * this routine is responsible for allocating buffer space and + * packing if required. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL peer addressing + * @param convertor (IN) Data type convertor + * @param reserve (IN) Additional bytes requested by upper layer to precede user data + * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) + */ + +mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* endpoint, + struct mca_mpool_base_registration_t* registration, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size) +{ + mca_btl_udapl_frag_t* frag; + int rc; + + MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag, rc); + if(NULL == frag) { + return NULL; + } + + frag->segment.seg_len = *size; + frag->segment.seg_addr.pval = convertor->pBaseBuf + convertor->bConverted; + + frag->base.des_src = NULL; + frag->base.des_src_cnt = 0; + frag->base.des_dst = &frag->segment; + frag->base.des_dst_cnt = 1; + frag->base.des_flags = 0; + return &frag->base; +} + + +/** + * Initiate an asynchronous send. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param descriptor (IN) Description of the data to be transfered + * @param tag (IN) The tag value used to notify the peer. + */ + +int mca_btl_udapl_send( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* endpoint, + struct mca_btl_base_descriptor_t* descriptor, + mca_btl_base_tag_t tag) + +{ + /* mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; */ + mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)descriptor; + frag->endpoint = endpoint; + /* TODO */ + return OMPI_ERR_NOT_IMPLEMENTED; +} + + +/** + * Initiate an asynchronous put. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param descriptor (IN) Description of the data to be transferred + */ + +int mca_btl_udapl_put( + mca_btl_base_module_t* btl, + mca_btl_base_endpoint_t* endpoint, + mca_btl_base_descriptor_t* descriptor) +{ + /* mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; */ + mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*) descriptor; + frag->endpoint = endpoint; + /* TODO */ + return OMPI_ERR_NOT_IMPLEMENTED; +} + + +/** + * Initiate an asynchronous get. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param descriptor (IN) Description of the data to be transferred + * + */ + +int mca_btl_udapl_get( + mca_btl_base_module_t* btl, + mca_btl_base_endpoint_t* endpoint, + mca_btl_base_descriptor_t* descriptor) +{ + /* mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; */ + mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*) descriptor; + frag->endpoint = endpoint; + /* TODO */ + return OMPI_ERR_NOT_IMPLEMENTED; +} + + +/* + * Cleanup/release module resources. + */ + +int mca_btl_udapl_finalize(struct mca_btl_base_module_t* btl) +{ + mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; + + if(udapl_btl->udapl_frag_eager.fl_num_allocated != + udapl_btl->udapl_frag_eager.super.opal_list_length){ + opal_output(0, "btl udapl_frag_eager: %d allocated %d returned \n", + udapl_btl->udapl_frag_eager.fl_num_allocated, + udapl_btl->udapl_frag_eager.super.opal_list_length); + } + if(udapl_btl->udapl_frag_max.fl_num_allocated != + udapl_btl->udapl_frag_max.super.opal_list_length) { + opal_output(0, "btl udapl_frag_max: %d allocated %d returned \n", + udapl_btl->udapl_frag_max.fl_num_allocated, + udapl_btl->udapl_frag_max.super.opal_list_length); + } + if(udapl_btl->udapl_frag_user.fl_num_allocated != + udapl_btl->udapl_frag_user.super.opal_list_length){ + opal_output(0, "btl udapl_frag_user: %d allocated %d returned \n", + udapl_btl->udapl_frag_user.fl_num_allocated, + udapl_btl->udapl_frag_user.super.opal_list_length); + } + + OBJ_DESTRUCT(&udapl_btl->udapl_lock); + OBJ_DESTRUCT(&udapl_btl->udapl_frag_eager); + OBJ_DESTRUCT(&udapl_btl->udapl_frag_max); + OBJ_DESTRUCT(&udapl_btl->udapl_frag_user); + free(udapl_btl); + return OMPI_SUCCESS; +} + diff --git a/ompi/mca/btl/udapl/btl_udapl.h b/ompi/mca/btl/udapl/btl_udapl.h new file mode 100644 index 0000000000..a1deca8cc2 --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl.h @@ -0,0 +1,313 @@ + +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_BTL_UDAPL_H +#define MCA_BTL_UDAPL_H + +/* Standard system includes */ +#include +#include + +/* Open MPI includes */ +#include "class/ompi_free_list.h" +#include "class/ompi_bitmap.h" +#include "opal/event/event.h" +#include "mca/pml/pml.h" +#include "mca/btl/btl.h" +#include "mca/btl/base/base.h" +#include "opal/util/output.h" +#include "mca/mpool/mpool.h" +#include "mca/btl/btl.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +#define MCA_BTL_HAS_MPOOL 1 + +/** + * uDAPL BTL component. + */ + +struct mca_btl_udapl_component_t { + mca_btl_base_component_1_0_0_t super; /**< base BTL component */ + + uint32_t udapl_num_btls; + /**< number of hcas available to the uDAPL component */ + + struct mca_btl_udapl_module_t *udapl_btls; + /**< array of available BTL modules */ + + int udapl_free_list_num; + /**< initial size of free lists */ + + int udapl_free_list_max; + /**< maximum size of free lists */ + + int udapl_free_list_inc; + /**< number of elements to alloc when growing free lists */ + + opal_list_t udapl_procs; + /**< list of udapl proc structures */ + + opal_mutex_t udapl_lock; + /**< lock for accessing module state */ + + char* udapl_mpool_name; + /**< name of memory pool */ + + bool leave_pinned; + /**< pin memory on first use and leave pinned */ +}; +typedef struct mca_btl_udapl_component_t mca_btl_udapl_component_t; + +extern mca_btl_udapl_component_t mca_btl_udapl_component; + + + +/** + * BTL Module Interface + */ +struct mca_btl_udapl_module_t { + mca_btl_base_module_t super; /**< base BTL interface */ + mca_btl_base_recv_reg_t udapl_reg[MCA_BTL_TAG_MAX]; + + /* free list of fragment descriptors */ + ompi_free_list_t udapl_frag_eager; + ompi_free_list_t udapl_frag_max; + ompi_free_list_t udapl_frag_user; + + /* lock for accessing module state */ + opal_mutex_t udapl_lock; + +#if MCA_BTL_HAS_MPOOL + struct mca_mpool_base_module_t* udapl_mpool; +#endif +}; +typedef struct mca_btl_udapl_module_t mca_btl_udapl_module_t; +extern mca_btl_udapl_module_t mca_btl_udapl_module; + + +/** + * Register uDAPL component parameters with the MCA framework + */ +extern int mca_btl_udapl_component_open(void); + +/** + * Any final cleanup before being unloaded. + */ +extern int mca_btl_udapl_component_close(void); + +/** + * uDAPL component initialization. + * + * @param num_btl_modules (OUT) Number of BTLs returned in BTL array. + * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE) + * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE) + */ +extern mca_btl_base_module_t** mca_btl_udapl_component_init( + int *num_btl_modules, + bool allow_multi_user_threads, + bool have_hidden_threads +); + + +/** + * uDAPL component progress. + */ +extern int mca_btl_udapl_component_progress(void); + + + +/** + * Cleanup any resources held by the BTL. + * + * @param btl BTL instance. + * @return OMPI_SUCCESS or error status on failure. + */ + +extern int mca_btl_udapl_finalize( + struct mca_btl_base_module_t* btl +); + + +/** + * PML->BTL notification of change in the process list. + * + * @param btl (IN) + * @param nprocs (IN) Number of processes + * @param procs (IN) Set of processes + * @param peers (OUT) Set of (optional) peer addressing info. + * @param peers (IN/OUT) Set of processes that are reachable via this BTL. + * @return OMPI_SUCCESS or error status on failure. + * + */ + +extern int mca_btl_udapl_add_procs( + struct mca_btl_base_module_t* btl, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_btl_base_endpoint_t** peers, + ompi_bitmap_t* reachable +); + +/** + * PML->BTL notification of change in the process list. + * + * @param btl (IN) BTL instance + * @param nproc (IN) Number of processes. + * @param procs (IN) Set of processes. + * @param peers (IN) Set of peer data structures. + * @return Status indicating if cleanup was successful + * + */ + +extern int mca_btl_udapl_del_procs( + struct mca_btl_base_module_t* btl, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_btl_base_endpoint_t** peers +); + + +/** + * Initiate an asynchronous send. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param descriptor (IN) Description of the data to be transfered + * @param tag (IN) The tag value used to notify the peer. + */ + +extern int mca_btl_udapl_send( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* btl_peer, + struct mca_btl_base_descriptor_t* descriptor, + mca_btl_base_tag_t tag +); + + +/** + * Initiate an asynchronous put. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param descriptor (IN) Description of the data to be transferred + */ + +extern int mca_btl_udapl_put( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* btl_peer, + struct mca_btl_base_descriptor_t* decriptor +); + + +/** + * Initiate an asynchronous get. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL addressing information + * @param descriptor (IN) Description of the data to be transferred + */ + +extern int mca_btl_udapl_get( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* btl_peer, + struct mca_btl_base_descriptor_t* decriptor +); + +/** + * Register a callback function that is called on receipt + * of a fragment. + * + * @param btl (IN) BTL module + * @return Status indicating if registration was successful + * + */ + +extern int mca_btl_udapl_register( + struct mca_btl_base_module_t* btl, + mca_btl_base_tag_t tag, + mca_btl_base_module_recv_cb_fn_t cbfunc, + void* cbdata); + +/** + * Allocate a descriptor with a segment of the requested size. + * Note that the BTL layer may choose to return a smaller size + * if it cannot support the request. + * + * @param btl (IN) BTL module + * @param size (IN) Request segment size. + */ + +extern mca_btl_base_descriptor_t* mca_btl_udapl_alloc( + struct mca_btl_base_module_t* btl, + size_t size); + + +/** + * Return a segment allocated by this BTL. + * + * @param btl (IN) BTL module + * @param descriptor (IN) Allocated descriptor. + */ + +extern int mca_btl_udapl_free( + struct mca_btl_base_module_t* btl, + mca_btl_base_descriptor_t* des); + + +/** + * Prepare a descriptor for send/rdma using the supplied + * convertor. If the convertor references data that is contigous, + * the descriptor may simply point to the user buffer. Otherwise, + * this routine is responsible for allocating buffer space and + * packing if required. + * + * @param btl (IN) BTL module + * @param endpoint (IN) BTL peer addressing + * @param convertor (IN) Data type convertor + * @param reserve (IN) Additional bytes requested by upper layer to precede user data + * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) +*/ + +mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* peer, + struct mca_mpool_base_registration_t*, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size +); + +extern mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst( + struct mca_btl_base_module_t* btl, + struct mca_btl_base_endpoint_t* peer, + struct mca_mpool_base_registration_t*, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size); + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif +#endif diff --git a/ompi/mca/btl/udapl/btl_udapl_component.c b/ompi/mca/btl/udapl/btl_udapl_component.c new file mode 100644 index 0000000000..3226342a7f --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl_component.c @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include "ompi/include/constants.h" +#include "opal/event/event.h" +#include "opal/util/if.h" +#include "opal/util/argv.h" +#include "opal/util/output.h" +#include "mca/pml/pml.h" +#include "mca/btl/btl.h" + +#include "mca/base/mca_base_param.h" +#include "mca/pml/base/pml_base_module_exchange.h" +#include "mca/errmgr/errmgr.h" +#include "mca/mpool/base/base.h" +#include "btl_udapl.h" +#include "btl_udapl_frag.h" +#include "btl_udapl_endpoint.h" +#include "mca/btl/base/base.h" +#include "datatype/convertor.h" + +mca_btl_udapl_component_t mca_btl_udapl_component = { + { + /* First, the mca_base_component_t struct containing meta information + about the component itself */ + + { + /* Indicate that we are a pml v1.0.0 component (which also implies a + specific MCA version) */ + + MCA_BTL_BASE_VERSION_1_0_0, + + "udapl", /* MCA component name */ + OMPI_MAJOR_VERSION, /* MCA component major version */ + OMPI_MINOR_VERSION, /* MCA component minor version */ + OMPI_RELEASE_VERSION, /* MCA component release version */ + mca_btl_udapl_component_open, /* component open */ + mca_btl_udapl_component_close /* component close */ + }, + + /* Next the MCA v1.0.0 component meta data */ + + { + /* Whether the component is checkpointable or not */ + + false + }, + + mca_btl_udapl_component_init, + mca_btl_udapl_component_progress, + } +}; + + +/* + * utility routines for parameter registration + */ + +static inline char* mca_btl_udapl_param_register_string( + const char* param_name, + const char* default_value) +{ + char *param_value; + int id = mca_base_param_register_string("btl","ib",param_name,NULL,default_value); + mca_base_param_lookup_string(id, ¶m_value); + return param_value; +} + +static inline int mca_btl_udapl_param_register_int( + const char* param_name, + int default_value) +{ + int id = mca_base_param_register_int("btl","ib",param_name,NULL,default_value); + int param_value = default_value; + mca_base_param_lookup_int(id,¶m_value); + return param_value; +} + +/* + * Called by MCA framework to open the component, registers + * component parameters. + */ + +int mca_btl_udapl_component_open(void) +{ + + + /* initialize state */ + mca_btl_udapl_component.udapl_num_btls=0; + mca_btl_udapl_component.udapl_btls=NULL; + + /* initialize objects */ + OBJ_CONSTRUCT(&mca_btl_udapl_component.udapl_procs, opal_list_t); + + /* register uDAPL component parameters */ + mca_btl_udapl_component.udapl_free_list_num = + mca_btl_udapl_param_register_int ("free_list_num", 8); + mca_btl_udapl_component.udapl_free_list_max = + mca_btl_udapl_param_register_int ("free_list_max", 1024); + mca_btl_udapl_component.udapl_free_list_inc = + mca_btl_udapl_param_register_int ("free_list_inc", 32); + mca_btl_udapl_component.udapl_mpool_name = + mca_btl_udapl_param_register_string("mpool", "ib"); + mca_btl_udapl_module.super.btl_exclusivity = + mca_btl_udapl_param_register_int ("exclusivity", 0); + mca_btl_udapl_module.super.btl_eager_limit = + mca_btl_udapl_param_register_int ("first_frag_size", 64*1024) - sizeof(mca_btl_base_header_t); + mca_btl_udapl_module.super.btl_min_send_size = + mca_btl_udapl_param_register_int ("min_send_size", 64*1024) - sizeof(mca_btl_base_header_t); + mca_btl_udapl_module.super.btl_max_send_size = + mca_btl_udapl_param_register_int ("max_send_size", 128*1024) - sizeof(mca_btl_base_header_t); + mca_btl_udapl_module.super.btl_min_rdma_size = + mca_btl_udapl_param_register_int("min_rdma_size", 1024*1024); + mca_btl_udapl_module.super.btl_max_rdma_size = + mca_btl_udapl_param_register_int("max_rdma_size", 1024*1024); + mca_btl_udapl_module.super.btl_flags = + mca_btl_udapl_param_register_int("flags", MCA_BTL_FLAGS_PUT); + return OMPI_SUCCESS; +} + +/* + * component cleanup - sanity checking of queue lengths + */ + +int mca_btl_udapl_component_close(void) +{ + return OMPI_SUCCESS; +} + +/* + * uDAPL component initialization: + * (1) read interface list from kernel and compare against component parameters + * then create a BTL instance for selected interfaces + * (2) setup uDAPL listen socket for incoming connection attempts + * (3) register BTL parameters with the MCA + */ + +mca_btl_base_module_t** mca_btl_udapl_component_init(int *num_btl_modules, + bool enable_progress_threads, + bool enable_mpi_threads) +{ + return NULL; +} + +/* + * uDAPL component progress. + */ + + +int mca_btl_udapl_component_progress() +{ + return 0; +} + diff --git a/ompi/mca/btl/udapl/btl_udapl_endpoint.c b/ompi/mca/btl/udapl/btl_udapl_endpoint.c new file mode 100644 index 0000000000..250dc171c2 --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl_endpoint.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "ompi_config.h" +#include +#include +#include "include/types.h" +#include "mca/ns/base/base.h" +#include "mca/oob/base/base.h" +#include "mca/rml/rml.h" +#include "mca/errmgr/errmgr.h" +#include "dps/dps.h" +#include "btl_udapl.h" +#include "btl_udapl_endpoint.h" +#include "btl_udapl_proc.h" +#include "btl_udapl_frag.h" + + +/* + * Initialize state of the endpoint instance. + * + */ + +static void mca_btl_udapl_endpoint_construct(mca_btl_base_endpoint_t* endpoint) +{ + endpoint->endpoint_btl = 0; + endpoint->endpoint_proc = 0; +} + +/* + * Destroy a endpoint + * + */ + +static void mca_btl_udapl_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) +{ +} + + +OBJ_CLASS_INSTANCE( + mca_btl_udapl_endpoint_t, + opal_list_item_t, + mca_btl_udapl_endpoint_construct, + mca_btl_udapl_endpoint_destruct); + diff --git a/ompi/mca/btl/udapl/btl_udapl_endpoint.h b/ompi/mca/btl/udapl/btl_udapl_endpoint.h new file mode 100644 index 0000000000..14eef12d3f --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl_endpoint.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_BTL_UDAPL_ENDPOINT_H +#define MCA_BTL_UDAPL_ENDPOINT_H + +#include "opal/class/opal_list.h" +#include "opal/event/event.h" +#include "mca/pml/pml.h" +#include "mca/btl/btl.h" +#include "btl_udapl_frag.h" +#include "btl_udapl.h" +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +OBJ_CLASS_DECLARATION(mca_btl_udapl_endpoint_t); + +/** + * An abstraction that represents a connection to a endpoint process. + * An instance of mca_btl_base_endpoint_t is associated w/ each process + * and BTL pair at startup. However, connections to the endpoint + * are established dynamically on an as-needed basis: + */ + +struct mca_btl_base_endpoint_t { + opal_list_item_t super; + + struct mca_btl_udapl_module_t* endpoint_btl; + /**< BTL instance that created this connection */ + + struct mca_btl_udapl_proc_t* endpoint_proc; + /**< proc structure corresponding to endpoint */ +}; + +typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t; +typedef mca_btl_base_endpoint_t mca_btl_udapl_endpoint_t; + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif +#endif diff --git a/ompi/mca/btl/udapl/btl_udapl_frag.c b/ompi/mca/btl/udapl/btl_udapl_frag.c new file mode 100644 index 0000000000..9259b46b6a --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl_frag.c @@ -0,0 +1,58 @@ +#include "btl_udapl_frag.h" + + + +static void mca_btl_udapl_frag_common_constructor(mca_btl_udapl_frag_t* frag) +{ + mca_btl_udapl_frag_common_constructor(frag); + frag->base.des_src = NULL; + frag->base.des_src_cnt = 0; + frag->base.des_dst = NULL; + frag->base.des_dst_cnt = 0; +} + +static void mca_btl_udapl_frag_eager_constructor(mca_btl_udapl_frag_t* frag) +{ + frag->registration = NULL; + frag->size = mca_btl_udapl_module.super.btl_eager_limit; + mca_btl_udapl_frag_common_constructor(frag); +} + +static void mca_btl_udapl_frag_max_constructor(mca_btl_udapl_frag_t* frag) +{ + frag->registration = NULL; + frag->size = mca_btl_udapl_module.super.btl_max_send_size; + mca_btl_udapl_frag_common_constructor(frag); +} + +static void mca_btl_udapl_frag_user_constructor(mca_btl_udapl_frag_t* frag) +{ + frag->size = 0; + mca_btl_udapl_frag_common_constructor(frag); +} + + +OBJ_CLASS_INSTANCE( + mca_btl_udapl_frag_t, + mca_btl_base_descriptor_t, + NULL, + NULL); + +OBJ_CLASS_INSTANCE( + mca_btl_udapl_frag_eager_t, + mca_btl_base_descriptor_t, + mca_btl_udapl_frag_eager_constructor, + NULL); + +OBJ_CLASS_INSTANCE( + mca_btl_udapl_frag_max_t, + mca_btl_base_descriptor_t, + mca_btl_udapl_frag_max_constructor, + NULL); + +OBJ_CLASS_INSTANCE( + mca_btl_udapl_frag_user_t, + mca_btl_base_descriptor_t, + mca_btl_udapl_frag_user_constructor, + NULL); + diff --git a/ompi/mca/btl/udapl/btl_udapl_frag.h b/ompi/mca/btl/udapl/btl_udapl_frag.h new file mode 100644 index 0000000000..4e067240d2 --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl_frag.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_BTL_UDAPL_FRAG_H +#define MCA_BTL_UDAPL_FRAG_H + + +#define MCA_BTL_UDAPL_FRAG_ALIGN (8) +#include "ompi_config.h" +#include "btl_udapl.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif +OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_t); + + + +/** + * uDAPL send fragment derived type. + */ +struct mca_btl_udapl_frag_t { + mca_btl_base_descriptor_t base; + mca_btl_base_segment_t segment; + struct mca_btl_base_endpoint_t *endpoint; + mca_btl_base_header_t *hdr; + size_t size; +#if MCA_BTL_HAS_MPOOL + struct mca_mpool_base_registration_t* registration; +#endif +}; +typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_t; +OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_t); + + +typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_eager_t; + +OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_t); + +typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_max_t; + +OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_max_t); + +typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_user_t; + +OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_user_t); + + +/* + * Macros to allocate/return descriptors from module specific + * free list(s). + */ + +#define MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(btl, frag, rc) \ +{ \ + \ + opal_list_item_t *item; \ + OMPI_FREE_LIST_WAIT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, item, rc); \ + frag = (mca_btl_udapl_frag_t*) item; \ +} + +#define MCA_BTL_UDAPL_FRAG_RETURN_EAGER(btl, frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, \ + (opal_list_item_t*)(frag)); \ +} + +#define MCA_BTL_UDAPL_FRAG_ALLOC_MAX(btl, frag, rc) \ +{ \ + \ + opal_list_item_t *item; \ + OMPI_FREE_LIST_WAIT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, item, rc); \ + frag = (mca_btl_udapl_frag_t*) item; \ +} + +#define MCA_BTL_UDAPL_FRAG_RETURN_MAX(btl, frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, \ + (opal_list_item_t*)(frag)); \ +} + + +#define MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag, rc) \ +{ \ + opal_list_item_t *item; \ + OMPI_FREE_LIST_WAIT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, item, rc); \ + frag = (mca_btl_udapl_frag_t*) item; \ +} + +#define MCA_BTL_UDAPL_FRAG_RETURN_USER(btl, frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, \ + (opal_list_item_t*)(frag)); \ +} + + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif +#endif diff --git a/ompi/mca/btl/udapl/btl_udapl_proc.c b/ompi/mca/btl/udapl/btl_udapl_proc.c new file mode 100644 index 0000000000..2c6797d924 --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl_proc.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "class/opal_hash_table.h" +#include "mca/pml/base/pml_base_module_exchange.h" + +#include "btl_udapl.h" +#include "btl_udapl_proc.h" + +static void mca_btl_udapl_proc_construct(mca_btl_udapl_proc_t* proc); +static void mca_btl_udapl_proc_destruct(mca_btl_udapl_proc_t* proc); + +OBJ_CLASS_INSTANCE(mca_btl_udapl_proc_t, + opal_list_item_t, mca_btl_udapl_proc_construct, + mca_btl_udapl_proc_destruct); + +void mca_btl_udapl_proc_construct(mca_btl_udapl_proc_t* proc) +{ + proc->proc_ompi = 0; + proc->proc_addr_count = 0; + proc->proc_endpoints = 0; + proc->proc_endpoint_count = 0; + OBJ_CONSTRUCT(&proc->proc_lock, opal_mutex_t); + /* add to list of all proc instance */ + OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock); + opal_list_append(&mca_btl_udapl_component.udapl_procs, &proc->super); + OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock); +} + +/* + * Cleanup uDAPL proc instance + */ + +void mca_btl_udapl_proc_destruct(mca_btl_udapl_proc_t* proc) +{ + /* remove from list of all proc instances */ + OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock); + opal_list_remove_item(&mca_btl_udapl_component.udapl_procs, &proc->super); + OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock); + + /* release resources */ + if(NULL != proc->proc_endpoints) { + free(proc->proc_endpoints); + } +} + + +/* + * Look for an existing uDAPL process instances based on the associated + * ompi_proc_t instance. + */ +static mca_btl_udapl_proc_t* mca_btl_udapl_proc_lookup_ompi(ompi_proc_t* ompi_proc) +{ + mca_btl_udapl_proc_t* udapl_proc; + + OPAL_THREAD_LOCK(&mca_btl_udapl_component.udapl_lock); + + for(udapl_proc = (mca_btl_udapl_proc_t*) + opal_list_get_first(&mca_btl_udapl_component.udapl_procs); + udapl_proc != (mca_btl_udapl_proc_t*) + opal_list_get_end(&mca_btl_udapl_component.udapl_procs); + udapl_proc = (mca_btl_udapl_proc_t*)opal_list_get_next(udapl_proc)) { + + if(udapl_proc->proc_ompi == ompi_proc) { + OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock); + return udapl_proc; + } + + } + + OPAL_THREAD_UNLOCK(&mca_btl_udapl_component.udapl_lock); + + return NULL; +} + +/* + * Create a uDAPL process structure. There is a one-to-one correspondence + * between a ompi_proc_t and a mca_btl_udapl_proc_t instance. We cache + * additional data (specifically the list of mca_btl_udapl_endpoint_t instances, + * and published addresses) associated w/ a given destination on this + * datastructure. + */ + +mca_btl_udapl_proc_t* mca_btl_udapl_proc_create(ompi_proc_t* ompi_proc) +{ + mca_btl_udapl_proc_t* module_proc = NULL; + + /* Check if we have already created a uDAPL proc + * structure for this ompi process */ + module_proc = mca_btl_udapl_proc_lookup_ompi(ompi_proc); + + if(module_proc != NULL) { + + /* Gotcha! */ + return module_proc; + } + + /* Oops! First time, gotta create a new uDAPL proc + * out of the ompi_proc ... */ + + module_proc = OBJ_NEW(mca_btl_udapl_proc_t); + + /* Initialize number of peer */ + module_proc->proc_endpoint_count = 0; + + module_proc->proc_ompi = ompi_proc; + + /* build a unique identifier (of arbitrary + * size) to represent the proc */ + module_proc->proc_guid = ompi_proc->proc_name; + + /* uDAPL module doesn't have addresses exported at + * initialization, so the addr_count is set to one. */ + module_proc->proc_addr_count = 1; + + /* XXX: Right now, there can be only 1 peer associated + * with a proc. Needs a little bit change in + * mca_btl_udapl_proc_t to allow on demand increasing of + * number of endpoints for this proc */ + + module_proc->proc_endpoints = (mca_btl_base_endpoint_t**) + malloc(module_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*)); + + if(NULL == module_proc->proc_endpoints) { + OBJ_RELEASE(module_proc); + return NULL; + } + return module_proc; +} + + +/* + * Note that this routine must be called with the lock on the process + * already held. Insert a btl instance into the proc array and assign + * it an address. + */ +int mca_btl_udapl_proc_insert(mca_btl_udapl_proc_t* module_proc, + mca_btl_base_endpoint_t* module_endpoint) +{ + /* insert into endpoint array */ + module_endpoint->endpoint_proc = module_proc; + module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint; + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/btl/udapl/btl_udapl_proc.h b/ompi/mca/btl/udapl/btl_udapl_proc.h new file mode 100644 index 0000000000..964a6823df --- /dev/null +++ b/ompi/mca/btl/udapl/btl_udapl_proc.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_BTL_UDAPL_PROC_H +#define MCA_BTL_UDAPL_PROC_H + +#include "mca/ns/ns.h" +#include "opal/class/opal_object.h" +#include "proc/proc.h" +#include "btl_udapl.h" +#include "btl_udapl_endpoint.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif +OBJ_CLASS_DECLARATION(mca_btl_udapl_proc_t); + +/** + * Represents the state of a remote process and the set of addresses + * that it exports. Also cache an instance of mca_btl_base_endpoint_t for + * each + * BTL instance that attempts to open a connection to the process. + */ +struct mca_btl_udapl_proc_t { + opal_list_item_t super; + /**< allow proc to be placed on a list */ + + ompi_proc_t *proc_ompi; + /**< pointer to corresponding ompi_proc_t */ + + orte_process_name_t proc_guid; + /**< globally unique identifier for the process */ + + size_t proc_addr_count; + /**< number of addresses published by endpoint */ + + struct mca_btl_base_endpoint_t **proc_endpoints; + /**< array of endpoints that have been created to access this proc */ + + size_t proc_endpoint_count; + /**< number of endpoints */ + + opal_mutex_t proc_lock; + /**< lock to protect against concurrent access to proc state */ +}; +typedef struct mca_btl_udapl_proc_t mca_btl_udapl_proc_t; + +mca_btl_udapl_proc_t* mca_btl_udapl_proc_create(ompi_proc_t* ompi_proc); +int mca_btl_udapl_proc_insert(mca_btl_udapl_proc_t*, mca_btl_base_endpoint_t*); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif +#endif diff --git a/ompi/mca/btl/udapl/configure.params b/ompi/mca/btl/udapl/configure.params new file mode 100644 index 0000000000..f18349bf29 --- /dev/null +++ b/ompi/mca/btl/udapl/configure.params @@ -0,0 +1,24 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Specific to this module + +PARAM_INIT_FILE=btl_udapl.c +PARAM_CONFIG_HEADER_FILE="udapl_config.h" +PARAM_CONFIG_FILES="Makefile"