From 9c3dcdc7a2d46287bdf0124319f86f98979761c0 Mon Sep 17 00:00:00 2001 From: Tim Woodall Date: Wed, 15 Jun 2005 19:10:26 +0000 Subject: [PATCH] loopback bmi This commit was SVN r6069. --- src/mca/bmi/self/.ompi_ignore | 0 src/mca/bmi/self/.ompi_unignore | 2 + src/mca/bmi/self/Makefile.am | 49 +++++ src/mca/bmi/self/bmi_self.c | 287 ++++++++++++++++++++++++++ src/mca/bmi/self/bmi_self.h | 245 ++++++++++++++++++++++ src/mca/bmi/self/bmi_self_component.c | 190 +++++++++++++++++ src/mca/bmi/self/bmi_self_frag.c | 55 +++++ src/mca/bmi/self/bmi_self_frag.h | 84 ++++++++ src/mca/bmi/self/bmi_sm_endpoint.h | 43 ++++ 9 files changed, 955 insertions(+) create mode 100644 src/mca/bmi/self/.ompi_ignore create mode 100644 src/mca/bmi/self/.ompi_unignore create mode 100644 src/mca/bmi/self/Makefile.am create mode 100644 src/mca/bmi/self/bmi_self.c create mode 100644 src/mca/bmi/self/bmi_self.h create mode 100644 src/mca/bmi/self/bmi_self_component.c create mode 100644 src/mca/bmi/self/bmi_self_frag.c create mode 100644 src/mca/bmi/self/bmi_self_frag.h create mode 100644 src/mca/bmi/self/bmi_sm_endpoint.h diff --git a/src/mca/bmi/self/.ompi_ignore b/src/mca/bmi/self/.ompi_ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/mca/bmi/self/.ompi_unignore b/src/mca/bmi/self/.ompi_unignore new file mode 100644 index 0000000000..5c437cac59 --- /dev/null +++ b/src/mca/bmi/self/.ompi_unignore @@ -0,0 +1,2 @@ +twoodall +gshipman \ No newline at end of file diff --git a/src/mca/bmi/self/Makefile.am b/src/mca/bmi/self/Makefile.am new file mode 100644 index 0000000000..0f59c076d8 --- /dev/null +++ b/src/mca/bmi/self/Makefile.am @@ -0,0 +1,49 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University. +# All rights reserved. +# Copyright (c) 2004-2005 The Trustees of the University of Tennessee. +# All rights reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Use the top-level Makefile.options + +include $(top_ompi_srcdir)/config/Makefile.options + +libmca_bmi_self_la_sources = \ + bmi_self.c \ + bmi_self.h \ + bmi_self_component.c \ + bmi_self_frag.c \ + bmi_self_frag.h + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if OMPI_BUILD_bmi_self_DSO +component_noinst = +component_install = mca_bmi_self.la +else +component_noinst = libmca_bmi_self.la +component_install = +endif + +mcacomponentdir = $(libdir)/openmpi +mcacomponent_LTLIBRARIES = $(component_install) +mca_bmi_self_la_SOURCES = $(libmca_bmi_self_la_sources) +mca_bmi_self_la_LDFLAGS = -module -avoid-version +mca_bmi_self_la_LIBADD = + +noinst_LTLIBRARIES = $(component_noinst) +libmca_bmi_self_la_SOURCES = $(libmca_bmi_self_la_sources) +libmca_bmi_self_la_LDFLAGS = -module -avoid-version +libmca_bmi_self_la_LIBADD = diff --git a/src/mca/bmi/self/bmi_self.c b/src/mca/bmi/self/bmi_self.c new file mode 100644 index 0000000000..1295f9702a --- /dev/null +++ b/src/mca/bmi/self/bmi_self.c @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include +#include +#include +#include +#include +#include + +#include "threads/mutex.h" +#include "datatype/convertor.h" +#include "include/sys/atomic.h" +#include "util/output.h" +#include "util/if.h" +#include "util/proc_info.h" +#include "util/printf.h" +#include "util/sys_info.h" +#include "class/ompi_fifo.h" +#include "class/ompi_free_list.h" +#include "mca/pml/pml.h" +#include "mca/bmi/bmi.h" +#include "mca/mpool/base/base.h" +#include "bmi_self.h" +#include "bmi_self_frag.h" + + +mca_bmi_base_module_t mca_bmi_self = { + &mca_bmi_self_component.super, + 0, /* bmi_eager_limit */ + 0, /* bmi_min_send_size */ + 0, /* bmi_max_send_size */ + 0, /* bmi_min_rdma_size */ + 0, /* bmi_max_rdma_size */ + 0, /* bmi_exclusivity */ + 0, /* bmi_latency */ + 0, /* bmi_bandwidth */ + 0, /* bmi flags */ + mca_bmi_self_add_procs_same_base_addr, + mca_bmi_self_del_procs, + mca_bmi_self_register, + mca_bmi_self_finalize, + mca_bmi_self_alloc, + mca_bmi_self_free, + mca_bmi_self_prepare_src, + NULL, + mca_bmi_self_send, + NULL, /* put */ + NULL /* get */ +}; + + +int mca_bmi_self_add_procs( + struct mca_bmi_base_module_t* bmi, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t **peers, + ompi_bitmap_t* reachability) +{ + size_t i; + for(i=0; iBMI Clean up any resources held by BMI module + * before the module is unloaded. + * + * @param bmi (IN) BMI module. + * + * Prior to unloading a BMI module, the MCA framework will call + * the BMI finalize method of the module. Any resources held by + * the BMI should be released and if required the memory corresponding + * to the BMI module freed. + * + */ + +int mca_bmi_self_finalize(struct mca_bmi_base_module_t* bmi) +{ + return OMPI_SUCCESS; +} + + +/** + * Register a callback function that is called on receipt + * of a fragment. + * + * @param bmi (IN) BMI module + * @return Status indicating if cleanup was successful + * + * When the process list changes, the PML notifies the BMI of the + * change, to provide the opportunity to cleanup or release any + * resources associated with the peer. + */ + +int mca_bmi_self_register( + struct mca_bmi_base_module_t* bmi, + mca_bmi_base_tag_t tag, + mca_bmi_base_module_recv_cb_fn_t cbfunc, + void* cbdata) +{ + mca_bmi_self_component.self_reg[tag].cbfunc = cbfunc; + mca_bmi_self_component.self_reg[tag].cbdata = cbdata; + return OMPI_SUCCESS; +} + + +/** + * Allocate a segment. + * + * @param bmi (IN) BMI module + * @param size (IN) Request segment size. + */ +extern mca_bmi_base_descriptor_t* mca_bmi_self_alloc( + struct mca_bmi_base_module_t* bmi, + size_t size) +{ + mca_bmi_self_frag_t* frag; + int rc; + if(size <= mca_bmi_self.bmi_eager_limit) { + MCA_BMI_SELF_FRAG_ALLOC_EAGER(frag,rc); + } else { + MCA_BMI_SELF_FRAG_ALLOC_SEND(frag,rc); + } + frag->base.des_flags = 0; + return (mca_bmi_base_descriptor_t*)frag; +} + +/** + * Return a segment allocated by this BMI. + * + * @param bmi (IN) BMI module + * @param segment (IN) Allocated segment. + */ +extern int mca_bmi_self_free( + struct mca_bmi_base_module_t* bmi, + mca_bmi_base_descriptor_t* des) +{ + mca_bmi_self_frag_t* frag = (mca_bmi_self_frag_t*)des; + if(frag->size <= mca_bmi_self.bmi_eager_limit) { + MCA_BMI_SELF_FRAG_RETURN_EAGER(frag); + } else if (frag->size <= mca_bmi_self.bmi_max_send_size) { + MCA_BMI_SELF_FRAG_RETURN_SEND(frag); + } else { + MCA_BMI_SELF_FRAG_RETURN_RDMA(frag); + } + return OMPI_SUCCESS; +} + + +/** + * Prepare data for send/put + * + * @param bmi (IN) BMI module + */ +struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_src( + struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* endpoint, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size) +{ + mca_bmi_self_frag_t* frag; + struct iovec iov; + uint32_t iov_count = 1; + size_t max_data = *size; + int32_t free_after; + int rc; + + /* non-contigous data */ + if(ompi_convertor_need_buffers(convertor)) { + MCA_BMI_SELF_FRAG_ALLOC_SEND(frag, rc); + if(NULL == frag) { + return NULL; + } + + if(reserve + max_data > frag->size) { + max_data = frag->size - reserve; + } + iov.iov_len = max_data; + iov.iov_base = (unsigned char*)(frag+1) + reserve; + + rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); + if(rc < 0) { + MCA_BMI_SELF_FRAG_RETURN_SEND(frag); + return NULL; + } + frag->base.des_flags = 0; + frag->segment.seg_addr.pval = frag+1; + frag->segment.seg_len = reserve + max_data; + *size = max_data; + } else { + MCA_BMI_SELF_FRAG_ALLOC_RDMA(frag, rc); + if(NULL == frag) { + return NULL; + } + iov.iov_len = max_data; + iov.iov_base = NULL; + + /* convertor should return offset into users buffer */ + rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); + if(rc < 0) { + MCA_BMI_SELF_FRAG_RETURN_RDMA(frag); + return NULL; + } + frag->base.des_flags = 0; + frag->segment.seg_addr.pval = iov.iov_base; + frag->segment.seg_len = reserve + max_data; + *size = max_data; + } + return &frag->base; +} + +/** + * Prepare data for receive. + */ +struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_dst( + struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* endpoint, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size) +{ + mca_bmi_self_frag_t* frag; + size_t max_data = *size; + int rc; + + MCA_BMI_SELF_FRAG_ALLOC_RDMA(frag, rc); + if(NULL == frag) { + return NULL; + } + + /* setup descriptor to point directly to user buffer */ + frag->base.des_flags = 0; + frag->segment.seg_addr.pval = (unsigned char*)convertor->pBaseBuf + convertor->bConverted; + frag->segment.seg_len = reserve + max_data; + return &frag->base; +} + + + +/** + * Initiate a send to the peer. + * + * @param bmi (IN) BMI module + * @param peer (IN) BMI peer addressing + */ + +int mca_bmi_self_send( + struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* endpoint, + struct mca_bmi_base_descriptor_t* descriptor, + mca_bmi_base_tag_t tag) +{ + mca_bmi_self_component.self_reg[tag].cbfunc(bmi,tag,descriptor,OMPI_SUCCESS); + return OMPI_SUCCESS; +} + + diff --git a/src/mca/bmi/self/bmi_self.h b/src/mca/bmi/self/bmi_self.h new file mode 100644 index 0000000000..4c75b93238 --- /dev/null +++ b/src/mca/bmi/self/bmi_self.h @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_BMI_SELF_H +#define MCA_BMI_SELF_H + +#include +#include +#include +#include +#include "class/ompi_free_list.h" +#include "class/ompi_bitmap.h" +#include "class/ompi_fifo.h" +#include "event/event.h" +#include "mca/pml/pml.h" +#include "mca/bmi/bmi.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + + +/** + * Shared Memory (SELF) BMI module. + */ +struct mca_bmi_self_component_t { + mca_bmi_base_component_1_0_0_t super; /**< base BMI component */ + int free_list_num; /**< initial size of free lists */ + int free_list_max; /**< maximum size of free lists */ + int free_list_inc; /**< number of elements to alloc when growing free lists */ + ompi_mutex_t self_lock; + ompi_free_list_t self_frags_eager; /**< free list of self first */ + ompi_free_list_t self_frags_send; /**< free list of self second */ + ompi_free_list_t self_frags_rdma; /**< free list of self second */ + mca_bmi_base_registration_t self_reg[256]; +}; +typedef struct mca_bmi_self_component_t mca_bmi_self_component_t; +extern mca_bmi_self_component_t mca_bmi_self_component; + +/** + * Register shared memory module parameters with the MCA framework + */ +extern int mca_bmi_self_component_open(void); + +/** + * Any final cleanup before being unloaded. + */ +extern int mca_bmi_self_component_close(void); + +/** + * SELF module initialization. + * + * @param num_bmis (OUT) Number of BMIs returned in BMI array. + * @param enable_progress_threads (IN) Flag indicating whether BMI is allowed to have progress threads + * @param enable_mpi_threads (IN) Flag indicating whether BMI must support multilple simultaneous invocations from different threads + * + */ +extern mca_bmi_base_module_t** mca_bmi_self_component_init( + int *num_bmis, + bool enable_progress_threads, + bool enable_mpi_threads +); + +extern mca_bmi_base_module_t mca_bmi_self; + + +/** + * Cleanup any resources held by the BMI. + * + * @param bmi BMI instance. + * @return OMPI_SUCCESS or error status on failure. + */ + +extern int mca_bmi_self_finalize( + struct mca_bmi_base_module_t* bmi +); + + +/** + * PML->BMI notification of change in the process list. + * PML->BMI Notification that a receive fragment has been matched. + * Called for message that is send from process with the virtual + * address of the shared memory segment being different than that of + * the receiver. + * + * @param bmi (IN) + * @param proc (IN) + * @param peer (OUT) + * @return OMPI_SUCCESS or error status on failure. + * + */ + +extern int mca_bmi_self_add_procs( + struct mca_bmi_base_module_t* bmi, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t** peers, + struct ompi_bitmap_t* reachability +); + + +/** + * PML->BMI notification of change in the process list. + * PML->BMI Notification that a receive fragment has been matched. + * Called for message that is send from process with the virtual + * address of the shared memory segment being the same as that of + * the receiver. + * + * @param bmi (IN) + * @param proc (IN) + * @param peer (OUT) + * @return OMPI_SUCCESS or error status on failure. + * + */ + +extern int mca_bmi_self_add_procs_same_base_addr( + struct mca_bmi_base_module_t* bmi, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t** peers, + ompi_bitmap_t* reachability +); + + +/** + * PML->BMI notification of change in the process list. + * + * @param bmi (IN) BMI instance + * @param proc (IN) Peer process + * @param peer (IN) Peer addressing information. + * @return Status indicating if cleanup was successful + * + */ +extern int mca_bmi_self_del_procs( + struct mca_bmi_base_module_t* bmi, + size_t nprocs, + struct ompi_proc_t **procs, + struct mca_bmi_base_endpoint_t **peers +); + + +/** + * Register a callback function that is called on receipt + * of a fragment. + * + * @param bmi (IN) BMI module + * @return Status indicating if cleanup was successful + * + * When the process list changes, the PML notifies the BMI of the + * change, to provide the opportunity to cleanup or release any + * resources associated with the peer. + */ + +extern int mca_bmi_self_register( + struct mca_bmi_base_module_t* bmi, + mca_bmi_base_tag_t tag, + mca_bmi_base_module_recv_cb_fn_t cbfunc, + void* cbdata +); + + +/** + * Allocate a segment. + * + * @param bmi (IN) BMI module + * @param size (IN) Request segment size. + */ +extern mca_bmi_base_descriptor_t* mca_bmi_self_alloc( + struct mca_bmi_base_module_t* bmi, + size_t size +); + +/** + * Return a segment allocated by this BMI. + * + * @param bmi (IN) BMI module + * @param segment (IN) Allocated segment. + */ +extern int mca_bmi_self_free( + struct mca_bmi_base_module_t* bmi, + mca_bmi_base_descriptor_t* segment +); + +/** + * Pack data + * + * @param bmi (IN) BMI module + * @param peer (IN) BMI peer addressing + */ +struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_src( + struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* endpoint, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size +); + +/** + * Prepare data for RDMA + * + * @param bmi (IN) BMI module + * @param peer (IN) BMI peer addressing + */ +struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_dst( + struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* endpoint, + struct ompi_convertor_t* convertor, + size_t reserve, + size_t* size +); + +/** + * Initiate a send to the peer. + * + * @param bmi (IN) BMI module + * @param peer (IN) BMI peer addressing + */ +extern int mca_bmi_self_send( + struct mca_bmi_base_module_t* bmi, + struct mca_bmi_base_endpoint_t* endpoint, + struct mca_bmi_base_descriptor_t* descriptor, + mca_bmi_base_tag_t tag +); + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif +#endif + diff --git a/src/mca/bmi/self/bmi_self_component.c b/src/mca/bmi/self/bmi_self_component.c new file mode 100644 index 0000000000..c75d12012c --- /dev/null +++ b/src/mca/bmi/self/bmi_self_component.c @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include +#include +#include +#include +#include +#include +#include /* for mkfifo */ + +#include "include/constants.h" +#include "include/sys/cache.h" +#include "event/event.h" +#include "util/if.h" +#include "util/argv.h" +#include "util/output.h" +#include "util/sys_info.h" +#include "util/proc_info.h" +#include "mca/pml/pml.h" +#include "mca/base/mca_base_param.h" +#include "bmi_self.h" +#include "bmi_self_frag.h" + + + +/* + * Shared Memory (SELF) component instance. + */ + +mca_bmi_self_component_t mca_bmi_self_component = { + { /* super is being filled in */ + /* First, the mca_base_component_t struct containing meta information + about the component itself */ + { + /* Indicate that we are a pml v1.0.0 component (which also implies a + specific MCA version) */ + MCA_BMI_BASE_VERSION_1_0_0, + "self", /* MCA component name */ + 1, /* MCA component major version */ + 0, /* MCA component minor version */ + 0, /* MCA component release version */ + mca_bmi_self_component_open, /* component open */ + mca_bmi_self_component_close /* component close */ + }, + + /* Next the MCA v1.0.0 component meta data */ + { + /* Whether the component is checkpointable or not */ + false + }, + + mca_bmi_self_component_init, + NULL, + } /* end super */ +}; + + +/* + * utility routines for parameter registration + */ + +static inline char* mca_bmi_self_param_register_string( + const char* param_name, + const char* default_value) +{ + char *param_value; + int id = mca_base_param_register_string("ptl","self",param_name,NULL,default_value); + mca_base_param_lookup_string(id, ¶m_value); + return param_value; +} + +static inline int mca_bmi_self_param_register_int( + const char* param_name, + int default_value) +{ + int id = mca_base_param_register_int("ptl","self",param_name,NULL,default_value); + int param_value = default_value; + mca_base_param_lookup_int(id,¶m_value); + return param_value; +} + + +/* + * Called by MCA framework to open the component, registers + * component parameters. + */ + +int mca_bmi_self_component_open(void) +{ + /* register SELF component parameters */ + mca_bmi_self_component.free_list_num = + mca_bmi_self_param_register_int("free_list_num", 256); + mca_bmi_self_component.free_list_max = + mca_bmi_self_param_register_int("free_list_max", -1); + mca_bmi_self_component.free_list_inc = + mca_bmi_self_param_register_int("free_list_inc", 256); + mca_bmi_self.bmi_eager_limit = + mca_bmi_self_param_register_int("eager_limit", 8*1024); + mca_bmi_self.bmi_min_send_size = + mca_bmi_self.bmi_max_send_size = + mca_bmi_self_param_register_int("max_send_size", 64*1024); + mca_bmi_self.bmi_min_rdma_size = + mca_bmi_self.bmi_max_rdma_size = + mca_bmi_self_param_register_int("max_rdma_size", -1); + mca_bmi_self.bmi_flags = + mca_bmi_self_param_register_int("flags", 1); + + /* initialize objects */ + OBJ_CONSTRUCT(&mca_bmi_self_component.self_lock, ompi_mutex_t); + OBJ_CONSTRUCT(&mca_bmi_self_component.self_frags_eager, ompi_free_list_t); + OBJ_CONSTRUCT(&mca_bmi_self_component.self_frags_send, ompi_free_list_t); + OBJ_CONSTRUCT(&mca_bmi_self_component.self_frags_rdma, ompi_free_list_t); + return OMPI_SUCCESS; +} + + +/* + * component cleanup - sanity checking of queue lengths + */ + +int mca_bmi_self_component_close(void) +{ + OBJ_DESTRUCT(&mca_bmi_self_component.self_lock); + OBJ_DESTRUCT(&mca_bmi_self_component.self_frags_eager); + OBJ_DESTRUCT(&mca_bmi_self_component.self_frags_send); + OBJ_DESTRUCT(&mca_bmi_self_component.self_frags_rdma); + return OMPI_SUCCESS; +} + + +/* + * SELF component initialization + */ +mca_bmi_base_module_t** mca_bmi_self_component_init( + int *num_ptls, + bool enable_progress_threads, + bool enable_mpi_threads) +{ + mca_bmi_base_module_t **ptls = NULL; + *num_ptls = 0; + + /* allocate the Shared Memory PTL */ + *num_ptls = 1; + ptls = malloc((*num_ptls)*sizeof(mca_bmi_base_module_t*)); + if (NULL == ptls) { + return NULL; + } + + /* initialize free lists */ + ompi_free_list_init(&mca_bmi_self_component.self_frags_eager, + sizeof(mca_bmi_self_frag_eager_t) + mca_bmi_self.bmi_eager_limit, + OBJ_CLASS(mca_bmi_self_frag_eager_t), + mca_bmi_self_component.free_list_num, + mca_bmi_self_component.free_list_max, + mca_bmi_self_component.free_list_inc, + NULL); + ompi_free_list_init(&mca_bmi_self_component.self_frags_send, + sizeof(mca_bmi_self_frag_send_t) + mca_bmi_self.bmi_max_send_size, + OBJ_CLASS(mca_bmi_self_frag_send_t), + mca_bmi_self_component.free_list_num, + mca_bmi_self_component.free_list_max, + mca_bmi_self_component.free_list_inc, + NULL); + ompi_free_list_init(&mca_bmi_self_component.self_frags_rdma, + sizeof(mca_bmi_self_frag_rdma_t), + OBJ_CLASS(mca_bmi_self_frag_rdma_t), + mca_bmi_self_component.free_list_num, + mca_bmi_self_component.free_list_max, + mca_bmi_self_component.free_list_inc, + NULL); + + /* get pointer to the ptls */ + ptls[0] = (mca_bmi_base_module_t *)(&mca_bmi_self); + return ptls; +} + diff --git a/src/mca/bmi/self/bmi_self_frag.c b/src/mca/bmi/self/bmi_self_frag.c new file mode 100644 index 0000000000..fc0f529c30 --- /dev/null +++ b/src/mca/bmi/self/bmi_self_frag.c @@ -0,0 +1,55 @@ +#include "bmi_self_frag.h" + + +static inline void mca_bmi_self_frag_constructor(mca_bmi_self_frag_t* frag) +{ + frag->segment.seg_addr.pval = frag+1; + frag->segment.seg_len = frag->size; + frag->base.des_src = &frag->segment; + frag->base.des_src_cnt = 1; + frag->base.des_dst = NULL; + frag->base.des_dst_cnt = 0; + frag->base.des_flags = 0; +} + +static void mca_bmi_self_frag_eager_constructor(mca_bmi_self_frag_t* frag) +{ + frag->size = mca_bmi_self.bmi_eager_limit; + mca_bmi_self_frag_constructor(frag); +} + +static void mca_bmi_self_frag_send_constructor(mca_bmi_self_frag_t* frag) +{ + frag->size = mca_bmi_self.bmi_max_send_size; + mca_bmi_self_frag_constructor(frag); +} + +static void mca_bmi_self_frag_rdma_constructor(mca_bmi_self_frag_t* frag) +{ + frag->size = 0; + mca_bmi_self_frag_constructor(frag); +} + +static void mca_bmi_self_frag_destructor(mca_bmi_self_frag_t* frag) +{ +} + + +OBJ_CLASS_INSTANCE( + mca_bmi_self_frag_eager_t, + mca_bmi_base_descriptor_t, + mca_bmi_self_frag_eager_constructor, + mca_bmi_self_frag_destructor); + +OBJ_CLASS_INSTANCE( + mca_bmi_self_frag_send_t, + mca_bmi_base_descriptor_t, + mca_bmi_self_frag_send_constructor, + mca_bmi_self_frag_destructor); + +OBJ_CLASS_INSTANCE( + mca_bmi_self_frag_rdma_t, + mca_bmi_base_descriptor_t, + mca_bmi_self_frag_rdma_constructor, + mca_bmi_self_frag_destructor); + diff --git a/src/mca/bmi/self/bmi_self_frag.h b/src/mca/bmi/self/bmi_self_frag.h new file mode 100644 index 0000000000..3ffdf759ff --- /dev/null +++ b/src/mca/bmi/self/bmi_self_frag.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_BMI_SELF_SEND_FRAG_H +#define MCA_BMI_SELF_SEND_FRAG_H + +#include +#include "ompi_config.h" +#include "class/ompi_free_list.h" +#include "bmi_self.h" + + +/** + * shared memory send fragment derived type. + */ +struct mca_bmi_self_frag_t { + mca_bmi_base_descriptor_t base; + mca_bmi_base_segment_t segment; + struct mca_bmi_base_endpoint_t *endpoint; + size_t size; +}; +typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_t; +typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_eager_t; +typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_send_t; +typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_rdma_t; + +OBJ_CLASS_DECLARATION(mca_bmi_self_frag_eager_t); +OBJ_CLASS_DECLARATION(mca_bmi_self_frag_send_t); +OBJ_CLASS_DECLARATION(mca_bmi_self_frag_rdma_t); + + +#define MCA_BMI_SELF_FRAG_ALLOC_EAGER(frag, rc) \ +{ \ + ompi_list_item_t* item; \ + OMPI_FREE_LIST_WAIT(&mca_bmi_self_component.self_frags_eager, item, rc); \ + frag = (mca_bmi_self_frag_t*)item; \ +} + +#define MCA_BMI_SELF_FRAG_RETURN_EAGER(frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&mca_bmi_self_component.self_frags_eager, (ompi_list_item_t*)(frag)); \ +} + +#define MCA_BMI_SELF_FRAG_ALLOC_SEND(frag, rc) \ +{ \ + ompi_list_item_t* item; \ + OMPI_FREE_LIST_WAIT(&mca_bmi_self_component.self_frags_send, item, rc); \ + frag = (mca_bmi_self_frag_t*)item; \ +} + +#define MCA_BMI_SELF_FRAG_RETURN_SEND(frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&mca_bmi_self_component.self_frags_send, (ompi_list_item_t*)(frag)); \ +} + +#define MCA_BMI_SELF_FRAG_ALLOC_RDMA(frag, rc) \ +{ \ + ompi_list_item_t* item; \ + OMPI_FREE_LIST_WAIT(&mca_bmi_self_component.self_frags_rdma, item, rc); \ + frag = (mca_bmi_self_frag_t*)item; \ +} + +#define MCA_BMI_SELF_FRAG_RETURN_RDMA(frag) \ +{ \ + OMPI_FREE_LIST_RETURN(&mca_bmi_self_component.self_frags_rdma, (ompi_list_item_t*)(frag)); \ +} + +#endif + diff --git a/src/mca/bmi/self/bmi_sm_endpoint.h b/src/mca/bmi/self/bmi_sm_endpoint.h new file mode 100644 index 0000000000..2ed43c6948 --- /dev/null +++ b/src/mca/bmi/self/bmi_sm_endpoint.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_BMI_SELF_ENDPOINT_H +#define MCA_BMI_SELF_ENDPOINT_H + +#if OMPI_ENABLE_PROGRESS_THREADS == 1 +#include "event/event.h" +#endif + +/** + * An abstraction that represents a connection to a endpoint process. + * An instance of mca_ptl_base_endpoint_t is associated w/ each process + * and BMI pair at startup. + */ + +struct mca_bmi_base_endpoint_t { + int my_selfp_rank; /**< My SELFP process rank. Used for accessing + * SELFP specfic data structures. */ + int peer_selfp_rank; /**< My peer's SELFP process rank. Used for accessing + * SELFP specfic data structures. */ +#if OMPI_ENABLE_PROGRESS_THREADS == 1 + int fifo_fd; /**< pipe/fifo used to signal endpoint that data is queued */ +#endif +}; + +#endif +