loopback bmi

This commit was SVN r6069.

src/mca/bmi/self/.ompi_unignore

@@ -0,0 +1,2 @@
+twoodall
+gshipman

src/mca/bmi/self/Makefile.am

@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Use the top-level Makefile.options
+
+include $(top_ompi_srcdir)/config/Makefile.options
+
+libmca_bmi_self_la_sources = \
+    bmi_self.c \
+    bmi_self.h \
+    bmi_self_component.c \
+    bmi_self_frag.c \
+    bmi_self_frag.h 
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_bmi_self_DSO
+component_noinst =
+component_install = mca_bmi_self.la
+else
+component_noinst = libmca_bmi_self.la
+component_install =
+endif
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_bmi_self_la_SOURCES = $(libmca_bmi_self_la_sources)
+mca_bmi_self_la_LDFLAGS = -module -avoid-version
+mca_bmi_self_la_LIBADD = 
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_bmi_self_la_SOURCES = $(libmca_bmi_self_la_sources)
+libmca_bmi_self_la_LDFLAGS = -module -avoid-version
+libmca_bmi_self_la_LIBADD = 

src/mca/bmi/self/bmi_self.c

@@ -0,0 +1,287 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "threads/mutex.h"
+#include "datatype/convertor.h"
+#include "include/sys/atomic.h"
+#include "util/output.h"
+#include "util/if.h"
+#include "util/proc_info.h"
+#include "util/printf.h"
+#include "util/sys_info.h"
+#include "class/ompi_fifo.h"
+#include "class/ompi_free_list.h"
+#include "mca/pml/pml.h"
+#include "mca/bmi/bmi.h"
+#include "mca/mpool/base/base.h"
+#include "bmi_self.h"
+#include "bmi_self_frag.h"
+
+
+mca_bmi_base_module_t mca_bmi_self = {
+    &mca_bmi_self_component.super,
+    0, /* bmi_eager_limit */
+    0, /* bmi_min_send_size */
+    0, /* bmi_max_send_size */
+    0, /* bmi_min_rdma_size */
+    0, /* bmi_max_rdma_size */
+    0, /* bmi_exclusivity */
+    0, /* bmi_latency */
+    0, /* bmi_bandwidth */
+    0, /* bmi flags */
+    mca_bmi_self_add_procs_same_base_addr,
+    mca_bmi_self_del_procs,
+    mca_bmi_self_register,
+    mca_bmi_self_finalize,
+    mca_bmi_self_alloc,
+    mca_bmi_self_free,
+    mca_bmi_self_prepare_src,
+    NULL,
+    mca_bmi_self_send, 
+    NULL,  /* put */
+    NULL   /* get */
+};
+
+
+int mca_bmi_self_add_procs(
+    struct mca_bmi_base_module_t* bmi, 
+    size_t nprocs, 
+    struct ompi_proc_t **procs, 
+    struct mca_bmi_base_endpoint_t **peers,
+    ompi_bitmap_t* reachability)
+{
+    size_t i;
+    for(i=0; i<nprocs; i++) {
+
+    }
+    return OMPI_SUCCESS;
+}
+
+
+int mca_bmi_self_del_procs(
+    struct mca_bmi_base_module_t* bmi, 
+    size_t nprocs,
+    struct ompi_proc_t **procs, 
+    struct mca_bmi_base_endpoint_t **peers)
+{
+    return OMPI_SUCCESS;
+}
+
+
+/**
+ * MCA->BMI Clean up any resources held by BMI module
+ * before the module is unloaded.
+ *
+ * @param bmi (IN)   BMI module.
+ *
+ * Prior to unloading a BMI module, the MCA framework will call
+ * the BMI finalize method of the module. Any resources held by
+ * the BMI should be released and if required the memory corresponding
+ * to the BMI module freed.
+ *
+ */
+
+int mca_bmi_self_finalize(struct mca_bmi_base_module_t* bmi)
+{
+    return OMPI_SUCCESS;
+}
+
+
+/**
+ * Register a callback function that is called on receipt
+ * of a fragment.
+ *
+ * @param bmi (IN)     BMI module
+ * @return             Status indicating if cleanup was successful
+ *
+ * When the process list changes, the PML notifies the BMI of the
+ * change, to provide the opportunity to cleanup or release any
+ * resources associated with the peer.
+ */
+
+int mca_bmi_self_register(
+    struct mca_bmi_base_module_t* bmi,
+    mca_bmi_base_tag_t tag,
+    mca_bmi_base_module_recv_cb_fn_t cbfunc,
+    void* cbdata)
+{
+    mca_bmi_self_component.self_reg[tag].cbfunc = cbfunc;
+    mca_bmi_self_component.self_reg[tag].cbdata = cbdata;
+    return OMPI_SUCCESS;
+}
+                                                                                                                 
+
+/**
+ * Allocate a segment.
+ *
+ * @param bmi (IN)      BMI module
+ * @param size (IN)     Request segment size.
+ */
+extern mca_bmi_base_descriptor_t* mca_bmi_self_alloc(
+    struct mca_bmi_base_module_t* bmi,
+    size_t size)
+{
+    mca_bmi_self_frag_t* frag;
+    int rc;
+    if(size <= mca_bmi_self.bmi_eager_limit) {
+        MCA_BMI_SELF_FRAG_ALLOC_EAGER(frag,rc);
+    } else {
+        MCA_BMI_SELF_FRAG_ALLOC_SEND(frag,rc);
+    }
+    frag->base.des_flags = 0;
+    return (mca_bmi_base_descriptor_t*)frag;
+}
+                                                                                                                   
+/**
+ * Return a segment allocated by this BMI.
+ *
+ * @param bmi (IN)      BMI module
+ * @param segment (IN)  Allocated segment.
+ */
+extern int mca_bmi_self_free(
+    struct mca_bmi_base_module_t* bmi,
+    mca_bmi_base_descriptor_t* des)
+{
+    mca_bmi_self_frag_t* frag = (mca_bmi_self_frag_t*)des;
+    if(frag->size <= mca_bmi_self.bmi_eager_limit) {
+        MCA_BMI_SELF_FRAG_RETURN_EAGER(frag);
+    } else if (frag->size <= mca_bmi_self.bmi_max_send_size) {
+        MCA_BMI_SELF_FRAG_RETURN_SEND(frag);
+    } else {
+        MCA_BMI_SELF_FRAG_RETURN_RDMA(frag);
+    }
+    return OMPI_SUCCESS;
+}
+
+
+/**
+ * Prepare data for send/put
+ *
+ * @param bmi (IN)      BMI module
+ */
+struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_src(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_endpoint_t* endpoint,
+    struct ompi_convertor_t* convertor,
+    size_t reserve,
+    size_t* size)
+{
+    mca_bmi_self_frag_t* frag;
+    struct iovec iov;
+    uint32_t iov_count = 1;
+    size_t max_data = *size;
+    int32_t free_after;
+    int rc;
+
+    /* non-contigous data */
+    if(ompi_convertor_need_buffers(convertor)) {
+        MCA_BMI_SELF_FRAG_ALLOC_SEND(frag, rc);
+        if(NULL == frag) {
+            return NULL;
+        }
+
+        if(reserve + max_data > frag->size) {
+            max_data = frag->size - reserve;
+        } 
+        iov.iov_len = max_data;
+        iov.iov_base = (unsigned char*)(frag+1) + reserve;
+
+        rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
+        if(rc < 0) {
+            MCA_BMI_SELF_FRAG_RETURN_SEND(frag);
+            return NULL;
+        }
+        frag->base.des_flags = 0;
+        frag->segment.seg_addr.pval = frag+1;
+        frag->segment.seg_len = reserve + max_data;
+        *size = max_data;
+    } else {
+        MCA_BMI_SELF_FRAG_ALLOC_RDMA(frag, rc);
+        if(NULL == frag) {
+            return NULL;
+        }
+        iov.iov_len = max_data;
+        iov.iov_base = NULL;
+
+        /* convertor should return offset into users buffer */
+        rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
+        if(rc < 0) {
+            MCA_BMI_SELF_FRAG_RETURN_RDMA(frag);
+            return NULL;
+        }
+        frag->base.des_flags = 0;
+        frag->segment.seg_addr.pval = iov.iov_base;
+        frag->segment.seg_len = reserve + max_data;
+        *size = max_data;
+    }
+    return &frag->base;
+}
+
+/**
+ * Prepare data for receive.
+ */
+struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_dst(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_endpoint_t* endpoint,
+    struct ompi_convertor_t* convertor,
+    size_t reserve,
+    size_t* size)
+{
+    mca_bmi_self_frag_t* frag;
+    size_t max_data = *size;
+    int rc;
+
+    MCA_BMI_SELF_FRAG_ALLOC_RDMA(frag, rc);
+    if(NULL == frag) {
+        return NULL;
+    }
+
+    /* setup descriptor to point directly to user buffer */
+    frag->base.des_flags = 0;
+    frag->segment.seg_addr.pval = (unsigned char*)convertor->pBaseBuf + convertor->bConverted;
+    frag->segment.seg_len = reserve + max_data;
+    return &frag->base;
+}
+
+ 
+ 
+/**
+ * Initiate a send to the peer.
+ *
+ * @param bmi (IN)      BMI module
+ * @param peer (IN)     BMI peer addressing
+ */
+
+int mca_bmi_self_send(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_endpoint_t* endpoint,
+    struct mca_bmi_base_descriptor_t* descriptor,
+    mca_bmi_base_tag_t tag)
+{
+    mca_bmi_self_component.self_reg[tag].cbfunc(bmi,tag,descriptor,OMPI_SUCCESS);
+    return OMPI_SUCCESS;
+}
+
+

src/mca/bmi/self/bmi_self.h

@@ -0,0 +1,245 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_BMI_SELF_H
+#define MCA_BMI_SELF_H
+
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include "class/ompi_free_list.h"
+#include "class/ompi_bitmap.h"
+#include "class/ompi_fifo.h"
+#include "event/event.h"
+#include "mca/pml/pml.h"
+#include "mca/bmi/bmi.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/**
+ * Shared Memory (SELF) BMI module.
+ */
+struct mca_bmi_self_component_t {
+    mca_bmi_base_component_1_0_0_t super;  /**< base BMI component */
+    int free_list_num;                     /**< initial size of free lists */
+    int free_list_max;                     /**< maximum size of free lists */
+    int free_list_inc;                     /**< number of elements to alloc when growing free lists */
+    ompi_mutex_t self_lock;
+    ompi_free_list_t self_frags_eager;     /**< free list of self first */
+    ompi_free_list_t self_frags_send;      /**< free list of self second */
+    ompi_free_list_t self_frags_rdma;      /**< free list of self second */
+    mca_bmi_base_registration_t self_reg[256];
+};
+typedef struct mca_bmi_self_component_t mca_bmi_self_component_t;
+extern mca_bmi_self_component_t mca_bmi_self_component;
+
+/**
+ * Register shared memory module parameters with the MCA framework
+ */
+extern int mca_bmi_self_component_open(void);
+
+/**
+ * Any final cleanup before being unloaded.
+ */
+extern int mca_bmi_self_component_close(void);
+
+/**
+ * SELF module initialization.
+ * 
+ * @param num_bmis (OUT)                  Number of BMIs returned in BMI array.
+ * @param enable_progress_threads (IN)    Flag indicating whether BMI is allowed to have progress threads
+ * @param enable_mpi_threads (IN)         Flag indicating whether BMI must support multilple simultaneous invocations from different threads
+ *
+ */
+extern mca_bmi_base_module_t** mca_bmi_self_component_init(
+    int *num_bmis, 
+    bool enable_progress_threads,
+    bool enable_mpi_threads
+);
+
+extern mca_bmi_base_module_t mca_bmi_self;
+
+
+/**
+ * Cleanup any resources held by the BMI.
+ * 
+ * @param bmi  BMI instance.
+ * @return     OMPI_SUCCESS or error status on failure.
+ */
+
+extern int mca_bmi_self_finalize(
+    struct mca_bmi_base_module_t* bmi
+);
+
+
+/**
+ * PML->BMI notification of change in the process list.
+ * PML->BMI Notification that a receive fragment has been matched.
+ * Called for message that is send from process with the virtual
+ * address of the shared memory segment being different than that of
+ * the receiver.
+ * 
+ * @param bmi (IN)
+ * @param proc (IN)  
+ * @param peer (OUT)
+ * @return     OMPI_SUCCESS or error status on failure.
+ * 
+ */
+
+extern int mca_bmi_self_add_procs(
+    struct mca_bmi_base_module_t* bmi,
+    size_t nprocs,
+    struct ompi_proc_t **procs,
+    struct mca_bmi_base_endpoint_t** peers,
+    struct ompi_bitmap_t* reachability
+);
+
+                                                                                                               
+/**
+ * PML->BMI notification of change in the process list.
+ * PML->BMI Notification that a receive fragment has been matched.
+ * Called for message that is send from process with the virtual
+ * address of the shared memory segment being the same as that of
+ * the receiver.
+ * 
+ * @param bmi (IN)
+ * @param proc (IN)  
+ * @param peer (OUT)
+ * @return     OMPI_SUCCESS or error status on failure.
+ * 
+ */
+
+extern int mca_bmi_self_add_procs_same_base_addr(
+    struct mca_bmi_base_module_t* bmi,
+    size_t nprocs,
+    struct ompi_proc_t **procs,
+    struct mca_bmi_base_endpoint_t** peers,
+    ompi_bitmap_t* reachability
+);
+
+                                                                                                               
+/**
+ * PML->BMI notification of change in the process list.
+ *
+ * @param bmi (IN)     BMI instance
+ * @param proc (IN)    Peer process
+ * @param peer (IN)    Peer addressing information.
+ * @return             Status indicating if cleanup was successful
+ *
+ */
+extern int mca_bmi_self_del_procs(
+    struct mca_bmi_base_module_t* bmi,
+    size_t nprocs,
+    struct ompi_proc_t **procs,
+    struct mca_bmi_base_endpoint_t **peers
+);
+
+
+/**
+ * Register a callback function that is called on receipt
+ * of a fragment.
+ *
+ * @param bmi (IN)     BMI module
+ * @return             Status indicating if cleanup was successful
+ *
+ * When the process list changes, the PML notifies the BMI of the
+ * change, to provide the opportunity to cleanup or release any
+ * resources associated with the peer.
+ */
+
+extern int mca_bmi_self_register(
+    struct mca_bmi_base_module_t* bmi,
+    mca_bmi_base_tag_t tag,
+    mca_bmi_base_module_recv_cb_fn_t cbfunc,
+    void* cbdata
+);
+                                                                                                                   
+
+/**
+ * Allocate a segment.
+ *
+ * @param bmi (IN)      BMI module
+ * @param size (IN)     Request segment size.
+ */
+extern mca_bmi_base_descriptor_t* mca_bmi_self_alloc(
+    struct mca_bmi_base_module_t* bmi,
+    size_t size
+);
+
+/**
+ * Return a segment allocated by this BMI.
+ *
+ * @param bmi (IN)      BMI module
+ * @param segment (IN)  Allocated segment.
+ */
+extern int mca_bmi_self_free(
+    struct mca_bmi_base_module_t* bmi,
+    mca_bmi_base_descriptor_t* segment
+);
+                                                                                                                   
+/**
+ * Pack data
+ *
+ * @param bmi (IN)      BMI module
+ * @param peer (IN)     BMI peer addressing
+ */
+struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_src(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_endpoint_t* endpoint,
+    struct ompi_convertor_t* convertor,
+    size_t reserve,
+    size_t* size
+);
+                                                                                                            
+/**
+ * Prepare data for RDMA
+ *
+ * @param bmi (IN)      BMI module
+ * @param peer (IN)     BMI peer addressing
+ */
+struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_dst(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_endpoint_t* endpoint,
+    struct ompi_convertor_t* convertor,
+    size_t reserve,
+    size_t* size
+);
+                                                                                                            
+/**
+ * Initiate a send to the peer.
+ *
+ * @param bmi (IN)      BMI module
+ * @param peer (IN)     BMI peer addressing
+ */
+extern int mca_bmi_self_send(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_endpoint_t* endpoint,
+    struct mca_bmi_base_descriptor_t* descriptor,
+    mca_bmi_base_tag_t tag
+);
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+

src/mca/bmi/self/bmi_self_component.c

@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>  /* for mkfifo */
+
+#include "include/constants.h"
+#include "include/sys/cache.h"
+#include "event/event.h"
+#include "util/if.h"
+#include "util/argv.h"
+#include "util/output.h"
+#include "util/sys_info.h"
+#include "util/proc_info.h"
+#include "mca/pml/pml.h"
+#include "mca/base/mca_base_param.h"
+#include "bmi_self.h"
+#include "bmi_self_frag.h"
+
+
+
+/*
+ * Shared Memory (SELF) component instance. 
+ */
+
+mca_bmi_self_component_t mca_bmi_self_component = {
+    {  /* super is being filled in */
+        /* First, the mca_base_component_t struct containing meta information
+          about the component itself */
+        {
+            /* Indicate that we are a pml v1.0.0 component (which also implies a
+               specific MCA version) */
+            MCA_BMI_BASE_VERSION_1_0_0,
+            "self", /* MCA component name */
+            1,  /* MCA component major version */
+            0,  /* MCA component minor version */
+            0,  /* MCA component release version */
+            mca_bmi_self_component_open,  /* component open */
+            mca_bmi_self_component_close  /* component close */
+        },
+
+        /* Next the MCA v1.0.0 component meta data */
+        {
+            /* Whether the component is checkpointable or not */
+            false
+        },
+
+        mca_bmi_self_component_init,  
+        NULL,
+    }  /* end super */
+};
+
+
+/*
+ * utility routines for parameter registration
+ */
+
+static inline char* mca_bmi_self_param_register_string(
+    const char* param_name, 
+    const char* default_value)
+{
+    char *param_value;
+    int id = mca_base_param_register_string("ptl","self",param_name,NULL,default_value);
+    mca_base_param_lookup_string(id, &param_value);
+    return param_value;
+}
+                                                                                                                            
+static inline int mca_bmi_self_param_register_int(
+    const char* param_name, 
+    int default_value)
+{
+    int id = mca_base_param_register_int("ptl","self",param_name,NULL,default_value);
+    int param_value = default_value;
+    mca_base_param_lookup_int(id,&param_value);
+    return param_value;
+}
+
+
+/*
+ *  Called by MCA framework to open the component, registers
+ *  component parameters.
+ */
+
+int mca_bmi_self_component_open(void)
+{
+    /* register SELF component parameters */
+    mca_bmi_self_component.free_list_num =
+        mca_bmi_self_param_register_int("free_list_num", 256);
+    mca_bmi_self_component.free_list_max =
+        mca_bmi_self_param_register_int("free_list_max", -1);
+    mca_bmi_self_component.free_list_inc =
+        mca_bmi_self_param_register_int("free_list_inc", 256);
+    mca_bmi_self.bmi_eager_limit = 
+        mca_bmi_self_param_register_int("eager_limit", 8*1024);
+    mca_bmi_self.bmi_min_send_size = 
+    mca_bmi_self.bmi_max_send_size = 
+        mca_bmi_self_param_register_int("max_send_size", 64*1024);
+    mca_bmi_self.bmi_min_rdma_size = 
+    mca_bmi_self.bmi_max_rdma_size =
+        mca_bmi_self_param_register_int("max_rdma_size", -1);
+    mca_bmi_self.bmi_flags =
+        mca_bmi_self_param_register_int("flags", 1);
+
+    /* initialize objects */
+    OBJ_CONSTRUCT(&mca_bmi_self_component.self_lock, ompi_mutex_t);
+    OBJ_CONSTRUCT(&mca_bmi_self_component.self_frags_eager, ompi_free_list_t);
+    OBJ_CONSTRUCT(&mca_bmi_self_component.self_frags_send, ompi_free_list_t);
+    OBJ_CONSTRUCT(&mca_bmi_self_component.self_frags_rdma, ompi_free_list_t);
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ * component cleanup - sanity checking of queue lengths
+ */
+
+int mca_bmi_self_component_close(void)
+{
+    OBJ_DESTRUCT(&mca_bmi_self_component.self_lock);
+    OBJ_DESTRUCT(&mca_bmi_self_component.self_frags_eager);
+    OBJ_DESTRUCT(&mca_bmi_self_component.self_frags_send);
+    OBJ_DESTRUCT(&mca_bmi_self_component.self_frags_rdma);
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ *  SELF component initialization
+ */
+mca_bmi_base_module_t** mca_bmi_self_component_init(
+    int *num_ptls, 
+    bool enable_progress_threads,
+    bool enable_mpi_threads)
+{
+    mca_bmi_base_module_t **ptls = NULL;
+    *num_ptls = 0;
+
+    /* allocate the Shared Memory PTL */
+    *num_ptls = 1;
+    ptls = malloc((*num_ptls)*sizeof(mca_bmi_base_module_t*));
+    if (NULL == ptls) {
+        return NULL;
+    }
+
+    /* initialize free lists */
+    ompi_free_list_init(&mca_bmi_self_component.self_frags_eager, 
+        sizeof(mca_bmi_self_frag_eager_t) + mca_bmi_self.bmi_eager_limit,
+        OBJ_CLASS(mca_bmi_self_frag_eager_t),
+        mca_bmi_self_component.free_list_num,
+        mca_bmi_self_component.free_list_max,
+        mca_bmi_self_component.free_list_inc,
+        NULL);
+    ompi_free_list_init(&mca_bmi_self_component.self_frags_send, 
+        sizeof(mca_bmi_self_frag_send_t) + mca_bmi_self.bmi_max_send_size,
+        OBJ_CLASS(mca_bmi_self_frag_send_t),
+        mca_bmi_self_component.free_list_num,
+        mca_bmi_self_component.free_list_max,
+        mca_bmi_self_component.free_list_inc,
+        NULL);
+    ompi_free_list_init(&mca_bmi_self_component.self_frags_rdma, 
+        sizeof(mca_bmi_self_frag_rdma_t),
+        OBJ_CLASS(mca_bmi_self_frag_rdma_t),
+        mca_bmi_self_component.free_list_num,
+        mca_bmi_self_component.free_list_max,
+        mca_bmi_self_component.free_list_inc,
+        NULL);
+
+    /* get pointer to the ptls */
+    ptls[0] = (mca_bmi_base_module_t *)(&mca_bmi_self);
+    return ptls;
+}
+

src/mca/bmi/self/bmi_self_frag.c

@@ -0,0 +1,55 @@
+#include "bmi_self_frag.h"
+
+
+static inline void mca_bmi_self_frag_constructor(mca_bmi_self_frag_t* frag)
+{
+    frag->segment.seg_addr.pval = frag+1;
+    frag->segment.seg_len = frag->size;
+    frag->base.des_src = &frag->segment;
+    frag->base.des_src_cnt = 1;
+    frag->base.des_dst = NULL;
+    frag->base.des_dst_cnt = 0;
+    frag->base.des_flags = 0;
+}
+
+static void mca_bmi_self_frag_eager_constructor(mca_bmi_self_frag_t* frag)
+{
+    frag->size = mca_bmi_self.bmi_eager_limit;
+    mca_bmi_self_frag_constructor(frag);
+}
+
+static void mca_bmi_self_frag_send_constructor(mca_bmi_self_frag_t* frag)
+{
+    frag->size = mca_bmi_self.bmi_max_send_size;
+    mca_bmi_self_frag_constructor(frag);
+}
+
+static void mca_bmi_self_frag_rdma_constructor(mca_bmi_self_frag_t* frag)
+{
+    frag->size = 0;
+    mca_bmi_self_frag_constructor(frag);
+}
+
+static void mca_bmi_self_frag_destructor(mca_bmi_self_frag_t* frag)
+{
+}
+
+
+OBJ_CLASS_INSTANCE(
+    mca_bmi_self_frag_eager_t,
+    mca_bmi_base_descriptor_t,
+    mca_bmi_self_frag_eager_constructor,
+    mca_bmi_self_frag_destructor);
+
+OBJ_CLASS_INSTANCE(
+    mca_bmi_self_frag_send_t,
+    mca_bmi_base_descriptor_t,
+    mca_bmi_self_frag_send_constructor,
+    mca_bmi_self_frag_destructor);
+
+OBJ_CLASS_INSTANCE(
+    mca_bmi_self_frag_rdma_t,
+    mca_bmi_base_descriptor_t,
+    mca_bmi_self_frag_rdma_constructor,
+    mca_bmi_self_frag_destructor);
+

src/mca/bmi/self/bmi_self_frag.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_BMI_SELF_SEND_FRAG_H
+#define MCA_BMI_SELF_SEND_FRAG_H
+
+#include <sys/types.h>
+#include "ompi_config.h"
+#include "class/ompi_free_list.h"
+#include "bmi_self.h"
+
+
+/**
+ * shared memory send fragment derived type.
+ */
+struct mca_bmi_self_frag_t {
+    mca_bmi_base_descriptor_t base;
+    mca_bmi_base_segment_t segment;
+    struct mca_bmi_base_endpoint_t *endpoint;
+    size_t size;
+};
+typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_t;
+typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_eager_t;
+typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_send_t;
+typedef struct mca_bmi_self_frag_t mca_bmi_self_frag_rdma_t;
+
+OBJ_CLASS_DECLARATION(mca_bmi_self_frag_eager_t);
+OBJ_CLASS_DECLARATION(mca_bmi_self_frag_send_t);
+OBJ_CLASS_DECLARATION(mca_bmi_self_frag_rdma_t);
+
+
+#define MCA_BMI_SELF_FRAG_ALLOC_EAGER(frag, rc)                              \
+{                                                                            \
+    ompi_list_item_t* item;                                                  \
+    OMPI_FREE_LIST_WAIT(&mca_bmi_self_component.self_frags_eager, item, rc); \
+    frag = (mca_bmi_self_frag_t*)item;                                       \
+}
+
+#define MCA_BMI_SELF_FRAG_RETURN_EAGER(frag)                                 \
+{                                                                            \
+    OMPI_FREE_LIST_RETURN(&mca_bmi_self_component.self_frags_eager, (ompi_list_item_t*)(frag)); \
+}
+
+#define MCA_BMI_SELF_FRAG_ALLOC_SEND(frag, rc)                               \
+{                                                                            \
+    ompi_list_item_t* item;                                                  \
+    OMPI_FREE_LIST_WAIT(&mca_bmi_self_component.self_frags_send, item, rc);  \
+    frag = (mca_bmi_self_frag_t*)item;                                       \
+}
+
+#define MCA_BMI_SELF_FRAG_RETURN_SEND(frag)                                  \
+{                                                                            \
+    OMPI_FREE_LIST_RETURN(&mca_bmi_self_component.self_frags_send, (ompi_list_item_t*)(frag)); \
+}
+
+#define MCA_BMI_SELF_FRAG_ALLOC_RDMA(frag, rc)                               \
+{                                                                            \
+    ompi_list_item_t* item;                                                  \
+    OMPI_FREE_LIST_WAIT(&mca_bmi_self_component.self_frags_rdma, item, rc);  \
+    frag = (mca_bmi_self_frag_t*)item;                                       \
+}
+
+#define MCA_BMI_SELF_FRAG_RETURN_RDMA(frag)                                  \
+{                                                                            \
+    OMPI_FREE_LIST_RETURN(&mca_bmi_self_component.self_frags_rdma, (ompi_list_item_t*)(frag)); \
+}
+
+#endif
+

src/mca/bmi/self/bmi_sm_endpoint.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_BMI_SELF_ENDPOINT_H
+#define MCA_BMI_SELF_ENDPOINT_H
+
+#if OMPI_ENABLE_PROGRESS_THREADS == 1
+#include "event/event.h"
+#endif
+
+/**
+ *  An abstraction that represents a connection to a endpoint process.
+ *  An instance of mca_ptl_base_endpoint_t is associated w/ each process 
+ *  and BMI pair at startup.
+ */
+
+struct mca_bmi_base_endpoint_t {
+    int my_selfp_rank;    /**< My SELFP process rank.  Used for accessing
+                         *   SELFP specfic data structures. */
+    int peer_selfp_rank;  /**< My peer's SELFP process rank.  Used for accessing
+                         *   SELFP specfic data structures. */
+#if OMPI_ENABLE_PROGRESS_THREADS == 1
+    int fifo_fd;        /**< pipe/fifo used to signal endpoint that data is queued */
+#endif
+};
+
+#endif
+