Initial commit of uDAPL mpool component. Just a stripped copy of the GM mpool component right now.

This commit was SVN r8802.

ompi/mca/mpool/udapl/.ompi_unignore

@@ -0,0 +1 @@
+afriedle

ompi/mca/mpool/udapl/Makefile.am

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Use the top-level Makefile.options
+
+
+
+AM_CPPFLAGS = $(mpool_udapl_CPPFLAGS)
+
+sources = \
+	mpool_udapl.h \
+	mpool_udapl_module.c \
+	mpool_udapl_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_mpool_udapl_DSO
+component_noinst =
+component_install = mca_mpool_udapl.la
+else
+component_noinst = libmca_mpool_udapl.la
+component_install =
+endif
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_mpool_udapl_la_SOURCES = $(sources)
+mca_mpool_udapl_la_LIBADD = \
+    $(mpool_udapl_LIBS) \
+    $(top_ompi_builddir)/ompi/libmpi.la \
+    $(top_ompi_builddir)/orte/liborte.la \
+    $(top_ompi_builddir)/opal/libopal.la
+mca_mpool_udapl_la_LDFLAGS = -module -avoid-version $(mpool_udapl_LDFLAGS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_mpool_udapl_la_SOURCES = $(sources)
+libmca_mpool_udapl_la_LIBADD = $(mpool_udapl_LIBS)
+libmca_mpool_udapl_la_LDFLAGS = -module -avoid-version $(mpool_udapl_LDFLAGS)

ompi/mca/mpool/udapl/configure.m4

@@ -0,0 +1,47 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+
+# MCA_mpool_udapl_CONFIG([action-if-can-compile], 
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_mpool_udapl_CONFIG],[
+    OMPI_CHECK_UDAPL([mpool_udapl],
+                     [mpool_udapl_happy="yes"],
+                     [mpool_udapl_happy="no"])
+
+    AS_IF([test "$mpool_udapl_happy" = "yes"],
+          [mpool_udapl_WRAPPER_EXTRA_LDFLAGS="$mpool_udapl_LDFLAGS"
+           mpool_udapl_WRAPPER_EXTRA_LIBS="$mpool_udapl_LIBS"
+           $1],
+          [$2])
+
+    # Borrowed from MVAPI BTL - a data structure in the uDAPL headers
+    # is not fully ISO C.  Remove -pedantic to silence a warning.
+    btl_udapl_CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
+    AS_IF([test "$btl_udapl_CFLAGS" != "$CFLAGS" -a "$btl_udapl_happy" = "yes"],
+          [AC_MSG_WARN([Removed -pedantic from CFLAGS for
+uDAPL component because the uDAPL headers are not fully ISO C])])
+
+    # substitute in the things needed to build uDAPL
+    AC_SUBST([mpool_udapl_CFLAGS])
+    AC_SUBST([mpool_udapl_CPPFLAGS])
+    AC_SUBST([mpool_udapl_LDFLAGS])
+    AC_SUBST([mpool_udapl_LIBS])
+])dnl

ompi/mca/mpool/udapl/configure.params

@@ -0,0 +1,24 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_INIT_FILE=mpool_udapl_module.c
+PARAM_CONFIG_FILES="Makefile"
+

ompi/mca/mpool/udapl/mpool_udapl.h

@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_MPOOL_UDAPL_H
+#define MCA_MPOOL_UDAPL_H
+
+#include "ompi_config.h"
+#include <dat/udat.h>
+#include "opal/class/opal_list.h"
+#include "class/ompi_free_list.h"
+#include "opal/event/event.h"
+#include "mca/mpool/mpool.h"
+#include "mca/allocator/allocator.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/*#define MCA_MPOOL_GM_HAVE_REGISTER*/
+
+struct mca_mpool_udapl_component_t {
+    mca_mpool_base_component_t super;
+    char* rcache_name; 
+};
+
+typedef struct mca_mpool_udapl_component_t mca_mpool_udapl_component_t;
+
+OMPI_COMP_EXPORT extern mca_mpool_udapl_component_t mca_mpool_udapl_component;
+
+
+
+struct mca_mpool_base_resources_t {
+    DAT_IA_HANDLE udapl_ia;
+}; 
+typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;  
+
+struct mca_mpool_udapl_module_t {
+    mca_mpool_base_module_t super;
+    DAT_IA_HANDLE udapl_ia;
+}; 
+typedef struct mca_mpool_udapl_module_t mca_mpool_udapl_module_t; 
+
+
+
+/* 
+ *  Initializes the mpool module. 
+ */ 
+void mca_mpool_udapl_module_init(mca_mpool_udapl_module_t* mpool); 
+
+
+/*
+ *  Returns base address of shared memory mapping.
+ */
+void* mca_mpool_udapl_base(mca_mpool_base_module_t*);
+
+/**
+  *  Allocate block of shared memory.
+  */
+void* mca_mpool_udapl_alloc( 
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    uint32_t flags, 
+    mca_mpool_base_registration_t** registration);
+
+/**
+  * realloc function typedef
+  */
+void* mca_mpool_udapl_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration);
+
+/**
+  * register function typedef
+  */
+int mca_mpool_udapl_register(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    uint32_t flags, 
+    mca_mpool_base_registration_t** registration);
+
+int mca_mpool_udapl_deregister(
+    mca_mpool_base_module_t* mpool, 
+    mca_mpool_base_registration_t*);
+
+
+/**
+  * free function typedef
+  */
+void mca_mpool_udapl_free(
+    mca_mpool_base_module_t* mpool, 
+    void * addr, 
+    mca_mpool_base_registration_t* registration);
+
+int mca_mpool_udapl_find(
+                      struct mca_mpool_base_module_t* mpool, 
+                      void* addr, 
+                      size_t size, 
+                      ompi_pointer_array_t *regs,
+                      uint32_t *cnt
+                      );
+    
+int mca_mpool_udapl_release(
+                            struct mca_mpool_base_module_t* mpool, 
+                            mca_mpool_base_registration_t* registraion
+                            ); 
+
+int mca_mpool_udapl_retain(
+                            struct mca_mpool_base_module_t* mpool, 
+                            mca_mpool_base_registration_t* registraion
+                            ); 
+
+
+
+void* mca_common_udapl_segment_alloc(
+    struct mca_mpool_base_module_t* module, 
+    size_t* size, 
+    mca_mpool_base_registration_t** registration);
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

ompi/mca/mpool/udapl/mpool_udapl_component.c

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "opal/util/output.h"
+#include "mca/base/base.h"
+#include "mca/base/mca_base_param.h"
+#include "mca/allocator/base/base.h"
+#include "mpool_udapl.h"
+#include "util/proc_info.h"
+#include "util/sys_info.h"
+#include <unistd.h> 
+
+/*
+ * Local functions
+ */
+static int mca_mpool_udapl_open(void);
+static mca_mpool_base_module_t* mca_mpool_udapl_init(
+    struct mca_mpool_base_resources_t* resources);
+
+mca_mpool_udapl_component_t mca_mpool_udapl_component = {
+    {
+      /* First, the mca_base_component_t struct containing meta
+         information about the component itself */
+
+      {
+        /* Indicate that we are a mpool v1.0.0 component (which also
+           implies a specific MCA version) */
+
+        MCA_MPOOL_BASE_VERSION_1_0_0,
+
+        "udapl", /* MCA component name */
+        OMPI_MAJOR_VERSION,  /* MCA component major version */
+        OMPI_MINOR_VERSION,  /* MCA component minor version */
+        OMPI_RELEASE_VERSION,  /* MCA component release version */
+        mca_mpool_udapl_open,  /* component open  */
+        NULL /* component close */
+      },
+
+      /* Next the MCA v1.0.0 component meta data */
+      
+      {
+        /* Whether the component is checkpointable or not */
+        false
+      },
+
+      mca_mpool_udapl_init
+    }
+};
+
+
+
+
+/**
+  * component open/close/init function
+  */
+static int mca_mpool_udapl_open(void)
+{
+    opal_output(0, "mpool_udapl_open\n");
+    return OMPI_SUCCESS;
+}
+
+
+/* Allocates a segment of memory and registers with uDAPL, user_out returns the memory handle. */ 
+static mca_mpool_base_module_t* mca_mpool_udapl_init(
+     struct mca_mpool_base_resources_t* resources)
+{
+    mca_mpool_udapl_module_t* udapl_mpool; 
+
+    opal_output(0, "mpool_udapl_init\n");
+
+    mca_base_param_reg_string(&mca_mpool_udapl_component.super.mpool_version, 
+                              "rcache_name", 
+                              "The name of the registration cache the mpool should use", 
+                              false, 
+                              false, 
+                              "rb", 
+                              &(mca_mpool_udapl_component.rcache_name)); 
+
+    udapl_mpool = (mca_mpool_udapl_module_t*)malloc(sizeof(mca_mpool_udapl_module_t)); 
+    mca_mpool_udapl_module_init(udapl_mpool); 
+    udapl_mpool->udapl_ia = resources->udapl_ia;
+    return &udapl_mpool->super;
+}
+
+

ompi/mca/mpool/udapl/mpool_udapl_module.c

@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <string.h>
+#include "opal/util/output.h"
+#include "mpool_udapl.h"
+#include "ompi/mca/rcache/rcache.h"
+#include "ompi/mca/rcache/base/base.h"
+#include "ompi/mca/mpool/base/base.h"
+
+extern uint32_t mca_mpool_base_page_size;
+extern uint32_t mca_mpool_base_page_size_log; 
+uint64_t mca_mpool_udapl_mem_registered; 
+
+/* 
+ *  Initializes the mpool module.
+ */ 
+void mca_mpool_udapl_module_init(mca_mpool_udapl_module_t* mpool)
+{
+    opal_output(0, "mpool_udapl_module_init\n");
+
+    mpool->super.mpool_component = &mca_mpool_udapl_component.super; 
+    mpool->super.mpool_base = NULL; /* no base .. */ 
+    mpool->super.mpool_alloc = mca_mpool_udapl_alloc; 
+    mpool->super.mpool_realloc = mca_mpool_udapl_realloc; 
+    mpool->super.mpool_free = mca_mpool_udapl_free; 
+    mpool->super.mpool_register = mca_mpool_udapl_register; 
+    mpool->super.mpool_deregister = mca_mpool_udapl_deregister; 
+    mpool->super.mpool_find = mca_mpool_udapl_find; 
+    mpool->super.mpool_retain = mca_mpool_udapl_retain;
+    mpool->super.mpool_release = mca_mpool_udapl_release; 
+    mpool->super.mpool_finalize = NULL; 
+    mpool->super.rcache = 
+        mca_rcache_base_module_create(mca_mpool_udapl_component.rcache_name);
+    mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM; 
+    mca_mpool_udapl_mem_registered = 0;
+}
+
+
+/**
+  * allocate function 
+  */
+void* mca_mpool_udapl_alloc(
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    uint32_t flags, 
+    mca_mpool_base_registration_t** registration)
+{
+    void *addr = malloc(size+align);
+
+    opal_output(0, "mpool_udapl_alloc\n");
+
+    if(NULL == addr)
+        return NULL;
+    if(OMPI_SUCCESS != mca_mpool_udapl_register(mpool,addr,size+align, flags, registration)) {
+        free(addr);
+        return NULL;
+    }
+    return addr;
+}
+
+
+/* 
+ * register memory 
+ */ 
+int mca_mpool_udapl_register(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    uint32_t flags, 
+    mca_mpool_base_registration_t** registration)
+{
+    mca_mpool_udapl_module_t * udapl_mpool = (mca_mpool_udapl_module_t*) mpool; 
+    mca_mpool_base_registration_t* reg = OBJ_NEW(mca_mpool_base_registration_t);
+    int rc;
+
+    opal_output(0, "mpool_udapl_register\n");
+
+    if(NULL == reg) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    reg->mpool = mpool;
+    reg->base = down_align_addr(addr, mca_mpool_base_page_size_log); 
+    reg->flags = flags; 
+    reg->bound = up_align_addr((void*) ((unsigned long) addr + size -1)
+                               , mca_mpool_base_page_size_log);
+    
+#if 0
+    if((rc = gm_register_memory(udapl_mpool->port, reg->base, reg->bound - reg->base + 1)) != GM_SUCCESS) {
+        opal_output(0, "[%s:%d] error(%d) registering udapl memory\n", __FILE__, __LINE__, rc);
+        assert(0);
+        OBJ_RELEASE(reg);
+        
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    mca_mpool_udapl_mem_registered += reg->bound - reg->base + 1;
+#endif
+    
+    OPAL_THREAD_ADD32(&reg->ref_count,1);
+
+    if(flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) { 
+        mpool->rcache->rcache_insert(mpool->rcache, 
+                                     (mca_mpool_base_registration_t*) reg, 
+                                     flags); 
+    }
+    *registration = reg;
+    /* opal_output(0,"registered memory from %p to %p\n", reg->base, reg->bound); */
+    return OMPI_SUCCESS; 
+}
+
+
+/* 
+ * deregister memory 
+ */ 
+int mca_mpool_udapl_deregister(mca_mpool_base_module_t* mpool, 
+                              mca_mpool_base_registration_t* reg)
+{
+    int rc;
+
+    opal_output(0, "mpool_udapl_deregister\n");
+
+    if(reg->flags & (MCA_MPOOL_FLAGS_CACHE | MCA_MPOOL_FLAGS_PERSIST)) { 
+        mpool->rcache->rcache_delete(mpool->rcache, 
+                                     reg, 
+                                     reg->flags); 
+    }
+#if 0
+    if((rc = mca_mpool_udapl_release(mpool, reg)) != GM_SUCCESS) { 
+        opal_output(0, "[%s:%d] error(%d) deregistering udapl memory\n", __FILE__, __LINE__, rc);
+        assert(0);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+
+    }
+#endif
+    return OMPI_SUCCESS;
+}
+
+/**
+  * realloc function 
+  */
+void* mca_mpool_udapl_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration)
+{
+    void *new_addr = mca_mpool_udapl_alloc(mpool,size,0, (*registration)->flags, registration);
+
+    opal_output(0, "mpool_udapl_realloc\n");
+
+    if(new_addr == NULL) {
+        return NULL;
+    }
+    memcpy(new_addr,addr,size);
+    mca_mpool_udapl_free(mpool,addr,*registration);
+    return new_addr;
+}
+
+/**
+  * free function 
+  */
+void mca_mpool_udapl_free(mca_mpool_base_module_t* mpool, void * addr,
+                         mca_mpool_base_registration_t* registration)
+{
+    opal_output(0, "mpool_udapl_free\n");
+
+    mpool->mpool_deregister(mpool, registration);
+    free(addr);
+}
+
+int mca_mpool_udapl_find(
+    struct mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    ompi_pointer_array_t *regs,
+    uint32_t *cnt)
+{
+    opal_output(0, "mpool_udapl_find\n");
+
+    return mpool->rcache->rcache_find(mpool->rcache, 
+                                      addr, 
+                                      size, 
+                                      regs, 
+                                      cnt); 
+}
+ 
+int mca_mpool_udapl_release(
+    struct mca_mpool_base_module_t* mpool, 
+    mca_mpool_base_registration_t* reg)
+{
+    mca_mpool_udapl_module_t * mpool_udapl = (mca_mpool_udapl_module_t*) mpool; 
+
+    opal_output(0, "mpool_udapl_release\n");
+
+    if(0 == OPAL_THREAD_ADD32(&reg->ref_count, -1)) {
+#if 0
+        int rc = gm_deregister_memory(
+                                  mpool_udapl->port,
+                                  reg->base,
+                                  reg->bound - reg->base + 1);
+        if(GM_SUCCESS != rc) { 
+            opal_output(0, "[%s:%d] error(%d) deregistering udapl memory\n", __FILE__, __LINE__, rc); 
+            return OMPI_ERROR; 
+        }
+#endif
+        /* opal_output(0,"deregistering udapl memory\n"); */
+        mca_mpool_udapl_mem_registered -= reg->bound - reg->base + 1;
+        OBJ_RELEASE(reg);
+    }
+    else { 
+        /* opal_output(0, "release says ref_count is %d\n", reg->ref_count);  */
+    }
+    
+    return OMPI_SUCCESS; 
+}
+
+int mca_mpool_udapl_retain(
+    struct mca_mpool_base_module_t* mpool, 
+    mca_mpool_base_registration_t* registration)
+{
+    opal_output(0, "mpool_udapl_retain\n");
+    OPAL_THREAD_ADD32(&registration->ref_count, 1); 
+    return OMPI_SUCCESS; 
+}
+