get the sm MCA "stuff" fixed, fix a bug in memory alignment for the

sm mpool allocation, create sm ptl control shared memory region,
    add comments.

This commit was SVN r2042.

src/mca/common/sm/common_sm_mmap.c

@@ -81,10 +81,10 @@ mca_common_sm_mmap_t* mca_common_sm_mmap_init(size_t size, char *file_name,
     mca_common_sm_mmap_t* map;
     struct stat s_stat;
     unsigned char *addr;
-    size_t tmp;
+    size_t tmp,mem_offset;
 
     /* input parameter error checks */
-    if( (size <= sizeof(mca_common_sm_segment_t) ) ||
+    if( (size < sizeof(mca_common_sm_segment_t) ) ||
                 ( file_name == NULL ) || 
                 ( size_ctl_structure < sizeof(mca_common_sm_segment_t ) ) ||
                 ( data_seg_alignment == 0 ) ) {
@@ -141,26 +141,34 @@ mca_common_sm_mmap_t* mca_common_sm_mmap_init(size_t size, char *file_name,
     /* set up the map object */
     map = OBJ_NEW(mca_common_sm_mmap_t);
     strncpy(map->map_path, file_name, PATH_MAX);
+    /* the first entry in the file is the control strcuture.  the first
+       entry in the control structure is an mca_common_sm_segment_t
+       element */
     map->map_seg = seg;
+
     /* allign start of data segment */
     addr=(unsigned char *)seg+size_ctl_structure;
-    tmp=(size_t)(addr+1+data_seg_alignment);
+    tmp=(size_t)(addr+1);
     addr=(unsigned char *)( (tmp/data_seg_alignment)*data_seg_alignment);
     /* is addr past end of file ? */
     if( (unsigned char*)seg+size < addr ){
-        ompi_output(0, "mca_common_sm_mmap_init: memory region too small len %d \n",size);
+        ompi_output(0, "mca_common_sm_mmap_init: memory region too small len %d  addr %p\n",
+                size,addr);
+        fchmod(fd, 0600);
         close(fd);
         munmap(seg,size);
         return NULL;
     }
-    map->map_addr = addr;
+    mem_offset=addr-(unsigned char *)seg;
-    map->map_size = size - (addr-(unsigned char *)seg);
+    map->map_addr = seg;
+    map->data_addr = addr;
+    map->map_size = size;
 
     /* initialize the segment - only the first process to open the file */
     if( !file_previously_opened ) {
         spinunlock(&seg->seg_lock);
         seg->seg_inited = false;
-        seg->seg_offset = 0;
+        seg->seg_offset = mem_offset;
         seg->seg_size = size;
     }
 
@@ -178,6 +186,14 @@ mca_common_sm_mmap_t* mca_common_sm_mmap_init(size_t size, char *file_name,
 }
 
 
+/**
+ *  allocate memory from a previously allocated shared memory
+ *  block.
+ *
+ *  @param size size of request, in bytes (IN)
+ * 
+ *  @retval addr virtual address
+ */
 void* mca_common_sm_mmap_alloc(size_t* size)
 {
     mca_common_sm_mmap_t* map = mca_common_sm_mmap;
@@ -188,7 +204,8 @@ void* mca_common_sm_mmap_alloc(size_t* size)
     if(seg->seg_offset + *size > map->map_size) {
         addr = NULL;
     } else {
-        addr = map->map_addr + seg->seg_offset;
+        /* add base address to segment offset */
+        addr = map->data_addr + seg->seg_offset;
         seg->seg_offset += *size;
     }
     spinunlock(&seg->seg_lock);

src/mca/common/sm/common_sm_mmap.h

@@ -6,18 +6,26 @@
 #include "class/ompi_list.h"
 
 struct mca_common_sm_segment_t {
+    /* lock to control atomic access */
     ompi_lock_data_t seg_lock;
+    /* is the segment ready for use */
     volatile bool seg_inited;
+    /* Offset to next available memory location available for allocation */
     size_t seg_offset;
+    /* total size of the segment */
     size_t seg_size;
 };
 typedef struct mca_common_sm_segment_t mca_common_sm_segment_t;
 
 
 struct mca_common_sm_mmap_t {
+    /* double link list element */
     ompi_list_item_t map_item;
     mca_common_sm_segment_t* map_seg;
+    /* base address of the mmap'ed file */
     unsigned char  *map_addr;
+    /* base address of data segment */
+    unsigned char  *data_addr;
     size_t map_size;
     char map_path[PATH_MAX];
 };

src/mca/mpool/base/mpool_base_open.c

@@ -35,7 +35,8 @@ ompi_list_t mca_mpool_base_modules;
  */
 int mca_mpool_base_open(void)
 {
-  /* Open up all available components */
+  /* Open up all available components - and populate the
+     mca_mpool_base_components list */
 
   if (OMPI_SUCCESS != 
       mca_base_components_open("mpool", 0, mca_mpool_base_static_components, 

src/mca/mpool/sm/mpool_sm_component.c

@@ -127,6 +127,9 @@ mca_mpool_sm_init(bool *allow_multi_user_threads)
         return NULL;
     }
 
+    /* setup function pointers */
+
+
     /* setup allocator */
     mca_mpool_sm_component.sm_allocator = allocator_component->allocator_init(
         allow_multi_user_threads,

src/mca/ptl/sm/src/ptl_sm.c

@@ -64,15 +64,6 @@ int mca_ptl_sm_add_procs(
     bool threads;
     char file_name[PATH_MAX];
 
-    /* debug */
-    fprintf(stderr," mca_ptlsm_add_procs \n");
-    fflush(stderr);
-    /* end debug */
-
-    /* initialize the shared memory pool */
-    /*mca_mpool_component_lookup("sm"); */
-    /*mca_mpool_sm_init(&threads); */
-
     /* initializion */
     for(i=0 ; i < nprocs ; i++ ) {
         peers[i]=NULL;
@@ -188,18 +179,14 @@ int mca_ptl_sm_add_procs(
             ompi_process_info.job_session_dir,
             ompi_system_info.nodename);
     size=sizeof(mca_ptl_sm_module_resource_t);
-    /* debug */
-    fprintf(stderr," AAA %s \n",
-            mca_ptl_sm_component.sm_resouce_ctl_file);
-    fflush(stderr);
-    /* end debug */
     if(NULL == 
             (mca_ptl_sm_component.resource_ctl = 
              mca_common_sm_mmap_init(size,
                  mca_ptl_sm_component.sm_resouce_ctl_file,
                  sizeof(mca_ptl_sm_module_resource_t), 8 ))) 
     {
-        ompi_output(0, "mca_ptl_sm_add_procs: unable to create shared memory PTL coordinating strucure\n");
+        ompi_output(0, "mca_ptl_sm_add_procs: unable to create shared memory PTL coordinating strucure :: size %ld \n",
+                size);
         return_code=OMPI_ERROR;
         goto CLEANUP;
     }

src/mca/ptl/sm/src/ptl_sm.h

@@ -38,15 +38,15 @@ struct mca_ptl_sm_component_t {
     char* sm_mpool_name;                  /**< name of shared memory pool module */
     mca_mpool_base_module_t* sm_mpool; /**< shared memory pool */
     void* sm_mpool_base;                  /**< base address of shared memory pool */
+    ompi_free_list_t sm_send_requests;    /**< free list of sm send requests -- sendreq + sendfrag */
+    ompi_free_list_t sm_send_frags;       /**< free list of sm send fragments */
+    ompi_free_list_t sm_recv_frags;       /**< free list of sm recv fragments */
+    ompi_mutex_t sm_lock;
     char* sm_resouce_ctl_file;     /**< name of shared memory file used 
                                             to coordinate resource usage */
     mca_ptl_sm_module_resource_t *resource_ctl;
                                           /**< shared memory coordination data.
                                             Resides in shared memory */
-    ompi_free_list_t sm_send_requests;    /**< free list of sm send requests -- sendreq + sendfrag */
-    ompi_free_list_t sm_send_frags;       /**< free list of sm send fragments */
-    ompi_free_list_t sm_recv_frags;       /**< free list of sm recv fragments */
-    ompi_mutex_t sm_lock;
 };
 typedef struct mca_ptl_sm_component_t mca_ptl_sm_component_t;
 extern mca_ptl_sm_component_t mca_ptl_sm_component;

src/mca/ptl/sm/src/ptl_sm_component.c

@@ -25,6 +25,7 @@
 #include "ptl_sm_sendreq.h"
 #include "ptl_sm_sendfrag.h"
 #include "ptl_sm_recvfrag.h"
+#include "mca/common/sm/common_sm_mmap.h"
 
 
 
@@ -152,17 +153,9 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
     *have_hidden_threads = OMPI_HAVE_THREADS;
 
     /* lookup shared memory pool */
-    sm_mpool_component = mca_mpool_component_lookup(mca_ptl_sm_component.sm_mpool_name);
-
     mca_ptl_sm_component.sm_mpool =
-        sm_mpool_component->mpool_init(allow_multi_user_threads); /* RLG */
+        mca_mpool_module_lookup(mca_ptl_sm_component.sm_mpool_name);
-    /*mca_ptl_sm_component.sm_mpool =
+
-     * mca_mpool_component_lookup(mca_ptl_sm_component.sm_mpool_name);  RLG */
-    if(NULL == mca_ptl_sm_component.sm_mpool) {
-        ompi_output(0, "mca_ptl_sm_component_init: unable to locate shared memory pool: %s\n",
-            mca_ptl_sm_component.sm_mpool_name);
-        return NULL;
-    }
     mca_ptl_sm_component.sm_mpool_base = mca_ptl_sm_component.sm_mpool->mpool_base();
 
     /* initialize free lists */