changes for the shared memory implementation.

This commit was SVN r1872.

src/mca/mpool/sm/mpool_sm_component.c

@@ -8,6 +8,8 @@
 #include "mca/allocator/base/base.h"
 #include "mpool_sm.h"
 #include "mpool_sm_mmap.h"
+#include "util/proc_info.h"
+#include "util/sys_info.h"
 
 /*
  * Local functions
@@ -84,6 +86,7 @@ static int mca_mpool_sm_open(void)
 static mca_mpool_base_module_t* 
 mca_mpool_sm_init(bool *allow_multi_user_threads)
 {
+    char file_name[PATH_MAX];
     mca_allocator_base_component_t* allocator_component = mca_allocator_component_lookup(
         mca_mpool_sm_component.sm_allocator_name);
 
@@ -103,7 +106,23 @@ mca_mpool_sm_init(bool *allow_multi_user_threads)
     }
     
     /* create initial shared memory mapping */
-    if(NULL == (mca_mpool_sm_component.sm_mmap = mca_mpool_sm_mmap_init(mca_mpool_sm_component.sm_size))) {
+    memset(&(file_name[0]),0,PATH_MAX);
+    if( (strlen(ompi_process_info.job_session_dir) +
+            strlen(ompi_system_info.nodename)+
+            /* length of fixe name part */
+            17 ) >= PATH_MAX ) {
+            ompi_output(0, "mca_mpool_sm_init: name of backing file too long \n");
+            return NULL;
+    }
+    sprintf(&(file_name[0]),"%s/shared_mem_pool.%s",
+            ompi_process_info.job_session_dir,
+            ompi_system_info.nodename);
+    if(NULL == 
+            (mca_mpool_sm_component.sm_mmap = 
+             mca_mpool_sm_mmap_init(mca_mpool_sm_component.sm_size,
+                 &(file_name[0]),sizeof(mca_mpool_sm_segment_t), 8 )
+             )) 
+    {
         ompi_output(0, "mca_mpool_sm_init: unable to create shared memory mapping");
         return NULL;
     }

src/mca/mpool/sm/mpool_sm_mmap.c

@@ -25,101 +25,149 @@ OBJ_CLASS_INSTANCE(
 );
 
 
-static mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_open(char* path)
+static int mca_mpool_sm_mmap_open(char* path)
 {
-    mca_mpool_sm_mmap_t* map;
-    mca_mpool_sm_segment_t* seg;
     int fd = -1;
-    struct stat sbuf;
 
+    /* loop until file can be opened, or until an erro, other than
+     * access error, occurs */
     while(fd < 0) {
         struct timespec ts;
         fd = open(path, O_CREAT|O_RDWR, 0000); 
         if(fd < 0 && errno != EACCES) {
             ompi_output(0, 
             "mca_ptl_sm_mmap_open: open %s failed with errno=%d\n", path, errno);
-            return NULL;
+            return -1;
         }
         ts.tv_sec = 0; 
         ts.tv_nsec = 500000;
         nanosleep(&ts,NULL);
     }
 
-    if(fstat(fd, &sbuf) != 0) {
+    /* return */
-        ompi_output(0, "mca_mpool_sm_mmap_open: fstat failed with errno=%d\n", errno);
+    return fd;
-        return NULL;
-    }
 
-    /* map the file and initialize segment state */
-    seg = mmap(NULL, sbuf.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-    if(NULL == seg) {
-        ompi_output(0, "mca_mpool_sm_mmap_open: mmap failed with errno=%d\n", errno);
-        return NULL;
-    }
-    close(fd);
-                                                                                              
-    map = OBJ_NEW(mca_mpool_sm_mmap_t);
-    map->map_seg = seg;
-    map->map_addr = (unsigned char*)(seg + 1);
-    map->map_size = seg->seg_size - sizeof(mca_mpool_sm_segment_t);
-    close(fd);
-    return map;
 }
 
+/**
+ *  This routine is used to set up a shared memory file, backed
+ *  by a specified file.  It is assumed that the file does not
+ *  exist before any of the current set of processes try and open
+ *  it.
+ *
+ *  @param size - size of the file, in bytes (IN)
+ *
+ *  @param file_name  name of file to be opened. (IN)
+ *
+ *  @param size_ctl_structure  size of the control structure at
+ *                             the head of the file. The control structure 
+ *                             is assumed to have mca_mpool_sm_segment_t
+ *                             as its first segment (IN)
+ *
+ *  @param data_set_alignment  alignment of the data segment.  this
+ *                             follows the control structure (IN)
+ */
 
-mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size)
+mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size, char *file_name, 
+        size_t size_ctl_structure, size_t data_seg_alignment)
 {
-    int fd;
+    int fd,return_code=OMPI_SUCCESS;
+    bool file_previously_opened;
     mca_mpool_sm_segment_t* seg;
     mca_mpool_sm_mmap_t* map;
-    char path[PATH_MAX];
+    struct stat s_stat;
+    unsigned char *addr;
+    size_t tmp;
 
-    sprintf(path, "%s/mmap.%s", ompi_process_info.job_session_dir, ompi_system_info.nodename);
+    /* input parameter error checks */
-    /* debug */
+    if( (size <= sizeof(mca_mpool_sm_segment_t) ) ||
-    fprintf(stderr," open %s \n",path);
+                ( file_name == NULL ) || 
-    fflush(stderr);
+                ( size_ctl_structure < sizeof(mca_mpool_sm_segment_t ) ) ||
-    /* end debug */
+                ( data_seg_alignment == 0 ) ) {
-    fd = open(path, O_CREAT|O_RDWR, 0000); 
-    if(fd < 0) {
-        if(errno == EACCES)
-            return mca_mpool_sm_mmap_open(path);
-        ompi_output(0, 
-        "mca_mpool_sm_mmap_init: open %s failed with errno=%d\n", 
-        path,errno);
         return NULL;
     }
 
-    /* truncate the file to the requested size */
+    /* debug */
-    if(ftruncate(fd, size) != 0) {
+    fprintf(stderr," open %s \n",file_name);
-        ompi_output(0, "mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n", errno);
+    fflush(stderr);
+    /* end debug */
+
+    /* open the backing file - only the first process accessing this
+     * file will succeed. */
+    fd=mca_mpool_sm_mmap_open(file_name);
+    if( -1 == fd ) {
+        ompi_output(0, "mca_mpool_sm_mmap_init: mca_mpool_sm_mmap_open failed \n");
         return NULL;
     }
 
+    /* figure out if I am first to attach to file */
+    file_previously_opened=false;
+    return_code=fstat(fd,&s_stat);
+    if( 0 > return_code ) {
+        ompi_output(0, "mca_mpool_sm_mmap_init: fstat failed with errno=%d\n", errno);
+        close(fd);
+        return NULL;
+    }
+    if(0 < s_stat.st_size){
+        file_previously_opened=true;
+    }
+
+    /* first process to open the file, so needs to
+     * initialize it */
+    if( !file_previously_opened ) {
+        /* truncate the file to the requested size */
+        if(ftruncate(fd, size) != 0) {
+            ompi_output(0, 
+                    "mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n",
+                    errno);
+            close(fd);
+            return NULL;
+        }
+    }
+
     /* map the file and initialize segment state */
     seg = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-    if(seg == (void*)-1) {
+    if( (void*)-1 == seg ) {
-        ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n", errno);
+        ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n",
+                errno);
+        close(fd);
         return NULL;
     }
 
-    spinunlock(&seg->seg_lock);
+    /* set up the map object */
-    seg->seg_offset = 0;
-    seg->seg_size = size;
-
     map = OBJ_NEW(mca_mpool_sm_mmap_t);
-    strncpy(map->map_path, path, PATH_MAX);
+    strncpy(map->map_path, file_name, PATH_MAX);
     map->map_seg = seg;
-    map->map_addr = (unsigned char*)(seg+1);
+    /* allign start of data segment */
-    map->map_size = size - sizeof(mca_mpool_sm_segment_t);
+    addr=(unsigned char *)seg+size_ctl_structure;
+    tmp=(size_t)(addr+1+data_seg_alignment);
+    addr=(unsigned char *)( (tmp/data_seg_alignment)*data_seg_alignment);
+    /* is addr past end of file ? */
+    if( (unsigned char*)seg+size < addr ){
+        ompi_output(0, "mca_mpool_sm_mmap_init: memory region too small len %d \n",size);
+        close(fd);
+        munmap(seg,size);
+        return NULL;
+    }
+    map->map_addr = addr;
+    map->map_size = size - (addr-(unsigned char *)seg);
+    close(fd);
+
+    /* initialize the segment - only the first process to open the file */
+    if( !file_previously_opened ) {
+        spinunlock(&seg->seg_lock);
+        seg->seg_inited = false;
+        seg->seg_offset = 0;
+        seg->seg_size = size;
+    }
 
     /* enable access by other processes on this host */
     if(fchmod(fd, 0600) != 0) {
         ompi_output(0, "mca_mpool_sm_mmap_init: fchmod failed with errno=%d\n", errno);
         OBJ_RELEASE(map);
-        close(fd);
         return NULL;
     }
-    close(fd);
+
     return map;
 }
 

src/mca/mpool/sm/mpool_sm_mmap.h

@@ -7,6 +7,7 @@
 
 struct mca_mpool_sm_segment_t {
     ompi_lock_data_t seg_lock;
+    volatile bool seg_inited;
     size_t seg_offset;
     size_t seg_size;
 };
@@ -25,7 +26,27 @@ typedef struct mca_mpool_sm_mmap_t mca_mpool_sm_mmap_t;
 OBJ_CLASS_DECLARATION(mca_mpool_sm_mmap_t);
 
 
-mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size);
+/**
+ *  This routine is used to set up a shared memory file, backed
+ *  by a specified file.  It is assumed that the file does not
+ *  exist before any of the current set of processes try and open
+ *  it.
+ *
+ *  @param size - size of the file, in bytes (IN)
+ *
+ *  @param file_name  name of file to be opened. (IN)
+ *
+ *  @param size_ctl_structure  size of the control structure at
+ *                             the head of the file. The control structure
+ *                             is assumed to have mca_mpool_sm_segment_t
+ *                             as its first segment (IN)
+ *
+ *  @param data_set_alignment  alignment of the data segment.  this
+ *                             follows the control structure (IN)
+ */
+
+mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size, char *file_name,
+                size_t size_ctl_structure, size_t data_seg_alignment);
 void* mca_mpool_sm_mmap_alloc(size_t* size);
 void  mca_mpool_sm_mmap_free(void* addr);