diff --git a/src/mca/mpool/sm/mpool_sm_component.c b/src/mca/mpool/sm/mpool_sm_component.c index d3cb02d4d2..0dc28ec385 100644 --- a/src/mca/mpool/sm/mpool_sm_component.c +++ b/src/mca/mpool/sm/mpool_sm_component.c @@ -8,6 +8,8 @@ #include "mca/allocator/base/base.h" #include "mpool_sm.h" #include "mpool_sm_mmap.h" +#include "util/proc_info.h" +#include "util/sys_info.h" /* * Local functions @@ -84,6 +86,7 @@ static int mca_mpool_sm_open(void) static mca_mpool_base_module_t* mca_mpool_sm_init(bool *allow_multi_user_threads) { + char file_name[PATH_MAX]; mca_allocator_base_component_t* allocator_component = mca_allocator_component_lookup( mca_mpool_sm_component.sm_allocator_name); @@ -103,7 +106,23 @@ mca_mpool_sm_init(bool *allow_multi_user_threads) } /* create initial shared memory mapping */ - if(NULL == (mca_mpool_sm_component.sm_mmap = mca_mpool_sm_mmap_init(mca_mpool_sm_component.sm_size))) { + memset(&(file_name[0]),0,PATH_MAX); + if( (strlen(ompi_process_info.job_session_dir) + + strlen(ompi_system_info.nodename)+ + /* length of fixe name part */ + 17 ) >= PATH_MAX ) { + ompi_output(0, "mca_mpool_sm_init: name of backing file too long \n"); + return NULL; + } + sprintf(&(file_name[0]),"%s/shared_mem_pool.%s", + ompi_process_info.job_session_dir, + ompi_system_info.nodename); + if(NULL == + (mca_mpool_sm_component.sm_mmap = + mca_mpool_sm_mmap_init(mca_mpool_sm_component.sm_size, + &(file_name[0]),sizeof(mca_mpool_sm_segment_t), 8 ) + )) + { ompi_output(0, "mca_mpool_sm_init: unable to create shared memory mapping"); return NULL; } diff --git a/src/mca/mpool/sm/mpool_sm_mmap.c b/src/mca/mpool/sm/mpool_sm_mmap.c index 06ccc0f51f..868d23db68 100644 --- a/src/mca/mpool/sm/mpool_sm_mmap.c +++ b/src/mca/mpool/sm/mpool_sm_mmap.c @@ -25,101 +25,149 @@ OBJ_CLASS_INSTANCE( ); -static mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_open(char* path) +static int mca_mpool_sm_mmap_open(char* path) { - mca_mpool_sm_mmap_t* map; - mca_mpool_sm_segment_t* seg; int fd = -1; - struct stat sbuf; + /* loop until file can be opened, or until an erro, other than + * access error, occurs */ while(fd < 0) { struct timespec ts; fd = open(path, O_CREAT|O_RDWR, 0000); if(fd < 0 && errno != EACCES) { ompi_output(0, "mca_ptl_sm_mmap_open: open %s failed with errno=%d\n", path, errno); - return NULL; + return -1; } ts.tv_sec = 0; ts.tv_nsec = 500000; nanosleep(&ts,NULL); } - if(fstat(fd, &sbuf) != 0) { - ompi_output(0, "mca_mpool_sm_mmap_open: fstat failed with errno=%d\n", errno); - return NULL; - } + /* return */ + return fd; - /* map the file and initialize segment state */ - seg = mmap(NULL, sbuf.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); - if(NULL == seg) { - ompi_output(0, "mca_mpool_sm_mmap_open: mmap failed with errno=%d\n", errno); - return NULL; - } - close(fd); - - map = OBJ_NEW(mca_mpool_sm_mmap_t); - map->map_seg = seg; - map->map_addr = (unsigned char*)(seg + 1); - map->map_size = seg->seg_size - sizeof(mca_mpool_sm_segment_t); - close(fd); - return map; } +/** + * This routine is used to set up a shared memory file, backed + * by a specified file. It is assumed that the file does not + * exist before any of the current set of processes try and open + * it. + * + * @param size - size of the file, in bytes (IN) + * + * @param file_name name of file to be opened. (IN) + * + * @param size_ctl_structure size of the control structure at + * the head of the file. The control structure + * is assumed to have mca_mpool_sm_segment_t + * as its first segment (IN) + * + * @param data_set_alignment alignment of the data segment. this + * follows the control structure (IN) + */ -mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size) +mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size, char *file_name, + size_t size_ctl_structure, size_t data_seg_alignment) { - int fd; + int fd,return_code=OMPI_SUCCESS; + bool file_previously_opened; mca_mpool_sm_segment_t* seg; mca_mpool_sm_mmap_t* map; - char path[PATH_MAX]; + struct stat s_stat; + unsigned char *addr; + size_t tmp; - sprintf(path, "%s/mmap.%s", ompi_process_info.job_session_dir, ompi_system_info.nodename); - /* debug */ - fprintf(stderr," open %s \n",path); - fflush(stderr); - /* end debug */ - fd = open(path, O_CREAT|O_RDWR, 0000); - if(fd < 0) { - if(errno == EACCES) - return mca_mpool_sm_mmap_open(path); - ompi_output(0, - "mca_mpool_sm_mmap_init: open %s failed with errno=%d\n", - path,errno); + /* input parameter error checks */ + if( (size <= sizeof(mca_mpool_sm_segment_t) ) || + ( file_name == NULL ) || + ( size_ctl_structure < sizeof(mca_mpool_sm_segment_t ) ) || + ( data_seg_alignment == 0 ) ) { return NULL; } - /* truncate the file to the requested size */ - if(ftruncate(fd, size) != 0) { - ompi_output(0, "mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n", errno); + /* debug */ + fprintf(stderr," open %s \n",file_name); + fflush(stderr); + /* end debug */ + + /* open the backing file - only the first process accessing this + * file will succeed. */ + fd=mca_mpool_sm_mmap_open(file_name); + if( -1 == fd ) { + ompi_output(0, "mca_mpool_sm_mmap_init: mca_mpool_sm_mmap_open failed \n"); return NULL; } + /* figure out if I am first to attach to file */ + file_previously_opened=false; + return_code=fstat(fd,&s_stat); + if( 0 > return_code ) { + ompi_output(0, "mca_mpool_sm_mmap_init: fstat failed with errno=%d\n", errno); + close(fd); + return NULL; + } + if(0 < s_stat.st_size){ + file_previously_opened=true; + } + + /* first process to open the file, so needs to + * initialize it */ + if( !file_previously_opened ) { + /* truncate the file to the requested size */ + if(ftruncate(fd, size) != 0) { + ompi_output(0, + "mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n", + errno); + close(fd); + return NULL; + } + } + /* map the file and initialize segment state */ seg = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); - if(seg == (void*)-1) { - ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n", errno); + if( (void*)-1 == seg ) { + ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n", + errno); + close(fd); return NULL; } - spinunlock(&seg->seg_lock); - seg->seg_offset = 0; - seg->seg_size = size; - + /* set up the map object */ map = OBJ_NEW(mca_mpool_sm_mmap_t); - strncpy(map->map_path, path, PATH_MAX); + strncpy(map->map_path, file_name, PATH_MAX); map->map_seg = seg; - map->map_addr = (unsigned char*)(seg+1); - map->map_size = size - sizeof(mca_mpool_sm_segment_t); + /* allign start of data segment */ + addr=(unsigned char *)seg+size_ctl_structure; + tmp=(size_t)(addr+1+data_seg_alignment); + addr=(unsigned char *)( (tmp/data_seg_alignment)*data_seg_alignment); + /* is addr past end of file ? */ + if( (unsigned char*)seg+size < addr ){ + ompi_output(0, "mca_mpool_sm_mmap_init: memory region too small len %d \n",size); + close(fd); + munmap(seg,size); + return NULL; + } + map->map_addr = addr; + map->map_size = size - (addr-(unsigned char *)seg); + close(fd); + + /* initialize the segment - only the first process to open the file */ + if( !file_previously_opened ) { + spinunlock(&seg->seg_lock); + seg->seg_inited = false; + seg->seg_offset = 0; + seg->seg_size = size; + } /* enable access by other processes on this host */ if(fchmod(fd, 0600) != 0) { ompi_output(0, "mca_mpool_sm_mmap_init: fchmod failed with errno=%d\n", errno); OBJ_RELEASE(map); - close(fd); return NULL; } - close(fd); + return map; } diff --git a/src/mca/mpool/sm/mpool_sm_mmap.h b/src/mca/mpool/sm/mpool_sm_mmap.h index 87fc0b230a..f1e49724e2 100644 --- a/src/mca/mpool/sm/mpool_sm_mmap.h +++ b/src/mca/mpool/sm/mpool_sm_mmap.h @@ -7,6 +7,7 @@ struct mca_mpool_sm_segment_t { ompi_lock_data_t seg_lock; + volatile bool seg_inited; size_t seg_offset; size_t seg_size; }; @@ -25,7 +26,27 @@ typedef struct mca_mpool_sm_mmap_t mca_mpool_sm_mmap_t; OBJ_CLASS_DECLARATION(mca_mpool_sm_mmap_t); -mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size); +/** + * This routine is used to set up a shared memory file, backed + * by a specified file. It is assumed that the file does not + * exist before any of the current set of processes try and open + * it. + * + * @param size - size of the file, in bytes (IN) + * + * @param file_name name of file to be opened. (IN) + * + * @param size_ctl_structure size of the control structure at + * the head of the file. The control structure + * is assumed to have mca_mpool_sm_segment_t + * as its first segment (IN) + * + * @param data_set_alignment alignment of the data segment. this + * follows the control structure (IN) + */ + +mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size, char *file_name, + size_t size_ctl_structure, size_t data_seg_alignment); void* mca_mpool_sm_mmap_alloc(size_t* size); void mca_mpool_sm_mmap_free(void* addr);