changes for the shared memory implementation.
This commit was SVN r1872.
Этот коммит содержится в:
Rich Graham
родитель
bddf372ac1
Коммит
5e196fd257
@ -8,6 +8,8 @@
|
||||
#include "mca/allocator/base/base.h"
|
||||
#include "mpool_sm.h"
|
||||
#include "mpool_sm_mmap.h"
|
||||
#include "util/proc_info.h"
|
||||
#include "util/sys_info.h"
|
||||
|
||||
/*
|
||||
* Local functions
|
||||
@ -84,6 +86,7 @@ static int mca_mpool_sm_open(void)
|
||||
static mca_mpool_base_module_t*
|
||||
mca_mpool_sm_init(bool *allow_multi_user_threads)
|
||||
{
|
||||
char file_name[PATH_MAX];
|
||||
mca_allocator_base_component_t* allocator_component = mca_allocator_component_lookup(
|
||||
mca_mpool_sm_component.sm_allocator_name);
|
||||
|
||||
@ -103,7 +106,23 @@ mca_mpool_sm_init(bool *allow_multi_user_threads)
|
||||
}
|
||||
|
||||
/* create initial shared memory mapping */
|
||||
if(NULL == (mca_mpool_sm_component.sm_mmap = mca_mpool_sm_mmap_init(mca_mpool_sm_component.sm_size))) {
|
||||
memset(&(file_name[0]),0,PATH_MAX);
|
||||
if( (strlen(ompi_process_info.job_session_dir) +
|
||||
strlen(ompi_system_info.nodename)+
|
||||
/* length of fixe name part */
|
||||
17 ) >= PATH_MAX ) {
|
||||
ompi_output(0, "mca_mpool_sm_init: name of backing file too long \n");
|
||||
return NULL;
|
||||
}
|
||||
sprintf(&(file_name[0]),"%s/shared_mem_pool.%s",
|
||||
ompi_process_info.job_session_dir,
|
||||
ompi_system_info.nodename);
|
||||
if(NULL ==
|
||||
(mca_mpool_sm_component.sm_mmap =
|
||||
mca_mpool_sm_mmap_init(mca_mpool_sm_component.sm_size,
|
||||
&(file_name[0]),sizeof(mca_mpool_sm_segment_t), 8 )
|
||||
))
|
||||
{
|
||||
ompi_output(0, "mca_mpool_sm_init: unable to create shared memory mapping");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -25,101 +25,149 @@ OBJ_CLASS_INSTANCE(
|
||||
);
|
||||
|
||||
|
||||
static mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_open(char* path)
|
||||
static int mca_mpool_sm_mmap_open(char* path)
|
||||
{
|
||||
mca_mpool_sm_mmap_t* map;
|
||||
mca_mpool_sm_segment_t* seg;
|
||||
int fd = -1;
|
||||
struct stat sbuf;
|
||||
|
||||
/* loop until file can be opened, or until an erro, other than
|
||||
* access error, occurs */
|
||||
while(fd < 0) {
|
||||
struct timespec ts;
|
||||
fd = open(path, O_CREAT|O_RDWR, 0000);
|
||||
if(fd < 0 && errno != EACCES) {
|
||||
ompi_output(0,
|
||||
"mca_ptl_sm_mmap_open: open %s failed with errno=%d\n", path, errno);
|
||||
return NULL;
|
||||
return -1;
|
||||
}
|
||||
ts.tv_sec = 0;
|
||||
ts.tv_nsec = 500000;
|
||||
nanosleep(&ts,NULL);
|
||||
}
|
||||
|
||||
if(fstat(fd, &sbuf) != 0) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_open: fstat failed with errno=%d\n", errno);
|
||||
return NULL;
|
||||
}
|
||||
/* return */
|
||||
return fd;
|
||||
|
||||
/* map the file and initialize segment state */
|
||||
seg = mmap(NULL, sbuf.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
if(NULL == seg) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_open: mmap failed with errno=%d\n", errno);
|
||||
return NULL;
|
||||
}
|
||||
close(fd);
|
||||
|
||||
map = OBJ_NEW(mca_mpool_sm_mmap_t);
|
||||
map->map_seg = seg;
|
||||
map->map_addr = (unsigned char*)(seg + 1);
|
||||
map->map_size = seg->seg_size - sizeof(mca_mpool_sm_segment_t);
|
||||
close(fd);
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* This routine is used to set up a shared memory file, backed
|
||||
* by a specified file. It is assumed that the file does not
|
||||
* exist before any of the current set of processes try and open
|
||||
* it.
|
||||
*
|
||||
* @param size - size of the file, in bytes (IN)
|
||||
*
|
||||
* @param file_name name of file to be opened. (IN)
|
||||
*
|
||||
* @param size_ctl_structure size of the control structure at
|
||||
* the head of the file. The control structure
|
||||
* is assumed to have mca_mpool_sm_segment_t
|
||||
* as its first segment (IN)
|
||||
*
|
||||
* @param data_set_alignment alignment of the data segment. this
|
||||
* follows the control structure (IN)
|
||||
*/
|
||||
|
||||
mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size)
|
||||
mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size, char *file_name,
|
||||
size_t size_ctl_structure, size_t data_seg_alignment)
|
||||
{
|
||||
int fd;
|
||||
int fd,return_code=OMPI_SUCCESS;
|
||||
bool file_previously_opened;
|
||||
mca_mpool_sm_segment_t* seg;
|
||||
mca_mpool_sm_mmap_t* map;
|
||||
char path[PATH_MAX];
|
||||
struct stat s_stat;
|
||||
unsigned char *addr;
|
||||
size_t tmp;
|
||||
|
||||
sprintf(path, "%s/mmap.%s", ompi_process_info.job_session_dir, ompi_system_info.nodename);
|
||||
/* debug */
|
||||
fprintf(stderr," open %s \n",path);
|
||||
fflush(stderr);
|
||||
/* end debug */
|
||||
fd = open(path, O_CREAT|O_RDWR, 0000);
|
||||
if(fd < 0) {
|
||||
if(errno == EACCES)
|
||||
return mca_mpool_sm_mmap_open(path);
|
||||
ompi_output(0,
|
||||
"mca_mpool_sm_mmap_init: open %s failed with errno=%d\n",
|
||||
path,errno);
|
||||
/* input parameter error checks */
|
||||
if( (size <= sizeof(mca_mpool_sm_segment_t) ) ||
|
||||
( file_name == NULL ) ||
|
||||
( size_ctl_structure < sizeof(mca_mpool_sm_segment_t ) ) ||
|
||||
( data_seg_alignment == 0 ) ) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* truncate the file to the requested size */
|
||||
if(ftruncate(fd, size) != 0) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n", errno);
|
||||
/* debug */
|
||||
fprintf(stderr," open %s \n",file_name);
|
||||
fflush(stderr);
|
||||
/* end debug */
|
||||
|
||||
/* open the backing file - only the first process accessing this
|
||||
* file will succeed. */
|
||||
fd=mca_mpool_sm_mmap_open(file_name);
|
||||
if( -1 == fd ) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_init: mca_mpool_sm_mmap_open failed \n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* figure out if I am first to attach to file */
|
||||
file_previously_opened=false;
|
||||
return_code=fstat(fd,&s_stat);
|
||||
if( 0 > return_code ) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_init: fstat failed with errno=%d\n", errno);
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
if(0 < s_stat.st_size){
|
||||
file_previously_opened=true;
|
||||
}
|
||||
|
||||
/* first process to open the file, so needs to
|
||||
* initialize it */
|
||||
if( !file_previously_opened ) {
|
||||
/* truncate the file to the requested size */
|
||||
if(ftruncate(fd, size) != 0) {
|
||||
ompi_output(0,
|
||||
"mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n",
|
||||
errno);
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* map the file and initialize segment state */
|
||||
seg = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
if(seg == (void*)-1) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n", errno);
|
||||
if( (void*)-1 == seg ) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n",
|
||||
errno);
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
spinunlock(&seg->seg_lock);
|
||||
seg->seg_offset = 0;
|
||||
seg->seg_size = size;
|
||||
|
||||
/* set up the map object */
|
||||
map = OBJ_NEW(mca_mpool_sm_mmap_t);
|
||||
strncpy(map->map_path, path, PATH_MAX);
|
||||
strncpy(map->map_path, file_name, PATH_MAX);
|
||||
map->map_seg = seg;
|
||||
map->map_addr = (unsigned char*)(seg+1);
|
||||
map->map_size = size - sizeof(mca_mpool_sm_segment_t);
|
||||
/* allign start of data segment */
|
||||
addr=(unsigned char *)seg+size_ctl_structure;
|
||||
tmp=(size_t)(addr+1+data_seg_alignment);
|
||||
addr=(unsigned char *)( (tmp/data_seg_alignment)*data_seg_alignment);
|
||||
/* is addr past end of file ? */
|
||||
if( (unsigned char*)seg+size < addr ){
|
||||
ompi_output(0, "mca_mpool_sm_mmap_init: memory region too small len %d \n",size);
|
||||
close(fd);
|
||||
munmap(seg,size);
|
||||
return NULL;
|
||||
}
|
||||
map->map_addr = addr;
|
||||
map->map_size = size - (addr-(unsigned char *)seg);
|
||||
close(fd);
|
||||
|
||||
/* initialize the segment - only the first process to open the file */
|
||||
if( !file_previously_opened ) {
|
||||
spinunlock(&seg->seg_lock);
|
||||
seg->seg_inited = false;
|
||||
seg->seg_offset = 0;
|
||||
seg->seg_size = size;
|
||||
}
|
||||
|
||||
/* enable access by other processes on this host */
|
||||
if(fchmod(fd, 0600) != 0) {
|
||||
ompi_output(0, "mca_mpool_sm_mmap_init: fchmod failed with errno=%d\n", errno);
|
||||
OBJ_RELEASE(map);
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
close(fd);
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
|
||||
struct mca_mpool_sm_segment_t {
|
||||
ompi_lock_data_t seg_lock;
|
||||
volatile bool seg_inited;
|
||||
size_t seg_offset;
|
||||
size_t seg_size;
|
||||
};
|
||||
@ -25,7 +26,27 @@ typedef struct mca_mpool_sm_mmap_t mca_mpool_sm_mmap_t;
|
||||
OBJ_CLASS_DECLARATION(mca_mpool_sm_mmap_t);
|
||||
|
||||
|
||||
mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size);
|
||||
/**
|
||||
* This routine is used to set up a shared memory file, backed
|
||||
* by a specified file. It is assumed that the file does not
|
||||
* exist before any of the current set of processes try and open
|
||||
* it.
|
||||
*
|
||||
* @param size - size of the file, in bytes (IN)
|
||||
*
|
||||
* @param file_name name of file to be opened. (IN)
|
||||
*
|
||||
* @param size_ctl_structure size of the control structure at
|
||||
* the head of the file. The control structure
|
||||
* is assumed to have mca_mpool_sm_segment_t
|
||||
* as its first segment (IN)
|
||||
*
|
||||
* @param data_set_alignment alignment of the data segment. this
|
||||
* follows the control structure (IN)
|
||||
*/
|
||||
|
||||
mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size, char *file_name,
|
||||
size_t size_ctl_structure, size_t data_seg_alignment);
|
||||
void* mca_mpool_sm_mmap_alloc(size_t* size);
|
||||
void mca_mpool_sm_mmap_free(void* addr);
|
||||
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user