basesmuma: clean up the setup code and ensure mapped files are unmapped
We were leaking file descriptors when coll/ml was in use. It turn out this was because basesmuma was failing to unmap files it had previously mapped. This commit cleans up the setup code to ensure that we only attempt to map the control files once per module and then ensures the files are unmapped when the module is released. cmr=v1.8.2:reviewer=manjugv This commit was SVN r31737.
Этот коммит содержится в:
Nathan Hjelm
родитель
93060d1cf2
Коммит
c13c21d476
@ -1154,6 +1154,11 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
bcol_basesmuma_smcm_file_t input, char *base_fname,
|
||||
bool map_all);
|
||||
|
||||
/* clean up the backing files associated with a basesmuma bcol module */
|
||||
int bcol_basesmuma_smcm_release_connections (mca_bcol_basesmuma_module_t *sm_bcol_module,
|
||||
mca_sbgp_base_module_t *sbgp_module, opal_list_t *peer_list,
|
||||
bcol_basesmuma_smcm_proc_item_t ***back_files);
|
||||
|
||||
/*
|
||||
* this function initializes the internal scratch buffers and control
|
||||
* structures that will be used by the module
|
||||
|
||||
@ -305,7 +305,7 @@ int bcol_basesmuma_bank_init_opti(struct mca_bcol_base_memory_block_desc_t *payl
|
||||
&(cs->sm_connections_list),
|
||||
&(sm_bcol->payload_backing_files_info),
|
||||
sm_bcol->super.sbgp_partner_module->group_comm,
|
||||
input_file,cs->payload_base_fname,
|
||||
input_file, cs->payload_base_fname,
|
||||
false);
|
||||
if( OMPI_SUCCESS != ret ) {
|
||||
goto exit_ERROR;
|
||||
|
||||
@ -97,7 +97,7 @@ static void
|
||||
mca_bcol_basesmuma_module_destruct(mca_bcol_basesmuma_module_t *sm_module)
|
||||
{
|
||||
/* local variables */
|
||||
int i;
|
||||
mca_sbgp_base_module_t *sbgp_module = sm_module->super.sbgp_partner_module;
|
||||
mca_bcol_basesmuma_component_t *cs = &mca_bcol_basesmuma_component;
|
||||
|
||||
/*
|
||||
@ -119,10 +119,9 @@ mca_bcol_basesmuma_module_destruct(mca_bcol_basesmuma_module_t *sm_module)
|
||||
/* Remove Lmsg Reduce Offsets Array */
|
||||
free_lmsg_reduce_offsets_array(sm_module);
|
||||
|
||||
|
||||
/* collective topology data */
|
||||
if( sm_module->fanout_read_tree) {
|
||||
for(i=0 ; i < sm_module->super.size_of_subgroup ; i++ ) {
|
||||
for (int i = 0 ; i < sm_module->super.size_of_subgroup ; i++ ) {
|
||||
if(0 < sm_module->fanout_read_tree[i].n_children ) {
|
||||
free(sm_module->fanout_read_tree[i].children_ranks);
|
||||
sm_module->fanout_read_tree[i].children_ranks=NULL;
|
||||
@ -137,7 +136,7 @@ mca_bcol_basesmuma_module_destruct(mca_bcol_basesmuma_module_t *sm_module)
|
||||
* size of subgroup) of array reduction_tree
|
||||
*/
|
||||
if( sm_module->reduction_tree) {
|
||||
for(i=0 ; i < sm_module->super.size_of_subgroup ; i++ ) {
|
||||
for (int i = 0 ; i < sm_module->super.size_of_subgroup ; i++ ) {
|
||||
if(0 < sm_module->reduction_tree[i].n_children ) {
|
||||
free(sm_module->reduction_tree[i].children_ranks);
|
||||
sm_module->reduction_tree[i].children_ranks=NULL;
|
||||
@ -198,7 +197,7 @@ mca_bcol_basesmuma_module_destruct(mca_bcol_basesmuma_module_t *sm_module)
|
||||
|
||||
#if 1
|
||||
if(sm_module->scatter_kary_tree) {
|
||||
for(i=0 ; i < sm_module->super.size_of_subgroup ; i++ ) {
|
||||
for (int i = 0 ; i < sm_module->super.size_of_subgroup ; i++ ) {
|
||||
if(0 < sm_module->scatter_kary_tree[i].n_children) {
|
||||
free(sm_module->scatter_kary_tree[i].children_ranks);
|
||||
sm_module->scatter_kary_tree[i].children_ranks=NULL;
|
||||
@ -217,20 +216,20 @@ mca_bcol_basesmuma_module_destruct(mca_bcol_basesmuma_module_t *sm_module)
|
||||
sm_module->ml_mem.num_banks,
|
||||
sm_module->ml_mem.num_buffers_per_bank);
|
||||
|
||||
for (i = 0; i < BCOL_NUM_OF_FUNCTIONS; i++){
|
||||
for (int i = 0; i < BCOL_NUM_OF_FUNCTIONS; i++){
|
||||
/* gvm FIX: Go through the list and destroy each item */
|
||||
/* Destroy the function table object for each bcol type list */
|
||||
OPAL_LIST_DESTRUCT((&sm_module->super.bcol_fns_table[i]));
|
||||
}
|
||||
|
||||
if (NULL != sm_module->payload_backing_files_info) {
|
||||
free(sm_module->payload_backing_files_info);
|
||||
sm_module->payload_backing_files_info = NULL;
|
||||
bcol_basesmuma_smcm_release_connections (sm_module, sbgp_module, &cs->sm_connections_list,
|
||||
&sm_module->payload_backing_files_info);
|
||||
}
|
||||
|
||||
if (NULL != sm_module->ctl_backing_files_info) {
|
||||
free(sm_module->ctl_backing_files_info);
|
||||
sm_module->ctl_backing_files_info = NULL;
|
||||
bcol_basesmuma_smcm_release_connections (sm_module, sbgp_module, &cs->sm_connections_list,
|
||||
&sm_module->ctl_backing_files_info);
|
||||
}
|
||||
|
||||
if (NULL != sm_module->ml_mem.bank_release_counter) {
|
||||
|
||||
@ -327,16 +327,67 @@ exit_ERROR:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int base_bcol_basesmuma_setup_ctl (mca_bcol_basesmuma_module_t *sm_bcol_module,
|
||||
mca_bcol_basesmuma_component_t *cs)
|
||||
{
|
||||
const int my_index = sm_bcol_module->super.sbgp_partner_module->my_index;;
|
||||
bcol_basesmuma_smcm_file_t input_file;
|
||||
int ret;
|
||||
|
||||
/* exchange remote addressing information if it has not already been done */
|
||||
if (NULL == sm_bcol_module->ctl_backing_files_info) {
|
||||
input_file.file_name=cs->sm_ctl_structs->map_path;
|
||||
input_file.size=cs->sm_ctl_structs->map_size;
|
||||
input_file.size_ctl_structure=0;
|
||||
input_file.data_seg_alignment=BASESMUMA_CACHE_LINE_SIZE;
|
||||
input_file.mpool_size=cs->sm_ctl_structs->map_size;
|
||||
ret = bcol_basesmuma_smcm_allgather_connection(sm_bcol_module,
|
||||
sm_bcol_module->super.sbgp_partner_module,
|
||||
&(cs->sm_connections_list),
|
||||
&(sm_bcol_module->ctl_backing_files_info),
|
||||
sm_bcol_module->super.sbgp_partner_module->group_comm,
|
||||
input_file, cs->clt_base_fname,
|
||||
false);
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
/* fill in the pointer to other ranks scartch shared memory */
|
||||
if (NULL == sm_bcol_module->shared_memory_scratch_space) {
|
||||
sm_bcol_module->shared_memory_scratch_space =
|
||||
calloc (sm_bcol_module->super.sbgp_partner_module->group_size, sizeof (void *));
|
||||
if (!sm_bcol_module->shared_memory_scratch_space) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "Cannot allocate memory for shared_memory_scratch_space.");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
for (int i = 0 ; i < sm_bcol_module->super.sbgp_partner_module->group_size ; ++i) {
|
||||
if (i == my_index) {
|
||||
/* local file data is not cached in this list */
|
||||
continue;
|
||||
}
|
||||
|
||||
sm_bcol_module->shared_memory_scratch_space[i] =
|
||||
(void *)((intptr_t) sm_bcol_module->ctl_backing_files_info[i]->sm_mmap +
|
||||
cs->scratch_offset_from_base_ctl_file);
|
||||
}
|
||||
|
||||
sm_bcol_module->shared_memory_scratch_space[my_index] =
|
||||
(void *)((intptr_t) cs->sm_ctl_structs->map_addr + cs->scratch_offset_from_base_ctl_file);
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int base_bcol_basesmuma_setup_ctl_struct(
|
||||
mca_bcol_basesmuma_module_t *sm_bcol_module,
|
||||
mca_bcol_basesmuma_component_t *cs,
|
||||
sm_buffer_mgmt *ctl_mgmt)
|
||||
{
|
||||
int ret=OMPI_SUCCESS,i,n_ctl,n_levels;
|
||||
int n_ctl, n_levels;
|
||||
int n_ctl_structs;
|
||||
int cnt;
|
||||
size_t malloc_size;
|
||||
bcol_basesmuma_smcm_file_t input_file;
|
||||
|
||||
/*
|
||||
* set my no user-data conrol structures
|
||||
@ -371,98 +422,46 @@ int base_bcol_basesmuma_setup_ctl_struct(
|
||||
ctl_mgmt->num_mem_banks ) *
|
||||
ctl_mgmt->size_of_group *
|
||||
sizeof(void *);
|
||||
ctl_mgmt->ctl_buffs= malloc(malloc_size);
|
||||
if( !ctl_mgmt->ctl_buffs ) {
|
||||
ret=OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit_ERROR;
|
||||
ctl_mgmt->ctl_buffs = malloc(malloc_size);
|
||||
if (!ctl_mgmt->ctl_buffs) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* exchange remote addressing information */
|
||||
input_file.file_name=cs->sm_ctl_structs->map_path;
|
||||
input_file.size=cs->sm_ctl_structs->map_size;
|
||||
input_file.size_ctl_structure=0;
|
||||
input_file.data_seg_alignment=BASESMUMA_CACHE_LINE_SIZE;
|
||||
input_file.mpool_size=cs->sm_ctl_structs->map_size;
|
||||
ret=bcol_basesmuma_smcm_allgather_connection(
|
||||
sm_bcol_module,
|
||||
sm_bcol_module->super.sbgp_partner_module,
|
||||
&(cs->sm_connections_list),
|
||||
&(sm_bcol_module->ctl_backing_files_info),
|
||||
sm_bcol_module->super.sbgp_partner_module->group_comm,
|
||||
input_file, cs->clt_base_fname,
|
||||
false);
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
goto exit_ERROR;
|
||||
}
|
||||
|
||||
/* fill in the pointer to other ranks scartch shared memory */
|
||||
sm_bcol_module->shared_memory_scratch_space=
|
||||
malloc(sizeof(void *)*
|
||||
sm_bcol_module->super.sbgp_partner_module->group_size);
|
||||
if( !sm_bcol_module->shared_memory_scratch_space ) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "Cannot allocate memory for shared_memory_scratch_space.\n");
|
||||
ret = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit_ERROR;
|
||||
}
|
||||
for(i=0 ; i < sm_bcol_module->super.sbgp_partner_module->group_size ; i++ ) {
|
||||
if(i == sm_bcol_module->super.sbgp_partner_module->my_index) {
|
||||
/* local file data is not cached in thi slist */
|
||||
continue;
|
||||
}
|
||||
sm_bcol_module->shared_memory_scratch_space[i]=(void *)(
|
||||
(char *)(sm_bcol_module->ctl_backing_files_info[i]->sm_mmap)+
|
||||
cs->scratch_offset_from_base_ctl_file);
|
||||
}
|
||||
i=sm_bcol_module->super.sbgp_partner_module->my_index;
|
||||
sm_bcol_module->shared_memory_scratch_space[i]=(void *)(
|
||||
(char *)(cs->sm_ctl_structs->map_addr)+cs->scratch_offset_from_base_ctl_file);
|
||||
|
||||
/*
|
||||
* setup the no-data buffer managment data
|
||||
*/
|
||||
n_ctl=ctl_mgmt->num_mem_banks;
|
||||
ctl_mgmt->ctl_buffs_mgmt=(mem_bank_management_t *)
|
||||
malloc(sizeof(mem_bank_management_t)*n_ctl);
|
||||
if( !ctl_mgmt->ctl_buffs_mgmt ) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "Cannot allocate memory for ctl_buffs_mgmt. ret = %d\n",ret);
|
||||
ret = OMPI_ERROR;
|
||||
goto exit_ERROR;
|
||||
n_ctl = ctl_mgmt->num_mem_banks;
|
||||
ctl_mgmt->ctl_buffs_mgmt = (mem_bank_management_t *) calloc (n_ctl, sizeof (mem_bank_management_t));
|
||||
if (!ctl_mgmt->ctl_buffs_mgmt) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "Cannot allocate memory for ctl_buffs_mgmt");
|
||||
free (ctl_mgmt->ctl_buffs);
|
||||
ctl_mgmt->ctl_buffs = NULL;
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* initialize each individual element */
|
||||
cnt=cs->basesmuma_num_regions_per_bank*cs->basesmuma_num_mem_banks;
|
||||
for( i=0 ; i < n_ctl ; i++ ) {
|
||||
for (int i = 0 ; i < n_ctl ; ++i) {
|
||||
opal_list_item_t *item;
|
||||
opal_mutex_t *mutex_ptr;
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].bank_gen_counter= 0;
|
||||
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].available_buffers=
|
||||
ctl_mgmt->num_buffs_per_mem_bank;
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].number_of_buffers=
|
||||
ctl_mgmt->num_buffs_per_mem_bank;
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].n_buffs_freed= 0;
|
||||
mutex_ptr= &(ctl_mgmt->ctl_buffs_mgmt[i].mutex);
|
||||
mutex_ptr = &(ctl_mgmt->ctl_buffs_mgmt[i].mutex);
|
||||
OBJ_CONSTRUCT(mutex_ptr, opal_mutex_t);
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].index_shared_mem_ctl_structs=i;
|
||||
|
||||
item=(opal_list_item_t *)&(ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc);
|
||||
OBJ_CONSTRUCT(item,opal_list_item_t);
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc.sm_module=
|
||||
item = (opal_list_item_t *)&(ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc);
|
||||
OBJ_CONSTRUCT(item, opal_list_item_t);
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc.sm_module =
|
||||
sm_bcol_module;
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc.pool_index= i;
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc.pool_index = i;
|
||||
/* get the sm_buffer_mgmt pointer for the control structures */
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc.coll_buff=ctl_mgmt;
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc.ml_memory_block_descriptor=
|
||||
NULL;
|
||||
|
||||
cnt++;
|
||||
ctl_mgmt->ctl_buffs_mgmt[i].nb_barrier_desc.coll_buff = ctl_mgmt;
|
||||
}
|
||||
|
||||
|
||||
return ret;
|
||||
|
||||
exit_ERROR:
|
||||
|
||||
return ret;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -514,10 +513,9 @@ int base_bcol_basesmuma_setup_library_buffers(
|
||||
data_ptr=cs->sm_ctl_structs->data_addr;
|
||||
|
||||
for( i=0 ; i < max_elements ; i++ ) {
|
||||
list_data_t *item=OBJ_NEW(list_data_t);
|
||||
list_data_t *item = OBJ_NEW(list_data_t);
|
||||
if( !item ) {
|
||||
ret=OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit_ERROR;
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
item->data=(void *)data_ptr;
|
||||
opal_list_append(&(cs->ctl_structures),(opal_list_item_t *)item);
|
||||
@ -538,37 +536,30 @@ int base_bcol_basesmuma_setup_library_buffers(
|
||||
/* intialize no_userdata_ctl */
|
||||
sm_bcol_module->no_userdata_ctl=(list_data_t *)
|
||||
opal_list_remove_last(&(cs->ctl_structures));
|
||||
if( !sm_bcol_module->no_userdata_ctl) {
|
||||
ret=OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit_ERROR;
|
||||
}
|
||||
ret=base_bcol_basesmuma_setup_ctl_struct(
|
||||
sm_bcol_module, cs, &(sm_bcol_module->colls_no_user_data));
|
||||
if( OMPI_SUCCESS != ret ) {
|
||||
goto exit_ERROR;
|
||||
if (!sm_bcol_module->no_userdata_ctl) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* intialize userdata_ctl */
|
||||
sm_bcol_module->userdata_ctl=(list_data_t *)
|
||||
sm_bcol_module->userdata_ctl = (list_data_t *)
|
||||
opal_list_remove_last(&(cs->ctl_structures));
|
||||
if( !sm_bcol_module->userdata_ctl) {
|
||||
ret=OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit_ERROR;
|
||||
if (!sm_bcol_module->userdata_ctl) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
/* FIXME base_bcol_basesmuma_setup_ctl_struct will allocate
|
||||
* sm_bcol_module->shared_memory_scratch_space which was previously
|
||||
* allocated by the previous call to base_bcol_basesmuma_setup_ctl_struct
|
||||
* so let's free it in order to avoid a memory leak */
|
||||
if (sm_bcol_module->shared_memory_scratch_space) {
|
||||
free(sm_bcol_module->shared_memory_scratch_space);
|
||||
sm_bcol_module->shared_memory_scratch_space = NULL;
|
||||
ret = base_bcol_basesmuma_setup_ctl (sm_bcol_module, cs);
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret=base_bcol_basesmuma_setup_ctl_struct(
|
||||
sm_bcol_module, cs, &(sm_bcol_module->colls_with_user_data));
|
||||
ret = base_bcol_basesmuma_setup_ctl_struct (sm_bcol_module, cs, &(sm_bcol_module->colls_no_user_data));
|
||||
if( OMPI_SUCCESS != ret ) {
|
||||
goto exit_ERROR;
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = base_bcol_basesmuma_setup_ctl_struct (sm_bcol_module, cs, &(sm_bcol_module->colls_with_user_data));
|
||||
if( OMPI_SUCCESS != ret ) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* used for blocking recursive doubling barrier */
|
||||
@ -578,23 +569,19 @@ int base_bcol_basesmuma_setup_library_buffers(
|
||||
* of the shared memory file, and fill in the table with the
|
||||
* address of all the control structues.
|
||||
*/
|
||||
ret= base_bcol_basesmuma_exchange_ctl_params(sm_bcol_module, cs,
|
||||
ret = base_bcol_basesmuma_exchange_ctl_params(sm_bcol_module, cs,
|
||||
&(sm_bcol_module->colls_no_user_data),sm_bcol_module->no_userdata_ctl);
|
||||
if( OMPI_SUCCESS != ret ) {
|
||||
goto exit_ERROR;
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret= base_bcol_basesmuma_exchange_ctl_params(sm_bcol_module, cs,
|
||||
ret = base_bcol_basesmuma_exchange_ctl_params(sm_bcol_module, cs,
|
||||
&(sm_bcol_module->colls_with_user_data),sm_bcol_module->userdata_ctl);
|
||||
if( OMPI_SUCCESS != ret ) {
|
||||
goto exit_ERROR;
|
||||
return ret;
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
||||
exit_ERROR:
|
||||
|
||||
return ret;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(list_data_t,
|
||||
|
||||
@ -51,13 +51,28 @@ struct file_info_t {
|
||||
};
|
||||
|
||||
/* need to allocate space for the peer */
|
||||
static void bcol_basesmuma_smcm_proc_item_t_construct
|
||||
(bcol_basesmuma_smcm_proc_item_t * item) {
|
||||
static void bcol_basesmuma_smcm_proc_item_t_construct (bcol_basesmuma_smcm_proc_item_t * item)
|
||||
{
|
||||
memset ((char *) item + sizeof (item->item), 0, sizeof (*item) - sizeof (item->item));
|
||||
}
|
||||
|
||||
/* need to free the space for the peer */
|
||||
static void bcol_basesmuma_smcm_proc_item_t_destruct
|
||||
(bcol_basesmuma_smcm_proc_item_t * item) {
|
||||
static void bcol_basesmuma_smcm_proc_item_t_destruct (bcol_basesmuma_smcm_proc_item_t * item)
|
||||
{
|
||||
if (item->sm_mmap) {
|
||||
bcol_basesmuma_smcm_mmap_t *map = item->sm_mmap;
|
||||
|
||||
if (map->map_seg) {
|
||||
(void) munmap ((void *) map->map_seg, map->map_size);
|
||||
}
|
||||
|
||||
free (map);
|
||||
}
|
||||
|
||||
if (item->sm_file.file_name) {
|
||||
free (item->sm_file.file_name);
|
||||
item->sm_file.file_name = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(bcol_basesmuma_smcm_proc_item_t,
|
||||
@ -103,7 +118,7 @@ bcol_basesmuma_smcm_mmap_t* bcol_basesmuma_smcm_create_mmap(int fd, size_t size,
|
||||
/* is addr past end of file ? */
|
||||
if((unsigned char*)seg + size < addr) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "bcol_basesmuma_smcm_mmap_init: "
|
||||
"memory region too small len %lu addr %p\n",
|
||||
"memory region too small len %lu addr %p",
|
||||
(unsigned long)size, addr);
|
||||
return NULL;
|
||||
}
|
||||
@ -176,7 +191,7 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
signal(SIGABRT, SIG_DFL);
|
||||
/* sanity check */
|
||||
if (strlen(input.file_name) > SM_BACKING_FILE_NAME_MAX_LEN-1) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "backing file name too long: %s len :: %d\n",
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "backing file name too long: %s len :: %d",
|
||||
input.file_name, (int) strlen(input.file_name));
|
||||
return OMPI_ERR_BAD_PARAM;
|
||||
}
|
||||
@ -226,7 +241,7 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
sm_bcol_module->super.sbgp_partner_module->group_list,
|
||||
sm_bcol_module->super.sbgp_partner_module->group_comm);
|
||||
if( OMPI_SUCCESS != rc ) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "failed in comm_allgather_pml. Error code: %d\n", rc);
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "failed in comm_allgather_pml. Error code: %d", rc);
|
||||
goto Error;
|
||||
}
|
||||
|
||||
@ -253,6 +268,7 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
if (proc_temp->proc_name.vpid == item_ptr->peer.vpid &&
|
||||
proc_temp->proc_name.jobid == item_ptr->peer.jobid &&
|
||||
0 == strcmp (item_ptr->sm_file.file_name, rem_file->file_name)) {
|
||||
++item_ptr->refcnt;
|
||||
/* record file data */
|
||||
backing_files[i] = item_ptr;
|
||||
break;
|
||||
@ -283,6 +299,7 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
temp->sm_file.mpool_size = (size_t) rem_file->file_size;
|
||||
temp->sm_file.size_ctl_structure = (size_t) rem_file->size_ctl_structure;
|
||||
temp->sm_file.data_seg_alignment = (size_t) rem_file->data_seg_alignment;
|
||||
temp->refcnt = 1;
|
||||
|
||||
/* Phase Five:
|
||||
If map_all == true, then we map every peer's file
|
||||
@ -293,7 +310,8 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
*/
|
||||
fd = open(temp->sm_file.file_name, O_RDWR, 0600);
|
||||
if (0 > fd) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "SMCM Allgather failed to open sm backing file %s. errno = %d\n", temp->sm_file.file_name, errno);
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "SMCM Allgather failed to open sm backing file %s. errno = %d",
|
||||
temp->sm_file.file_name, errno);
|
||||
rc = OMPI_ERROR;
|
||||
goto Error;
|
||||
}
|
||||
@ -304,7 +322,7 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
temp->sm_file.size_ctl_structure,
|
||||
getpagesize());
|
||||
if (NULL == temp->sm_mmap) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "mmapping failed to map remote peer's file\n");
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "mmapping failed to map remote peer's file");
|
||||
OBJ_RELEASE(temp);
|
||||
rc = OMPI_ERROR;
|
||||
goto Error;
|
||||
@ -338,6 +356,25 @@ int bcol_basesmuma_smcm_allgather_connection(
|
||||
return rc;
|
||||
}
|
||||
|
||||
int bcol_basesmuma_smcm_release_connections (mca_bcol_basesmuma_module_t *sm_bcol_module,
|
||||
mca_sbgp_base_module_t *sbgp_module, opal_list_t *peer_list,
|
||||
bcol_basesmuma_smcm_proc_item_t ***back_files)
|
||||
{
|
||||
bcol_basesmuma_smcm_proc_item_t **smcm_procs = *back_files;
|
||||
|
||||
for (int i = 0 ; i < sbgp_module->group_size ; ++i) {
|
||||
if (smcm_procs[i] && 0 == --smcm_procs[i]->refcnt) {
|
||||
opal_list_remove_item (peer_list, (opal_list_item_t *) smcm_procs[i]);
|
||||
OBJ_RELEASE(smcm_procs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
free (smcm_procs);
|
||||
*back_files = NULL;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
|
||||
OBJ_CLASS_INSTANCE(
|
||||
@ -371,10 +408,10 @@ bcol_basesmuma_smcm_mmap_t *bcol_basesmuma_smcm_mem_reg(void *in_ptr,
|
||||
|
||||
fd = open(file_name, O_CREAT|O_RDWR,0600);
|
||||
if (fd < 0) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "basesmuma shared memory allocation open failed with errno: %d\n",
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "basesmuma shared memory allocation open failed with errno: %d",
|
||||
errno);
|
||||
} else if (0 != ftruncate(fd,length)) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "basesmuma shared memory allocation ftruncate failed with errno: %d\n",
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "basesmuma shared memory allocation ftruncate failed with errno: %d",
|
||||
errno);
|
||||
} else {
|
||||
|
||||
@ -430,7 +467,7 @@ bcol_basesmuma_smcm_mmap_t * bcol_basesmuma_smcm_reg_mmap(void *in_ptr,
|
||||
|
||||
/* is addr past the end of the file? */
|
||||
if ((unsigned char *) seg+length < myaddr) {
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "mca_bcol_basesmuma_sm_alloc_mmap: memory region too small len %lu add %p\n",
|
||||
opal_output (ompi_bcol_base_framework.framework_output, "mca_bcol_basesmuma_sm_alloc_mmap: memory region too small len %lu add %p",
|
||||
(unsigned long) length, myaddr);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
*
|
||||
* Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved.
|
||||
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
|
||||
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -72,6 +75,7 @@ typedef struct bcol_basesmuma_smcm_file_t bcol_basesmuma_smcm_file_t;
|
||||
|
||||
struct bcol_basesmuma_smcm_proc_item_t {
|
||||
opal_list_item_t item; /* can put me on a free list */
|
||||
int refcnt;
|
||||
ompi_process_name_t peer;
|
||||
bcol_basesmuma_smcm_file_t sm_file;
|
||||
bcol_basesmuma_smcm_mmap_t *sm_mmap; /* Pointer to peer's sm file */
|
||||
|
||||
Загрузка…
Ссылка в новой задаче
Block a user