From 0ab1ab374d6ebdeaac63c03c0046cd92c814b38b Mon Sep 17 00:00:00 2001 From: Rich Graham Date: Thu, 21 Oct 2004 22:40:25 +0000 Subject: [PATCH] starting to debug the shared memory implmentation. Changing process count references to int's rather than size_t's. use asprintf to store the name of the shared memory backing file. This commit was SVN r3275. --- src/mca/mpool/sm/mpool_sm_component.c | 22 ++++---- src/mca/ptl/sm/src/ptl_sm.c | 72 ++++++++++++++++++++++----- src/mca/ptl/sm/src/ptl_sm.h | 7 +-- src/mca/ptl/sm/src/ptl_sm_component.c | 40 ++------------- 4 files changed, 79 insertions(+), 62 deletions(-) diff --git a/src/mca/mpool/sm/mpool_sm_component.c b/src/mca/mpool/sm/mpool_sm_component.c index baa70da90b..853d9bc052 100644 --- a/src/mca/mpool/sm/mpool_sm_component.c +++ b/src/mca/mpool/sm/mpool_sm_component.c @@ -87,7 +87,9 @@ static int mca_mpool_sm_open(void) static mca_mpool_base_module_t* mca_mpool_sm_init(bool *allow_multi_user_threads) { - char file_name[PATH_MAX]; + char *file_name; + size_t len; + mca_allocator_base_component_t* allocator_component = mca_allocator_component_lookup( mca_mpool_sm_component.sm_allocator_name); @@ -107,26 +109,24 @@ mca_mpool_sm_init(bool *allow_multi_user_threads) } /* create initial shared memory mapping */ - memset(&(file_name[0]),0,PATH_MAX); - if( (strlen(ompi_process_info.job_session_dir) + - strlen(ompi_system_info.nodename)+ - /* length of fixe name part */ - 17 ) >= PATH_MAX ) { - ompi_output(0, "mca_mpool_sm_init: name of backing file too long \n"); - return NULL; - } - sprintf(&(file_name[0]),"%s/shared_mem_pool.%s", + len=asprintf(&file_name,"%s/shared_mem_pool.%s", ompi_process_info.job_session_dir, ompi_system_info.nodename); + if ( 0 > len ) { + return NULL; + } + if(NULL == (mca_common_sm_mmap = mca_common_sm_mmap_init(mca_mpool_sm_component.sm_size, - &(file_name[0]),sizeof(mca_common_sm_mmap_t), 8 ) + file_name,sizeof(mca_common_sm_mmap_t), 8 ) )) { + free(file_name); ompi_output(0, "mca_mpool_sm_init: unable to create shared memory mapping"); return NULL; } + free(file_name); /* setup function pointers */ diff --git a/src/mca/ptl/sm/src/ptl_sm.c b/src/mca/ptl/sm/src/ptl_sm.c index 2e7f5b36c0..eae9ed16b9 100644 --- a/src/mca/ptl/sm/src/ptl_sm.c +++ b/src/mca/ptl/sm/src/ptl_sm.c @@ -70,11 +70,10 @@ int mca_ptl_sm_add_procs( ompi_bitmap_t* reachability) { int return_code=OMPI_SUCCESS; - size_t i,j,proc,size,len,my_len,n_local_procs,n_to_allocate; + size_t i,j,proc,size,len,my_len,n_local_procs,n_to_allocate,length; mca_ptl_sm_exchange_t **sm_proc_info; ompi_proc_t* my_proc; /* pointer to caller's proc structure */ mca_ptl_sm_t *ptl_sm; - bool threads; ompi_fifo_t *fifo_addr; /* initializion */ @@ -197,6 +196,21 @@ int mca_ptl_sm_add_procs( mca_ptl_sm_component.sm_ctl_header=(mca_ptl_sm_module_resource_t *) mca_ptl_sm_component.mmap_file->map_seg; + + /* see if need to allocate space for extra procs */ + if( 0 > mca_ptl_sm_component.sm_max_procs ) { + /* no limit */ + if( 0 <= mca_ptl_sm_component.sm_extra_procs ) { + /* limit */ + mca_ptl_sm_component.sm_max_procs=n_local_procs+ + mca_ptl_sm_component.sm_extra_procs; + } else { + /* no limit */ + mca_ptl_sm_component.sm_max_procs=2*n_local_procs; + } + } + n_to_allocate=mca_ptl_sm_component.sm_max_procs; + /* Allocate a fixed size pointer array for the 2-D Shared memory queues. * Excess slots will be allocated for future growth. One could * make this array growable, but then one would need to uses mutexes @@ -215,17 +229,6 @@ int mca_ptl_sm_add_procs( goto CLEANUP; } - /* see if need to allocate space for extra procs */ - if( 0 > mca_ptl_sm_component.sm_max_procs ) { - if( 0 < mca_ptl_sm_component.sm_extra_procs ) { - mca_ptl_sm_component.sm_max_procs= - n_local_procs+mca_ptl_sm_component.sm_extra_procs; - } else { - mca_ptl_sm_component.sm_max_procs=n_local_procs; - } - } - n_to_allocate=mca_ptl_sm_component.sm_max_procs; - /* allocate array of ompi_fifo_t* elements - * offset relative to base segement is stored, so that * this can be used by other procs */ @@ -305,6 +308,49 @@ int mca_ptl_sm_add_procs( free(sm_proc_info); } + /* initialize some of the free-lists */ + if( !mca_ptl_sm.ptl_inited ) { + /* some initialization happens only the first time this routine + * is called, i.e. when ptl_inited is false */ + + /* initialize fragment descriptor free list */ + + /* + * first fragment + */ + + /* allocation will be for the fragment descriptor, payload buffer, + * and padding to ensure proper alignment can be acheived */ + length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+ + mca_ptl_sm_component.first_fragment_size; + + ompi_free_list_init(&mca_ptl_sm.sm_first_frags, length, + OBJ_CLASS(mca_ptl_sm_frag_t), + mca_ptl_sm_component.sm_first_frag_free_list_num, + mca_ptl_sm_component.sm_first_frag_free_list_max, + mca_ptl_sm_component.sm_first_frag_free_list_inc, + mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */ + + /* + * second and beyond fragments + */ + + /* allocation will be for the fragment descriptor, payload buffer, + * and padding to ensure proper alignment can be acheived */ + length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+ + mca_ptl_sm_component.max_fragment_size; + + ompi_free_list_init(&mca_ptl_sm.sm_second_frags, length, + OBJ_CLASS(mca_ptl_sm_second_frag_t), + mca_ptl_sm_component.sm_second_frag_free_list_num, + mca_ptl_sm_component.sm_second_frag_free_list_max, + mca_ptl_sm_component.sm_second_frag_free_list_inc, + mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */ + + /* set flag indicating ptl has been inited */ + mca_ptl_sm.ptl_inited=true; + } + /* update the local smp process count */ mca_ptl_sm_component.num_smp_procs+=n_local_procs; diff --git a/src/mca/ptl/sm/src/ptl_sm.h b/src/mca/ptl/sm/src/ptl_sm.h index 25a458c147..a4f1bd26b4 100644 --- a/src/mca/ptl/sm/src/ptl_sm.h +++ b/src/mca/ptl/sm/src/ptl_sm.h @@ -46,8 +46,8 @@ struct mca_ptl_sm_component_t { int sm_second_frag_free_list_num; /**< initial size of free lists */ int sm_second_frag_free_list_max; /**< maximum size of free lists */ int sm_second_frag_free_list_inc; /**< number of elements to alloc when growing free lists */ - size_t sm_max_procs; /**< upper limit on the number of processes using the shared memory pool */ - size_t sm_extra_procs; /**< number of extra procs to allow */ + int sm_max_procs; /**< upper limit on the number of processes using the shared memory pool */ + int sm_extra_procs; /**< number of extra procs to allow */ char* sm_mpool_name; /**< name of shared memory pool module */ mca_mpool_base_module_t* sm_mpool; /**< shared memory pool */ void* sm_mpool_base; /**< base address of shared memory pool */ @@ -134,8 +134,9 @@ struct mca_ptl_sm_t { fragments that are awaiting resources */ ompi_mutex_t sm_pending_ack_lock; - ompi_list_t sm_pending_ack; /* list of fragmnent that need to be + ompi_list_t sm_pending_ack; /**< list of fragmnent that need to be acked */ + bool ptl_inited; /**< flag indicating if ptl has been inited */ }; typedef struct mca_ptl_sm_t mca_ptl_sm_t; diff --git a/src/mca/ptl/sm/src/ptl_sm_component.c b/src/mca/ptl/sm/src/ptl_sm_component.c index 7c36c66161..2699e2f938 100644 --- a/src/mca/ptl/sm/src/ptl_sm_component.c +++ b/src/mca/ptl/sm/src/ptl_sm_component.c @@ -118,6 +118,8 @@ int mca_ptl_sm_component_open(void) mca_ptl_sm_param_register_int("second_frag_free_list_inc", 256); mca_ptl_sm_component.sm_max_procs = mca_ptl_sm_param_register_int("max_procs", -1); + mca_ptl_sm_component.sm_extra_procs = + mca_ptl_sm_param_register_int("sm_extra_procs", -1); mca_ptl_sm_component.sm_mpool_name = mca_ptl_sm_param_register_string("mpool", "sm"); mca_ptl_sm_component.first_fragment_size = @@ -186,7 +188,6 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init( bool *have_hidden_threads) { mca_ptl_base_module_t **ptls = NULL; - size_t length; *num_ptls = 0; *allow_multi_user_threads = true; @@ -198,40 +199,6 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init( mca_ptl_sm_component.sm_mpool_base = mca_ptl_sm_component.sm_mpool->mpool_base(); - /* initialize fragment descriptor free list */ - - /* - * first fragment - */ - - /* allocation will be for the fragment descriptor, payload buffer, - * and padding to ensure proper alignment can be acheived */ - length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+ - mca_ptl_sm_component.first_fragment_size; - - ompi_free_list_init(&mca_ptl_sm.sm_first_frags, length, - OBJ_CLASS(mca_ptl_sm_frag_t), - mca_ptl_sm_component.sm_first_frag_free_list_num, - mca_ptl_sm_component.sm_first_frag_free_list_max, - mca_ptl_sm_component.sm_first_frag_free_list_inc, - mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */ - - /* - * second and beyond fragments - */ - - /* allocation will be for the fragment descriptor, payload buffer, - * and padding to ensure proper alignment can be acheived */ - length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+ - mca_ptl_sm_component.max_fragment_size; - - ompi_free_list_init(&mca_ptl_sm.sm_second_frags, length, - OBJ_CLASS(mca_ptl_sm_second_frag_t), - mca_ptl_sm_component.sm_second_frag_free_list_num, - mca_ptl_sm_component.sm_second_frag_free_list_max, - mca_ptl_sm_component.sm_second_frag_free_list_inc, - mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */ - /* publish shared memory parameters with the MCA framework */ if(mca_ptl_sm_component_exchange() != OMPI_SUCCESS) return 0; @@ -263,6 +230,9 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init( mca_ptl_sm_component.num_smp_procs=0; mca_ptl_sm_component.my_smp_rank=-1; + /* set flag indicating ptl not inited */ + mca_ptl_sm.ptl_inited=false; + return ptls; }