starting to debug the shared memory implmentation. Changing

process count references to int's rather than size_t's.
use asprintf to store the name of the shared memory backing file.

This commit was SVN r3275.

src/mca/mpool/sm/mpool_sm_component.c

@@ -87,7 +87,9 @@ static int mca_mpool_sm_open(void)
 static mca_mpool_base_module_t* 
 mca_mpool_sm_init(bool *allow_multi_user_threads)
 {
-    char file_name[PATH_MAX];
+    char *file_name;
+    size_t len;
+
     mca_allocator_base_component_t* allocator_component = mca_allocator_component_lookup(
         mca_mpool_sm_component.sm_allocator_name);
 
@@ -107,26 +109,24 @@ mca_mpool_sm_init(bool *allow_multi_user_threads)
     }
     
     /* create initial shared memory mapping */
-    memset(&(file_name[0]),0,PATH_MAX);
-    if( (strlen(ompi_process_info.job_session_dir) +
-            strlen(ompi_system_info.nodename)+
-            /* length of fixe name part */
-            17 ) >= PATH_MAX ) {
-            ompi_output(0, "mca_mpool_sm_init: name of backing file too long \n");
-            return NULL;
-    }
-    sprintf(&(file_name[0]),"%s/shared_mem_pool.%s",
+    len=asprintf(&file_name,"%s/shared_mem_pool.%s",
             ompi_process_info.job_session_dir,
             ompi_system_info.nodename);
+    if ( 0 > len ) {
+        return NULL;
+    }
+
     if(NULL == 
             (mca_common_sm_mmap = 
              mca_common_sm_mmap_init(mca_mpool_sm_component.sm_size,
-                 &(file_name[0]),sizeof(mca_common_sm_mmap_t), 8 )
+                 file_name,sizeof(mca_common_sm_mmap_t), 8 )
              )) 
     {
+        free(file_name);
         ompi_output(0, "mca_mpool_sm_init: unable to create shared memory mapping");
         return NULL;
     }
+    free(file_name);
 
     /* setup function pointers */
 

src/mca/ptl/sm/src/ptl_sm.c

@@ -70,11 +70,10 @@ int mca_ptl_sm_add_procs(
     ompi_bitmap_t* reachability)
 {
     int return_code=OMPI_SUCCESS;
-    size_t i,j,proc,size,len,my_len,n_local_procs,n_to_allocate;
+    size_t i,j,proc,size,len,my_len,n_local_procs,n_to_allocate,length;
     mca_ptl_sm_exchange_t **sm_proc_info;
     ompi_proc_t* my_proc; /* pointer to caller's proc structure */
     mca_ptl_sm_t *ptl_sm;
-    bool threads;
     ompi_fifo_t *fifo_addr;
 
     /* initializion */
@@ -197,6 +196,21 @@ int mca_ptl_sm_add_procs(
         mca_ptl_sm_component.sm_ctl_header=(mca_ptl_sm_module_resource_t *)
             mca_ptl_sm_component.mmap_file->map_seg;
 
+            
+        /* see if need to allocate space for extra procs */
+        if(  0 > mca_ptl_sm_component.sm_max_procs ) {
+            /* no limit */
+            if( 0 <= mca_ptl_sm_component.sm_extra_procs ) {
+                /* limit */
+                mca_ptl_sm_component.sm_max_procs=n_local_procs+
+                    mca_ptl_sm_component.sm_extra_procs;
+            } else {
+                /* no limit */
+                mca_ptl_sm_component.sm_max_procs=2*n_local_procs;
+            }
+        }
+        n_to_allocate=mca_ptl_sm_component.sm_max_procs;
+
         /* Allocate a fixed size pointer array for the 2-D Shared memory queues.
          * Excess slots will be allocated for future growth.  One could
          * make this array growable, but then one would need to uses mutexes
@@ -215,17 +229,6 @@ int mca_ptl_sm_add_procs(
                 goto CLEANUP;
             }
 
-            /* see if need to allocate space for extra procs */
-            if(  0 > mca_ptl_sm_component.sm_max_procs ) {
-                if( 0 < mca_ptl_sm_component.sm_extra_procs ) {
-                    mca_ptl_sm_component.sm_max_procs=
-                        n_local_procs+mca_ptl_sm_component.sm_extra_procs;
-                } else {
-                    mca_ptl_sm_component.sm_max_procs=n_local_procs;
-                }
-            }
-            n_to_allocate=mca_ptl_sm_component.sm_max_procs;
-
             /* allocate array of ompi_fifo_t* elements -
              * offset relative to base segement is stored, so that
              * this can be used by other procs */
@@ -305,6 +308,49 @@ int mca_ptl_sm_add_procs(
         free(sm_proc_info);
     }
 
+    /* initialize some of the free-lists */
+    if( !mca_ptl_sm.ptl_inited ) {
+        /* some initialization happens only the first time this routine
+         * is called, i.e. when ptl_inited is false */
+
+        /* initialize fragment descriptor free list */
+
+        /* 
+         * first fragment 
+         */
+
+        /* allocation will be for the fragment descriptor, payload buffer,
+         * and padding to ensure proper alignment can be acheived */
+        length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+
+            mca_ptl_sm_component.first_fragment_size;
+
+        ompi_free_list_init(&mca_ptl_sm.sm_first_frags, length,
+                OBJ_CLASS(mca_ptl_sm_frag_t),
+                mca_ptl_sm_component.sm_first_frag_free_list_num,
+                mca_ptl_sm_component.sm_first_frag_free_list_max,
+                mca_ptl_sm_component.sm_first_frag_free_list_inc,
+                mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */
+
+        /* 
+         * second and beyond fragments 
+         */
+
+        /* allocation will be for the fragment descriptor, payload buffer,
+         * and padding to ensure proper alignment can be acheived */
+        length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+
+            mca_ptl_sm_component.max_fragment_size;
+
+        ompi_free_list_init(&mca_ptl_sm.sm_second_frags, length,
+                OBJ_CLASS(mca_ptl_sm_second_frag_t),
+                mca_ptl_sm_component.sm_second_frag_free_list_num,
+                mca_ptl_sm_component.sm_second_frag_free_list_max,
+                mca_ptl_sm_component.sm_second_frag_free_list_inc,
+                mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */
+
+        /* set flag indicating ptl has been inited */
+        mca_ptl_sm.ptl_inited=true;
+    }
+
     /* update the local smp process count */
     mca_ptl_sm_component.num_smp_procs+=n_local_procs;
 

src/mca/ptl/sm/src/ptl_sm.h

@@ -46,8 +46,8 @@ struct mca_ptl_sm_component_t {
     int sm_second_frag_free_list_num;       /**< initial size of free lists */
     int sm_second_frag_free_list_max;       /**< maximum size of free lists */
     int sm_second_frag_free_list_inc;       /**< number of elements to alloc when growing free lists */
-    size_t sm_max_procs;                      /**< upper limit on the number of processes using the shared memory pool */
-    size_t sm_extra_procs;                  /**< number of extra procs to allow */
+    int sm_max_procs;                      /**< upper limit on the number of processes using the shared memory pool */
+    int sm_extra_procs;                  /**< number of extra procs to allow */
     char* sm_mpool_name;                  /**< name of shared memory pool module */
     mca_mpool_base_module_t* sm_mpool; /**< shared memory pool */
     void* sm_mpool_base;                  /**< base address of shared memory pool */
@@ -134,8 +134,9 @@ struct mca_ptl_sm_t {
                                                     fragments that are
                                                     awaiting resources */
     ompi_mutex_t sm_pending_ack_lock;
-    ompi_list_t sm_pending_ack; /* list of fragmnent that need to be
+    ompi_list_t sm_pending_ack; /**< list of fragmnent that need to be
                                    acked */
+    bool ptl_inited;  /**< flag indicating if ptl has been inited */
 };
 typedef struct mca_ptl_sm_t mca_ptl_sm_t;
 

src/mca/ptl/sm/src/ptl_sm_component.c

@@ -118,6 +118,8 @@ int mca_ptl_sm_component_open(void)
         mca_ptl_sm_param_register_int("second_frag_free_list_inc", 256);
     mca_ptl_sm_component.sm_max_procs =
         mca_ptl_sm_param_register_int("max_procs", -1);
+    mca_ptl_sm_component.sm_extra_procs =
+        mca_ptl_sm_param_register_int("sm_extra_procs", -1);
     mca_ptl_sm_component.sm_mpool_name =
         mca_ptl_sm_param_register_string("mpool", "sm");
     mca_ptl_sm_component.first_fragment_size =
@@ -186,7 +188,6 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
     bool *have_hidden_threads)
 {
     mca_ptl_base_module_t **ptls = NULL;
-    size_t length;
 
     *num_ptls = 0;
     *allow_multi_user_threads = true;
@@ -198,40 +199,6 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
 
     mca_ptl_sm_component.sm_mpool_base = mca_ptl_sm_component.sm_mpool->mpool_base();
 
-    /* initialize fragment descriptor free list */
-
-    /* 
-     * first fragment 
-     */
-
-    /* allocation will be for the fragment descriptor, payload buffer,
-     * and padding to ensure proper alignment can be acheived */
-    length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+
-        mca_ptl_sm_component.first_fragment_size;
-
-    ompi_free_list_init(&mca_ptl_sm.sm_first_frags, length,
-        OBJ_CLASS(mca_ptl_sm_frag_t),
-        mca_ptl_sm_component.sm_first_frag_free_list_num,
-        mca_ptl_sm_component.sm_first_frag_free_list_max,
-        mca_ptl_sm_component.sm_first_frag_free_list_inc,
-        mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */
-
-    /* 
-     * second and beyond fragments 
-     */
-
-    /* allocation will be for the fragment descriptor, payload buffer,
-     * and padding to ensure proper alignment can be acheived */
-    length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+
-        mca_ptl_sm_component.max_fragment_size;
-
-    ompi_free_list_init(&mca_ptl_sm.sm_second_frags, length,
-        OBJ_CLASS(mca_ptl_sm_second_frag_t),
-        mca_ptl_sm_component.sm_second_frag_free_list_num,
-        mca_ptl_sm_component.sm_second_frag_free_list_max,
-        mca_ptl_sm_component.sm_second_frag_free_list_inc,
-        mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */
-
     /* publish shared memory parameters with the MCA framework */
     if(mca_ptl_sm_component_exchange() != OMPI_SUCCESS)
         return 0;
@@ -263,6 +230,9 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
     mca_ptl_sm_component.num_smp_procs=0;
     mca_ptl_sm_component.my_smp_rank=-1;
 
+    /* set flag indicating ptl not inited */
+    mca_ptl_sm.ptl_inited=false;
+
     return ptls;
 }