terminate the shared memory thread cleanly, at the end of the

job.  In mca_ptl_sm_component_close send termination request.

This commit was SVN r3784.

src/mca/pml/teg/src/pml_teg_sendreq.h

@@ -126,10 +126,6 @@ OBJ_CLASS_DECLARATION(mca_pml_teg_send_request_t);
     size_t first_fragment_size = ptl->ptl_first_frag_size;                 \
     int flags;                                                             \
                                                                            \
-    /* init/reinit request - do this here instead of init                  \
-     * as a persistent request may be reused, and there is                 \
-     * no additional cost                                                  \
-    */                                                                     \
     req->req_offset = 0;                                                   \
     req->req_lock = 0;                                                     \
     req->req_bytes_sent = 0;                                               \

src/mca/ptl/sm/src/ptl_sm.c

@@ -197,7 +197,6 @@ int mca_ptl_sm_add_procs_same_base_addr(
                }
                peer->peer_smp_rank=n_local_procs+
                    mca_ptl_sm_component.num_smp_procs;
-               n_local_procs++;
 
 #if OMPI_HAVE_THREADS == 1
                sprintf(path, "%s/sm_fifo.%d", ompi_process_info.job_session_dir, 
@@ -208,6 +207,7 @@ int mca_ptl_sm_add_procs_same_base_addr(
                    goto CLEANUP;
                }
 #endif
+               n_local_procs++;
                mca_ptl_sm_component.sm_proc_connect[proc]=SM_CONNECTED;
            }
        }

src/mca/ptl/sm/src/ptl_sm.h

@@ -53,6 +53,10 @@ extern mca_ptl_sm_module_resource_t mca_ptl_sm_module_resource;
 #define SM_CONNECTED_SAME_BASE_ADDR  2
 #define SM_CONNECTED_DIFFERENT_BASE_ADDR  3
 
+#if OMPI_HAVE_THREADS == 1
+#define DATA (char)0
+#define DONE (char)1
+#endif
 /**
  * Shared Memory (SM) PTL module.
  */
@@ -424,7 +428,7 @@ void mca_ptl_sm_component_event_thread(ompi_object_t*);
 #if OMPI_HAVE_THREADS == 1 
 #define MCA_PTL_SM_SIGNAL_PEER(peer) \
 { \
-    unsigned char cmd = 0; \
+    unsigned char cmd = DATA; \
     if(write(peer->fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) { \
         ompi_output(0, "mca_ptl_sm_send: write fifo failed: errno=%d\n", errno); \
     } \

src/mca/ptl/sm/src/ptl_sm_component.c

@@ -212,9 +212,16 @@ int mca_ptl_sm_component_close(void)
 #if OMPI_HAVE_THREADS == 1
     /* close/cleanup fifo create for event notification */
     if(mca_ptl_sm_component.sm_fifo_fd >= 0) {
+        /* write a done message down the pipe */
+        unsigned char cmd = DONE;
+        if( write(mca_ptl_sm_component.sm_fifo_fd,&cmd,sizeof(cmd)) != 
+                sizeof(cmd)){
+            ompi_output(0, "mca_ptl_sm_component_close: write fifo failed: errno=%d\n",
+                    errno);
+        }
+        ompi_thread_join(&mca_ptl_sm_component.sm_fifo_thread, NULL);
         close(mca_ptl_sm_component.sm_fifo_fd);
         unlink(mca_ptl_sm_component.sm_fifo_path);
-        ompi_thread_join(&mca_ptl_sm_component.sm_fifo_thread, NULL);
     }
 #endif
 
@@ -334,8 +341,13 @@ void mca_ptl_sm_component_event_thread(ompi_object_t* thread)
     while(1) {
         unsigned char cmd;
         if(read(mca_ptl_sm_component.sm_fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) {
+            /* error condition */
             return;
         }
+        if( DONE == cmd ){
+            /* return when done message received */
+            return;
+        } 
         mca_ptl_sm_component_progress(0);
     }
 }