figure out which procs are on my local host.

This commit was SVN r1366.

src/mca/ptl/sm/src/Makefile.am

@@ -14,8 +14,6 @@ noinst_LTLIBRARIES = libmca_ptl_sm.la
 libmca_ptl_sm_la_SOURCES = \
 	ptl_sm.c \
 	ptl_sm.h \
-	ptl_sm_mmap.c \
-	ptl_sm_mmap.h \
 	ptl_sm_module.c \
 	ptl_sm_sendreq.c \
 	ptl_sm_sendfrag.c \

src/mca/ptl/sm/src/ptl_sm.c

@@ -14,12 +14,13 @@
 #include "mca/ptl/base/ptl_base_recvfrag.h"
 #include "mca/base/mca_base_module_exchange.h"
 #include "ptl_sm.h"
+#include "util/sys_info.h"
 
 
 mca_ptl_sm_t mca_ptl_sm = {
     {
     &mca_ptl_sm_module.super,
-    0, /* ptl_exclusivity */
+    1, /* ptl_exclusivity */
     0, /* ptl_latency */
     0, /* ptl_andwidth */
     0, /* ptl_frag_first_size */
@@ -46,8 +47,10 @@ int mca_ptl_sm_add_procs(
     ompi_bitmap_t* reachability)
 {
     int proc,rc;
-    size_t size;
+    bool same_host;
+    size_t size,len,my_len;
     mca_ptl_sm_exchange_t **sm_proc_info;
+    ompi_proc_t* my_proc; /* pointer to caller's proc structure */
 
     /* allocate array to hold setup shared memory from all
      * other procs */
@@ -58,19 +61,57 @@ int mca_ptl_sm_add_procs(
         goto CLEANUP;
     }
 
+    /* get pointer to my proc structure */
+    my_proc=ompi_proc_local();
+    if( NULL == my_proc ) {
+        rc=OMPI_ERR_OUT_OF_RESOURCE;
+        goto CLEANUP;
+    }
+    my_len=strlen(ompi_system_info.nodename);
+
     /* get unique host identifier for each process in the list */
     for( proc=0 ; proc < nprocs; proc++ ) {
+        /* don't compare with self */
+        if( my_proc == procs[proc] ) {
+            continue;
+        }
         rc = mca_base_modex_recv(
-                &mca_ptl_sm_module.super.ptlm_version, proc,
+                &mca_ptl_sm_module.super.ptlm_version, procs[proc],
-                (void**)(sm_proc_info+proc), &size);
+                (void**)(&(sm_proc_info[proc])), &size);
         if(rc != OMPI_SUCCESS) {
             ompi_output(0, "mca_ptl_sm_add_procs: mca_base_modex_recv: failed with return value=%d", rc);
             goto CLEANUP;
         }
+       /* for zero length, just continue - comparison is meaningless*/
+       if( 0 >= size ) {
+           continue;
+       }
+       /* check to see if this proc is on my host */
+       len=strlen((char *)(sm_proc_info[proc]));
+       same_host=false;
+       if( len == my_len ) {
+           if( 0 == strncmp(ompi_system_info.nodename,
+                            (char *)(sm_proc_info[proc]),len) ) {
+               same_host=true;
+           }
+       }
+
+       /* add this proc to shared memory accessability list */
+       rc=ompi_bitmap_set_bit(reachability,proc);
+      if( OMPI_SUCCESS != rc ){
+          goto CLEANUP;
+      }
+
     }
 
     /* free local memory */
     if(sm_proc_info){
+       /* free the memory allocated by mca_base_modex_recv */
+        for( proc=0 ; proc < nprocs; proc++ ) {
+            if(sm_proc_info[proc]){
+                free(sm_proc_info[proc]);
+            }
+        }
         free(sm_proc_info);
     }
 
@@ -80,6 +121,8 @@ CLEANUP:
     if(sm_proc_info){
         free(sm_proc_info);
     }
+
+    return rc;
 }
 
 

src/mca/ptl/sm/src/ptl_sm.h

@@ -14,7 +14,6 @@
 #include "class/ompi_free_list.h"
 #include "mca/pml/pml.h"
 #include "mca/ptl/ptl.h"
-#include "ptl_sm_mmap.h"
 
 
 /**
@@ -29,7 +28,6 @@ struct mca_ptl_sm_module_1_0_0_t {
     ompi_free_list_t sm_send_requests;    /**< free list of sm send requests -- sendreq + sendfrag */
     ompi_free_list_t sm_send_frags;       /**< free list of sm send fragments */
     ompi_free_list_t sm_recv_frags;       /**< free list of sm recv fragments */
-    ompi_allocator_t sm_allocator;        /**< shared memory allocator */
     ompi_mutex_t sm_lock;
 };
 typedef struct mca_ptl_sm_module_1_0_0_t mca_ptl_sm_module_1_0_0_t;

src/mca/ptl/sm/src/ptl_sm_mmap.c

@@ -1,138 +0,0 @@
-#include <errno.h>
-#include <unistd.h>
-#include <string.h>
-#include <fcntl.h>
-#include <time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/errno.h>
-#include <sys/mman.h>
-
-#include "constants.h"
-#include "mca/pcm/pcm.h"
-#include "ptl_sm.h"
-#include "ptl_sm_mmap.h"
-
-
-OBJ_CLASS_INSTANCE(
-    mca_ptl_sm_mmap_t,
-    ompi_object_t,
-    NULL,
-    NULL
-);
-
-
-static mca_ptl_sm_mmap_t* mca_ptl_sm_mmap_open(size_t size)
-{
-    mca_ptl_sm_segment_t* seg;
-    mca_ptl_sm_mmap_t* map;
-    int fd = -1;
-
-    while(fd < 0) {
-        struct timespec ts;
-        /*fd = shm_open(mca_ptl_sm_module.sm_mmap_file, O_CREAT|O_RDWR, 0000); */
-        if(fd < 0 && errno != EACCES) {
-            ompi_output(0, "mca_ptl_sm_mmap_open: open failed with errno=%d\n", errno);
-            return NULL;
-        }
-        ts.tv_sec = 0; 
-        ts.tv_nsec = 500000;
-        nanosleep(&ts,NULL);
-    }
-
-    /* map the file and initialize segment state */
-    seg = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-    if(NULL == seg) {
-        ompi_output(0, "mca_ptl_sm_module_mmap: mmap failed with errno=%d\n", errno);
-        return NULL;
-    }
-    close(fd);
-
-    fprintf(stderr, "mapped at %08x", (unsigned int)seg);
-    map = OBJ_NEW(mca_ptl_sm_mmap_t);
-    map->sm_segment = seg;
-    map->sm_addr = (unsigned char*)(seg + 1);
-    map->sm_size = seg->seg_size;
-    return map;
-}
-
-
-mca_ptl_sm_mmap_t* mca_ptl_sm_mmap_init(size_t size)
-{
-    static int segnum = 0;
-
-    /* debug !!!!! */
-    return OMPI_SUCCESS;
-
-    ompi_job_handle_t job_handle = mca_pcm.pcm_handle_get();
-    char hostname[64];
-    int fd;
-    mca_ptl_sm_segment_t* seg;
-    mca_ptl_sm_mmap_t* map;
-
-    gethostname(hostname, sizeof(hostname));
-    sprintf(mca_ptl_sm_module.sm_mmap_file, "/%s.%s.%d", hostname, job_handle, segnum++);
-    /*fd = shm_open(mca_ptl_sm_module.sm_mmap_file, O_CREAT|O_RDWR, 0000); */
-    if(fd < 0) {
-        if(errno == EACCES)
-            return mca_ptl_sm_mmap_open(size);
-        ompi_output(0, "mca_ptl_sm_module_mmap: open failed with errno=%d\n", errno);
-        return NULL;
-    }
-                                                                                                                                          
-    /* truncate the file to the requested size */
-    if(ftruncate(fd, size) != 0) {
-        ompi_output(0, "mca_ptl_sm_module_mmap: ftruncate failed with errno=%d\n", errno);
-        return NULL;
-    }
-
-    /* map the file and initialize segment state */
-    seg = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-    if(NULL == seg) {
-        ompi_output(0, "mca_ptl_sm_module_mmap: mmap failed with errno=%d\n", errno);
-        return NULL;
-    }
-    fprintf(stderr, "mapped at %08x", (unsigned int)seg);
-
-    spinunlock(&seg->seg_lock);
-    seg->seg_offset = 0;
-    seg->seg_size = size;
-
-    map = OBJ_NEW(mca_ptl_sm_mmap_t);
-    map->sm_segment = seg;
-    map->sm_addr = (unsigned char*)(seg + 1);
-    map->sm_size = size;
-
-    /* enable access by other processes on this host */
-    if(fchmod(fd, 0600) != 0) {
-        ompi_output(0, "mca_ptl_sm_module_mmap: fchmod failed with errno=%d\n", errno);
-        OBJ_RELEASE(map);
-        close(fd);
-        return NULL;
-    }
-    close(fd);
-    return map;
-}
-
-
-void* mca_ptl_sm_mmap_alloc(ompi_allocator_t* allocator, size_t size)
-{
-    mca_ptl_sm_mmap_t* map = mca_ptl_sm_module.sm_mmap;
-    mca_ptl_sm_segment_t* seg = map->sm_segment;
-    void* addr;
-
-    spinlock(&seg->seg_lock);
-    addr = map->sm_addr + seg->seg_offset;
-    seg->seg_offset += size;
-    spinunlock(&seg->seg_lock);
-    return addr;
-}
-
-                                                                                                                                          
-void mca_ptl_sm_mmap_free(ompi_allocator_t* allocator, void* ptr)
-{
-    /* empty for now */
-}
-                                                                                                                                          
-                                                                                                                                          
-

src/mca/ptl/sm/src/ptl_sm_mmap.h

@@ -1,35 +0,0 @@
-#ifndef _PTL_SM_MMAP_H_
-#define _PTL_SM_MMAP_H_
-
-#include "class/ompi_object.h"
-#include "mem/allocator.h"
-#include "os/atomic.h"
-
-
-struct mca_ptl_sm_segment {
-    ompi_lock_data_t seg_lock;
-    size_t seg_offset;
-    size_t seg_size;
-};
-typedef struct mca_ptl_sm_segment mca_ptl_sm_segment_t;
-
-
-struct mca_ptl_sm_mmap {
-    ompi_object_t sm_base;
-    mca_ptl_sm_segment_t* sm_segment;
-    unsigned char* sm_addr;
-    size_t sm_size;
-};
-typedef struct mca_ptl_sm_mmap mca_ptl_sm_mmap_t;
-
-OBJ_CLASS_DECLARATION(mca_ptl_sm_mmap_t);
-
-
-
-mca_ptl_sm_mmap_t* mca_ptl_sm_mmap_init(size_t size);
-void* mca_ptl_sm_mmap_alloc(ompi_allocator_t*, size_t size);
-void  mca_ptl_sm_mmap_free(ompi_allocator_t*, void* alloc);
-
-
-#endif
-

src/mca/ptl/sm/src/ptl_sm_module.c

@@ -115,16 +115,10 @@ int mca_ptl_sm_module_open(void)
 
     /* initialize objects */
     OBJ_CONSTRUCT(&mca_ptl_sm_module.sm_lock, ompi_mutex_t);
-    OBJ_CONSTRUCT(&mca_ptl_sm_module.sm_allocator, ompi_allocator_t);
     OBJ_CONSTRUCT(&mca_ptl_sm_module.sm_send_requests, ompi_free_list_t);
     OBJ_CONSTRUCT(&mca_ptl_sm_module.sm_send_frags, ompi_free_list_t);
     OBJ_CONSTRUCT(&mca_ptl_sm_module.sm_recv_frags, ompi_free_list_t);
    
-    mca_ptl_sm_module.sm_allocator.alc_alloc_fn = mca_ptl_sm_mmap_alloc;
-    mca_ptl_sm_module.sm_allocator.alc_free_fn = mca_ptl_sm_mmap_free;
-
-    /* initialize state */
-    mca_ptl_sm_module.sm_mmap = NULL;
     return OMPI_SUCCESS;
 }
 
@@ -158,26 +152,30 @@ mca_ptl_t** mca_ptl_sm_module_init(
     *have_hidden_threads = OMPI_HAVE_THREADS;
 
     /* allocate a block of shared memory */
+/*
     mca_ptl_sm_module.sm_mmap = mca_ptl_sm_mmap_init(mca_ptl_sm_module.sm_min_alloc);
     if(NULL == mca_ptl_sm_module.sm_mmap)
          return NULL;
+*/
 
     /* initialize free lists */
+/*
     ompi_free_list_init(&mca_ptl_sm_module.sm_send_requests, 
         sizeof(mca_ptl_sm_send_request_t),
         OBJ_CLASS(mca_ptl_sm_send_request_t),
         mca_ptl_sm_module.sm_free_list_num,
         mca_ptl_sm_module.sm_free_list_max,
         mca_ptl_sm_module.sm_free_list_inc,
-        &mca_ptl_sm_module.sm_allocator); /* use shared-memory allocator */
+        &mca_ptl_sm_module.sm_allocator); *//* use shared-memory allocator */
 
+/*
     ompi_free_list_init(&mca_ptl_sm_module.sm_recv_frags, 
         sizeof(mca_ptl_sm_recv_frag_t),
         OBJ_CLASS(mca_ptl_sm_recv_frag_t),
         mca_ptl_sm_module.sm_free_list_num,
         mca_ptl_sm_module.sm_free_list_max,
         mca_ptl_sm_module.sm_free_list_inc,
-        &mca_ptl_sm_module.sm_allocator); /* use default allocator */
+        &mca_ptl_sm_module.sm_allocator); *//* use default allocator */
 
     /* publish shared memory parameters with the MCA framework */
     if(mca_ptl_sm_module_exchange() != OMPI_SUCCESS)
@@ -253,6 +251,7 @@ static int mca_ptl_sm_module_exchange()
     }
     mca_ptl_sm_setup_info.host_name[len]='\0';
 
+
     /* exchange setup information */
     size=sizeof(mca_ptl_sm_exchange_t);
     rc =  mca_base_modex_send(&mca_ptl_sm_module.super.ptlm_version,