- added PTL specific proc addressing info to datastructure attached to proc

- changed i/f functions to pass addressing info through to ptl

This commit was SVN r441.

src/mca/mpi/pml/pml.h

@@ -65,7 +65,6 @@ typedef int (*mca_pml_base_add_comm_fn_t)(struct lam_communicator_t*);
 typedef int (*mca_pml_base_del_comm_fn_t)(struct lam_communicator_t*);
 typedef int (*mca_pml_base_add_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs);
 typedef int (*mca_pml_base_del_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs);
-typedef int (*mca_pml_base_add_ptls_fn_t)(struct mca_ptl_t **ptls, size_t nptls);
 typedef int (*mca_pml_base_fini_fn_t)(void);
 typedef int (*mca_pml_base_progress_fn_t)(void);
 

src/mca/mpi/pml/teg/src/pml_ptl_array.h

@@ -12,7 +12,7 @@ extern lam_class_info_t mca_ptl_array_cls;
 
 struct mca_ptl_proc_t {
     double ptl_weight;               /* PTL weight for scheduling */
-    struct mca_ptl_addr_t* ptl_addr; /* PTL addressing info */
+    struct mca_ptl_peer_t* ptl_peer; /* PTL addressing info */
     mca_ptl_t *ptl;                  /* PTL implementation */
 };
 typedef struct mca_ptl_proc_t mca_ptl_proc_t;

src/mca/mpi/pml/teg/src/pml_teg.c

@@ -114,14 +114,14 @@ int mca_pml_teg_add_procs(lam_proc_t** procs, size_t nprocs)
             /* if the ptl can reach the destination proc it will return 
              * addressing information that will be cached on the proc
              */
-            struct mca_ptl_addr_t* ptl_addr;
-            int rc = ptl->ptl_add_proc(ptl, proc, &ptl_addr);
+            struct mca_ptl_peer_t* ptl_peer;
+            int rc = ptl->ptl_add_proc(ptl, proc, &ptl_peer);
             if(rc == LAM_SUCCESS) {
 
                 /* cache the ptl on the proc */
                 mca_ptl_proc_t* ptl_proc  = mca_ptl_array_insert(&proc_pml->proc_ptl_next);
                 ptl_proc->ptl = ptl;
-                ptl_proc->ptl_addr = ptl_addr;
+                ptl_proc->ptl_peer = ptl_peer;
                 ptl_proc->ptl_weight = 0;
 
                 /* if this ptl supports exclusive access then don't allow 
@@ -200,7 +200,7 @@ int mca_pml_teg_del_procs(lam_proc_t** procs, size_t nprocs)
             mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, f_index);
             mca_ptl_t* ptl = ptl_proc->ptl;
             
-            ptl->ptl_del_proc(ptl,proc,ptl_proc->ptl_addr);
+            ptl->ptl_del_proc(ptl,proc,ptl_proc->ptl_peer);
 
             /* remove this from next array so that we dont call it twice w/ 
              * the same address pointer
@@ -221,7 +221,7 @@ int mca_pml_teg_del_procs(lam_proc_t** procs, size_t nprocs)
             mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, n_index);
             mca_ptl_t* ptl = ptl_proc->ptl;
             if (ptl != 0)
-                ptl->ptl_del_proc(ptl,proc,ptl_proc->ptl_addr);
+                ptl->ptl_del_proc(ptl,proc,ptl_proc->ptl_peer);
         }
         
         /* do any required cleanup */

src/mca/mpi/pml/teg/src/pml_teg_sendreq.h

@@ -46,7 +46,7 @@ static inline int mca_pml_teg_send_request_start(
     // start the first fragment
     if(req->req_length < first_fragment_size)
         first_fragment_size = req->req_length;
-    rc = ptl->ptl_send(ptl, ptl_proc->ptl_addr, req, first_fragment_size, &complete);
+    rc = ptl->ptl_send(ptl, ptl_proc->ptl_peer, req, first_fragment_size, &complete);
     if(rc != LAM_SUCCESS)
         return rc;
 

src/mca/mpi/ptl/ptl.h

@@ -21,7 +21,7 @@
  */
 
 struct mca_ptl_t;
-struct mca_ptl_addr_t;
+struct mca_ptl_peer_t;
 struct mca_ptl_base_fragment_t;
 struct mca_ptl_base_send_request_t;
 
@@ -86,7 +86,7 @@ typedef struct mca_ptl_base_module_1_0_0_t mca_ptl_base_module_t;
 typedef int (*mca_ptl_base_add_proc_fn_t)(
     struct mca_ptl_t* ptl, 
     struct lam_proc_t* proc, 
-    struct mca_ptl_addr_t**
+    struct mca_ptl_peer_t**
 );
 
 /**
@@ -100,7 +100,7 @@ typedef int (*mca_ptl_base_add_proc_fn_t)(
 typedef int (*mca_ptl_base_del_proc_fn_t)(
     struct mca_ptl_t* ptl, 
     struct lam_proc_t* proc, 
-    struct mca_ptl_addr_t*
+    struct mca_ptl_peer_t*
 );
 
 /**
@@ -127,7 +127,7 @@ typedef int (*mca_ptl_base_request_alloc_fn_t)(
 
 typedef int (*mca_ptl_base_send_fn_t)(
     struct mca_ptl_t* ptl, 
-    struct mca_ptl_addr_t* ptl_addr, 
+    struct mca_ptl_peer_t* ptl_peer, 
     struct mca_ptl_base_send_request_t* send_request,
     size_t size,
     bool* complete

src/mca/mpi/ptl/tcp/.cvsignore

@@ -21,5 +21,5 @@ ltmain.sh
 missing
 mkinstalldirs
 stamp-h1
-teg_config.h
-teg_config.h.in
+tcp_config.h
+tcp_config.h.in

src/mca/mpi/ptl/tcp/src/Makefile.am

@@ -14,5 +14,6 @@ noinst_LTLIBRARIES = libmca_ptl_tcp.la
 libmca_ptl_tcp_la_SOURCES = \
 	ptl_tcp.c \
 	ptl_tcp.h \
+	ptl_tcp_peer.c \
 	ptl_tcp_module.c \
 	ptl_tcp_send.c 

src/mca/mpi/ptl/tcp/src/ptl_tcp.c

@@ -13,12 +13,12 @@
 mca_ptl_tcp_t mca_ptl_tcp = {
     {
     &mca_ptl_tcp_module.super,
-    0, /* ptl_exclusive */
+    0, /* ptl_exclusivity */
+    0, /* ptl_latency */
+    0, /* ptl_andwidth */
     0, /* ptl_frag_first_size */
     0, /* ptl_frag_min_size */
     0, /* ptl_frag_max_size */
-    0, /* ptl_latency */
-    0, /* ptl_andwidth */
     mca_ptl_tcp_add_proc,
     mca_ptl_tcp_del_proc,
     mca_ptl_tcp_fini,
@@ -31,10 +31,8 @@ mca_ptl_tcp_t mca_ptl_tcp = {
 int mca_ptl_tcp_create(int if_index)
 {
     mca_ptl_tcp_t* ptl = (mca_ptl_tcp_t*)LAM_MALLOC(sizeof(mca_ptl_tcp_t));
-    if(NULL == ptl) {
-        lam_output(0,"mca_ptl_tcp: unable to allocate ptl");
+    if(NULL == ptl)
         return LAM_ERR_OUT_OF_RESOURCE;
-    }
     memcpy(ptl, &mca_ptl_tcp, sizeof(mca_ptl_tcp));
     mca_ptl_tcp_module.tcp_ptls[mca_ptl_tcp_module.tcp_num_ptls++] = ptl;
 
@@ -45,7 +43,12 @@ int mca_ptl_tcp_create(int if_index)
 }
 
 
-int mca_ptl_tcp_add_proc(struct mca_ptl_t* ptl, struct lam_proc_t *procs, struct mca_ptl_addr_t** ptl_addr)
+int mca_ptl_tcp_add_proc(struct mca_ptl_t* ptl, struct lam_proc_t *proc, struct mca_ptl_peer_t** ptl_peer)
+{
+    return LAM_SUCCESS;
+}
+
+int mca_ptl_tcp_del_proc(struct mca_ptl_t* ptl, struct lam_proc_t *proc, struct mca_ptl_peer_t* ptl_peer)
 {
     return LAM_SUCCESS;
 }

src/mca/mpi/ptl/tcp/src/ptl_tcp.h

@@ -74,13 +74,13 @@ extern int mca_ptl_tcp_fini(
 extern int mca_ptl_tcp_add_proc(
     struct mca_ptl_t* ptl,
     struct lam_proc_t *procs,
-    struct mca_ptl_addr_t** addr
+    struct mca_ptl_peer_t** addr
 );
 
 extern int mca_ptl_tcp_del_proc(
     struct mca_ptl_t* ptl,
     struct lam_proc_t *procs,
-    struct mca_ptl_addr_t* addr
+    struct mca_ptl_peer_t* addr
 );
 
 extern int mca_ptl_tcp_request_alloc(
@@ -90,7 +90,7 @@ extern int mca_ptl_tcp_request_alloc(
                                                                                                  
 extern int mca_ptl_tcp_send(
     struct mca_ptl_t* ptl,
-    struct mca_ptl_addr_t* ptl_addr,
+    struct mca_ptl_peer_t* ptl_peer,
     struct mca_ptl_base_send_request_t*,
     size_t size,
     bool* complete

src/mca/mpi/ptl/tcp/src/ptl_tcp_peer.c

@@ -0,0 +1,35 @@
+#include <stdlib.h>
+#include <unistd.h>
+#include "ptl_tcp_peer.h"
+
+
+lam_class_info_t  mca_ptl_addr_cls = {
+    "mca_ptl_addr_t", 
+    &lam_list_cls,
+    (class_init_t)mca_ptl_tcp_peer_init, 
+    (class_destroy_t)mca_ptl_tcp_peer_destroy
+};
+
+
+void mca_ptl_tcp_peer_init(mca_ptl_peer_t* ptl_peer)
+{
+    SUPER_INIT(ptl_peer, &lam_list_cls);
+    ptl_peer->tcp_state = MCA_PTL_TCP_CLOSED;
+    ptl_peer->tcp_sd = -1;
+}
+
+void mca_ptl_tcp_peer_destroy(mca_ptl_peer_t* ptl_peer)
+{
+    mca_ptl_tcp_peer_close(ptl_peer);
+    SUPER_DESTROY(ptl_peer, &lam_list_cls);
+}
+
+void mca_ptl_tcp_peer_close(mca_ptl_peer_t* ptl_peer)
+{
+    if(ptl_peer->tcp_sd >= 0) {
+        close(ptl_peer->tcp_sd);
+        ptl_peer->tcp_sd = -1;
+    }
+    ptl_peer->tcp_state = MCA_PTL_TCP_CLOSED;
+}
+

src/mca/mpi/ptl/tcp/src/ptl_tcp_peer.h

@@ -0,0 +1,51 @@
+/* @file
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_PTL_TCP_PEER_H
+#define MCA_PTL_TCP_PEER_H
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include "lam/lfc/list.h"
+#include "lam/util/reactor.h"
+#include "mca/mpi/pml/pml.h"
+#include "mca/mpi/ptl/ptl.h"
+
+
+typedef enum { 
+    MCA_PTL_TCP_CLOSED, 
+    MCA_PTL_TCP_CONNECTING,
+    MCA_PTL_TCP_CONNECT_ACK,
+    MCA_PTL_TCP_CONNECTED,
+    MCA_PTL_TCP_FAILED
+} mca_ptl_tcp_state_t;
+    
+
+extern lam_class_info_t mca_ptl_peer_cls;
+
+/**
+ *  An abstraction that represents a connection to a peer process.
+ *  An instance of mca_ptl_peer_t is associated w/ each process 
+ *  in the group at startup. However, connections to the peer
+ *  are established dynamically on an as-needed basis:
+*/
+                                                                                                                            
+struct mca_ptl_peer_t {
+    lam_list_item_t     super;
+    struct lam_proc_t  *tcp_proc;
+    struct sockaddr_in  tcp_peer;
+    int                 tcp_sd;
+    mca_ptl_tcp_state_t tcp_state;
+};
+typedef struct mca_ptl_peer_t mca_ptl_peer_t;
+
+
+void mca_ptl_tcp_peer_init(mca_ptl_peer_t*);
+void mca_ptl_tcp_peer_destroy(mca_ptl_peer_t*);
+void mca_ptl_tcp_peer_close(mca_ptl_peer_t*);
+
+#endif
+

src/mca/mpi/ptl/tcp/src/ptl_tcp_send.c

@@ -5,7 +5,7 @@
                                                                                                                  
 int mca_ptl_tcp_send(
     struct mca_ptl_t* ptl,
-    struct mca_ptl_addr_t* ptl_addr,
+    struct mca_ptl_peer_t* ptl_addr,
     struct mca_ptl_base_send_request_t* sendreq,
     size_t size,
     bool* complete)