more tweaking of i/f functions

This commit was SVN r323.

src/mca/mpi/pml/pml.h

@@ -49,8 +49,11 @@ typedef struct mca_pml_base_status_t mca_pml_base_status_t;
  * PML interface functions
  */
 
-typedef int (*mca_pml_base_add_procs_fn_t)(struct lam_proc_t **procs, int nprocs);
+typedef int (*mca_pml_base_add_comm_fn_t)(struct lam_communicator_t*);
-typedef int (*mca_pml_base_add_ptls_fn_t)(struct mca_ptl_t **ptls, int nptls);
+typedef int (*mca_pml_base_del_comm_fn_t)(struct lam_communicator_t*);
+typedef int (*mca_pml_base_add_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs);
+typedef int (*mca_pml_base_del_procs_fn_t)(struct lam_proc_t **procs, size_t nprocs);
+typedef int (*mca_pml_base_add_ptls_fn_t)(struct mca_ptl_t **ptls, size_t nptls);
 typedef int (*mca_pml_base_progress_fn_t)(mca_pml_base_tstamp_t);
 
 typedef int (*mca_pml_base_irecv_init_fn_t)(
@@ -131,7 +134,10 @@ typedef struct mca_pml_base_module_1_0_0_t mca_pml_base_module_1_0_0_t;
 
 struct mca_pml_1_0_0_t {
 
+    mca_pml_base_add_comm_fn_t     pml_add_comm;
+    mca_pml_base_del_comm_fn_t     pml_del_comm;
     mca_pml_base_add_procs_fn_t    pml_add_procs;
+    mca_pml_base_del_procs_fn_t    pml_del_procs;
     mca_pml_base_add_ptls_fn_t     pml_add_ptls;
     mca_pml_base_fini_fn_t         pml_fini;
 

src/mca/mpi/pml/teg/src/Makefile.am

@@ -18,7 +18,7 @@ libmca_coll_basic_la_SOURCES = \
 	pml_teg_isend.c \
 	pml_teg_proc.c \
 	pml_teg_proc.h \
-	pml_teg_ptl_array.c  \
-	pml_teg_ptl_array.h \
 	pml_teg_sendreq.c \
-	pml_teg_sendreq.h
+	pml_teg_sendreq.h \
+	pml_ptl_array.c  \
+	pml_ptl_array.h 

src/mca/mpi/pml/teg/src/pml_ptl_array.c

@@ -0,0 +1,53 @@
+/*
+ * $HEADER$
+ */
+
+#include <string.h>
+#include "lam/util/malloc.h"
+#include "pml_ptl_array.h"
+
+
+lam_class_info_t mca_pml_teg_proc_cls = {
+    "mca_ptl_array_t",
+    &lam_object_cls,
+    (class_init_t) mca_ptl_array_init,
+    (class_destroy_t) mca_ptl_array_destroy
+};
+                                                                                                                             
+
+void mca_ptl_array_init(mca_ptl_array_t* ptl_array)
+{
+    SUPER_INIT(ptl_array, &lam_object_cls);
+    ptl_array->ptl_array = 0;
+    ptl_array->ptl_size = 0;
+    ptl_array->ptl_index = 0;
+    ptl_array->ptl_reserve = 0;
+}
+
+
+void mca_ptl_array_destroy(mca_ptl_array_t* ptl_array)
+{
+    if(ptl_array->ptl_array != 0)
+        LAM_FREE(ptl_array->ptl_array);
+    SUPER_DESTROY(ptl_array, &lam_object_cls);
+}
+
+
+int mca_ptl_array_reserve(mca_ptl_array_t* ptl_array, size_t size)
+{
+    mca_ptl_info_t *array;
+    if(ptl_array->ptl_reserve >= size)
+        return LAM_SUCCESS;
+    
+    array = (mca_ptl_info_t*)LAM_MALLOC(sizeof(mca_ptl_array_t)*size);
+    if(array == 0)
+        return LAM_ERR_OUT_OF_RESOURCE;
+    if(ptl_array->ptl_size) {
+         memcpy(ptl_array->ptl_array, array, ptl_array->ptl_size * sizeof(mca_ptl_info_t));
+         LAM_FREE(ptl_array->ptl_array);
+    }
+    memset(ptl_array->ptl_array+(size-ptl_array->ptl_size), 0, (size-ptl_array->ptl_size)*sizeof(mca_ptl_info_t));
+    ptl_array->ptl_reserve = size;
+    return LAM_SUCCESS;
+}
+

src/mca/mpi/pml/teg/src/pml_ptl_array.h

@@ -0,0 +1,48 @@
+/*
+ * $HEADER$
+ */
+
+#ifndef LAM_PTL_ARRAY_H
+#define LAM_PTL_ARRAY_H
+
+#include "mca/mpi/ptl/ptl.h"
+
+extern lam_class_info_t mca_ptl_array_cls;
+
+struct mca_ptl_info_t {
+    double ptl_weight;   /* PTL weight for scheduling */
+    mca_ptl_t *ptl;      /* PTL implementation */
+};
+typedef struct mca_ptl_info_t mca_ptl_info_t;
+
+struct mca_ptl_array_t {
+    lam_object_t     super;
+    mca_ptl_info_t*  ptl_array;   /* array of PTL info */
+    size_t           ptl_size;    /* number available */
+    size_t           ptl_reserve;
+    size_t           ptl_index;   /* last used index*/
+};
+typedef struct mca_ptl_array_t mca_ptl_array_t;
+
+
+void mca_ptl_array_init(mca_ptl_array_t*);
+void mca_ptl_array_destroy(mca_ptl_array_t*);
+int  mca_ptl_array_reserve(mca_ptl_array_t*, size_t);
+
+static inline void mca_ptl_array_set_size(mca_ptl_array_t* array, size_t size)
+{
+    if(array->ptl_size > array->ptl_reserve)
+        mca_ptl_array_reserve(array, size);
+    array->ptl_size = size;
+}
+
+static inline mca_ptl_info_t* mca_ptl_array_get_next(mca_ptl_array_t* ptl_array)
+{
+    mca_ptl_info_t* ptl_info = &ptl_array->ptl_array[ptl_array->ptl_index++];
+    if(ptl_array->ptl_index == ptl_array->ptl_size)
+        ptl_array->ptl_index = 0;
+    return ptl_info;
+}
+
+#endif
+

src/mca/mpi/pml/teg/src/pml_teg.c

@@ -1,6 +1,10 @@
+#include "lam/util/malloc.h"
+#include "mca/mpi/pml/pml.h"
+#include "mca/mpi/ptl/ptl.h"
 #include "mca/mpi/pml/base/pml_base_sendreq.h"
 #include "mca/mpi/pml/base/pml_base_recvreq.h"
 #include "pml_teg.h"
+#include "pml_teg_proc.h"
 
 #define mca_pml_teg_param_register_int(n,v) \
     mca_base_param_lookup_int( \
@@ -39,7 +43,10 @@ mca_pml_base_module_1_0_0_t mca_pml_teg_module = {
 
 mca_pml_teg_t mca_pml_teg = {
     {
+    mca_pml_teg_add_comm,
+    mca_pml_teg_del_comm,
     mca_pml_teg_add_procs,
+    mca_pml_teg_del_procs,
     mca_pml_teg_add_ptls,
     mca_pml_teg_fini,
     mca_pml_teg_irecv_init,
@@ -65,6 +72,53 @@ mca_pml_1_0_0_t* mca_pml_teg_init(int* priority, int* min_thread, int* max_threa
     *priority = 0;
     *min_thread = 0;
     *max_thread = 0;
+    mca_pml_teg.teg_ptls = 0;
+    mca_pml_teg.teg_num_ptls = 0;
+    lam_list_init(&mca_pml_teg.teg_incomplete_sends);
+    lam_mutex_init(&mca_pml_teg.teg_lock);
     return &mca_pml_teg.super;
 }
 
+int mca_pml_teg_add_ptls(struct mca_ptl_t** ptls, size_t nptls)
+{
+    mca_pml_teg.teg_ptls = ptls;
+    mca_pml_teg.teg_num_ptls = nptls;
+    return LAM_SUCCESS;
+}
+
+int mca_pml_teg_add_comm(lam_communicator_t* comm)
+{
+    return LAM_SUCCESS;
+}
+
+int mca_pml_teg_del_comm(lam_communicator_t* comm)
+{
+    return LAM_SUCCESS;
+}
+
+int mca_pml_teg_add_procs(lam_proc_t** procs, size_t nprocs)
+{
+    size_t i;
+    /* initialize each proc */
+    for(i=0; i<nprocs; i++) {
+        lam_proc_t *proc = procs[i];
+        if(proc->proc_pml == 0) {
+            /* allocate pml specific proc data */
+            mca_pml_proc_t* proc_pml = (mca_pml_proc_t*)LAM_MALLOC(sizeof(mca_pml_proc_t));
+            mca_pml_teg_proc_init(proc_pml);
+            /* preallocate space in array for max number of ptls */
+            mca_ptl_array_reserve(&proc_pml->proc_ptl_first, mca_pml_teg.teg_num_ptls);
+            mca_ptl_array_reserve(&proc_pml->proc_ptl_first, mca_pml_teg.teg_num_ptls);
+            proc_pml->proc_lam = proc;
+            proc->proc_pml = proc_pml;
+        }
+    }
+
+    /* allow each ptl to add itself to proc array */
+    for(i=0; i<mca_pml_teg.teg_num_ptls; i++) {
+        mca_ptl_t* ptl = mca_pml_teg.teg_ptls[i];
+        ptl->ptl_add_procs(ptl, procs, nprocs);
+    }
+    return LAM_SUCCESS;
+}
+

src/mca/mpi/pml/teg/src/pml_teg.h

@@ -9,6 +9,27 @@
 #include "lam/util/cmd_line.h"
 #include "mpi/request/request.h"
 #include "mca/mpi/pml/pml.h"
+#include "mca/mpi/ptl/ptl.h"
+
+
+/*
+ * TEG PML Interface
+ */
+
+struct mca_pml_teg_t {
+    mca_pml_t super;
+
+    /* available ptls */
+    mca_ptl_t** teg_ptls;
+    size_t teg_num_ptls;
+
+    /* incomplete posted sends */
+    lam_list_t  teg_incomplete_sends;
+    lam_mutex_t teg_lock;
+};
+typedef struct mca_pml_teg_t mca_pml_teg_t;
+
+extern mca_pml_teg_t mca_pml_teg;
 
 
 /*
@@ -31,34 +52,31 @@ extern mca_pml_1_0_0_t* mca_pml_teg_init(
 );
 
 
-
-/*
- * TEG PML Interface
- */
-
-struct mca_pml_teg_t {
-    mca_pml_t super;
-
-    /* incomplete posted sends */
-    lam_list_t  teg_incomplete_sends;
-    lam_mutex_t teg_lock;
-};
-typedef struct mca_pml_teg_t mca_pml_teg_t;
-
-extern mca_pml_teg_t mca_pml_teg;
-
 /*
  * PML interface functions.
  */
 
+extern int mca_pml_teg_add_comm(
+    struct lam_communicator_t* comm
+);
+
+extern int mca_pml_teg_del_comm(
+    struct lam_communicator_t* comm
+);
+
 extern int mca_pml_teg_add_procs(
     struct lam_proc_t **procs,
-    int nprocs
+    size_t nprocs
+);
+
+extern int mca_pml_teg_del_procs(
+    struct lam_proc_t **procs,
+    size_t nprocs
 );
 
 extern int mca_pml_teg_add_ptls(
     struct mca_ptl_t **ptls,
-    int nptls
+    size_t nptls
 );
 
 extern int mca_pml_teg_fini(void);

src/mca/mpi/pml/teg/src/pml_teg_proc.c

@@ -4,7 +4,7 @@
 
 #include "lam/atomic.h"
 #include "pml_teg_proc.h"
-#include "pml_teg_ptl_array.h"
+#include "pml_ptl_array.h"
 
 lam_class_info_t mca_pml_teg_proc_cls = { 
     "mca_pml_teg_proc_t", 

src/mca/mpi/pml/teg/src/pml_teg_proc.h

@@ -8,7 +8,7 @@
 #include "mpi/communicator/communicator.h"
 #include "mpi/group/group.h"
 #include "mpi/proc/proc.h"
-#include "pml_teg_ptl_array.h"
+#include "pml_ptl_array.h"
 
 extern lam_class_info_t mca_pml_teg_proc_cls;
 

src/mca/mpi/ptl/ptl.h

@@ -14,10 +14,10 @@
 
 
 /*
- * Set the default type to use version 1.0.0 of the PTL 
+ * PTL type 
  */
 
-typedef struct mca_ptl_module_1_0_0_t mca_ptl_module_t;
+struct mca_ptl_1_0_0_t;
 typedef struct mca_ptl_1_0_0_t mca_ptl_t;
 
 
@@ -26,15 +26,16 @@ typedef struct mca_ptl_1_0_0_t mca_ptl_t;
  */
 
 typedef struct mca_ptl_1_0_0_t** (*mca_ptl_init_1_0_0_fn_t)(int* num_ptls, int *thread_min, int *thread_max);
-typedef int (*mca_ptl_fini_1_0_0_fn_t)(struct mca_ptl_1_0_0_t*);
+typedef int (*mca_ptl_add_procs_fn_t)(mca_ptl_t*, struct lam_proc_t**, size_t nprocs);
+typedef int (*mca_ptl_fini_fn_t)(mca_ptl_t*);
                                                                                                          
 /*
  * PTL action functions.
  */
 
 typedef mca_pml_base_send_request_t* (*mca_ptl_request_alloc_fn_t)(mca_ptl_t*);
-typedef int (*mca_ptl_fragment_fn_t)(mca_ptl_t*, mca_pml_base_send_request_t*, size_t);
+typedef int (*mca_ptl_fragment_fn_t)(struct mca_ptl_t*, mca_pml_base_send_request_t*, size_t);
-typedef int (*mca_ptl_progress_fn_t)(mca_ptl_t*, mca_pml_base_tstamp_t);
+typedef int (*mca_ptl_progress_fn_t)(struct mca_ptl_t*, mca_pml_base_tstamp_t);
 
 /*
  * Struct used to pass PTL module information from the each PTL 
@@ -47,6 +48,7 @@ struct mca_ptl_module_1_0_0_t {
   mca_ptl_init_1_0_0_fn_t ptlm_init;
 };
 typedef struct mca_ptl_module_1_0_0_t mca_ptl_module_1_0_0_t;
+typedef struct mca_ptl_module_1_0_0_t mca_ptl_module_t;
 
 /*
  * Struct that represents the common state and interface functions
@@ -64,9 +66,11 @@ struct mca_ptl_1_0_0_t {
     size_t       ptl_endpoint_count;      /* number endpoints supported by this CDI */
 
     /* PTL function table */
-    mca_ptl_request_alloc_fn_t ptl_request_alloc;
+    mca_ptl_add_procs_fn_t      ptl_add_procs;
+    mca_ptl_fini_fn_t           ptl_fini;
     mca_ptl_fragment_fn_t       ptl_fragment;
     mca_ptl_progress_fn_t       ptl_progress;
+    mca_ptl_request_alloc_fn_t  ptl_request_alloc;
 
 };
 typedef struct mca_ptl_1_0_0_t mca_ptl_1_0_0_t;

src/mca/mpi/ptl/tcp/.cvsignore

@@ -0,0 +1,25 @@
+Makefile
+Makefile.in
+acinclude.m4
+aclocal.m4
+configure
+configure.ac
+config.log
+config.status
+libtool
+autom4te.cache
+.libs
+.deps
+*.la
+*.lo
+.lam*
+config.guess
+config.sub
+depcomp
+install-sh
+ltmain.sh
+missing
+mkinstalldirs
+stamp-h1
+teg_config.h
+teg_config.h.in

src/mca/mpi/ptl/tcp/src/.cvsignore

@@ -0,0 +1,25 @@
+Makefile
+Makefile.in
+acinclude.m4
+aclocal.m4
+configure
+configure.ac
+config.log
+config.status
+libtool
+autom4te.cache
+.libs
+.deps
+*.la
+*.lo
+.lam*
+config.guess
+config.sub
+depcomp
+install-sh
+ltmain.sh
+missing
+mkinstalldirs
+stamp-h1
+teg_config.h
+teg_config.h.in