Add req.[hc] and frag.[hc] for compilation

This commit was SVN r1420.

src/mca/ptl/elan/Makefile.am

@@ -6,6 +6,12 @@
 
 include $(top_ompi_srcdir)/config/Makefile.options
 
+AM_CFLAGS = \
+        -I$(top_lam_builddir)/src/include \
+        -I$(top_lam_srcdir)/src \
+        -I$(top_lam_srcdir)/src/include \
+        -lelan -lelanctrl -lrms -lrmscall -lelan4 
+
 SUBDIRS = src
 
 EXTRA_DIST = VERSION

src/mca/ptl/elan/src/Makefile.am

@@ -5,14 +5,22 @@
 
 include $(top_ompi_srcdir)/config/Makefile.options
 
+AM_CPPFLAGS =  \
+        -I$(top_ompi_builddir)/src/include \
+        -I$(top_ompi_srcdir)/src \
+        -I$(top_ompi_srcdir)/src/include
+
 noinst_LTLIBRARIES = libmca_ptl_elan.la
 libmca_ptl_elan_la_SOURCES = \
+	elan_config.h \
 	ptl_elan.h \
 	ptl_elan_frag.h \
 	ptl_elan_req.h \
 	ptl_elan_proc.h \
 	ptl_elan_addr.h \
 	ptl_elan_peer.h \
+	ptl_elan_frag.c \
+	ptl_elan_req.c \
 	ptl_elan_proc.c \
 	ptl_elan_module.c \
 	ptl_elan_priv.c \

src/mca/ptl/elan/src/ptl_elan.h

@@ -10,18 +10,13 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
-#include "mem/free_list.h"
+#include "class/ompi_free_list.h"
 #include "event/event.h"
 #include "mca/pml/pml.h"
 #include "mca/ptl/ptl.h"
 
 #define MCA_PTL_ELAN_STATISTICS 0
 
-/*#define __ELAN__*/
-/*#define __elan4__*/
-
-/*#include "elan/sys/misc_sys.h"*/
-/*#include "elan/sys/init_sys.h"*/
 #include "elan/elan.h"
 #include "elan/init.h"
 #include "rms/rmscall.h"
@@ -68,11 +63,11 @@ struct mca_ptl_elan_state_t {
      *   the complete structure of the ELAN_EPRIVSATE.
      */
     ELAN_LOCATION    elan_myloc;
-    void         *elan_cap;    /**< job capability */
+    void            *elan_cap;    /**< job capability */
-    void         *elan_estate; /**< Elan state of the 0th rail */
+    void            *elan_estate; /**< Elan state of the 0th rail */
-
+                  
-    ELAN_CTX     *elan_ctx;    /**< Elan ctx of the 0th rail */
+    ELAN_CTX        *elan_ctx;    /**< Elan ctx of the 0th rail */
-    ELAN_RAIL   **elan_rail;   /**< Rail control struct for all rails */
+    ELAN_RAIL      **elan_rail;   /**< Rail control struct for all rails */
     struct ompi_elan_rail_t ** all_rails;   /**< all rails */
     ELAN4_COOKIEPOOL *elan_cpool;
     ELAN_ESTATE **all_estates; /**< elan (priv)states of all rails */
@@ -83,8 +78,6 @@ struct mca_ptl_elan_module_1_0_0_t {
 
     mca_ptl_base_module_1_0_0_t super;       /**< base PTL module */
 
-    /* These parameters does not provided good freedom,
-     * It does not hurt to skip them */
     int   elan_free_list_num;     /**< initial size of free lists */
     int   elan_free_list_max;     /**< maximum size of free lists */
     int   elan_free_list_inc;     /**< # to alloc when growing lists */
@@ -101,13 +94,17 @@ struct mca_ptl_elan_module_1_0_0_t {
     ompi_list_t elan_reqs;        /**< all elan requests */
     ompi_list_t elan_prog_events; /**< events in progress */
     ompi_list_t elan_comp_events; /**< events completed, but to reclaim */
+    ompi_list_t  elan_procs;       /**< elan proc's */
+    ompi_list_t  elan_pending_acks;  
 
     ompi_free_list_t elan_events_free;/**< free events */
     ompi_free_list_t elan_reqs_free;  /**< all elan requests */
 
     ompi_event_t elan_send_event;  /**< event structure for sends */
     ompi_event_t elan_recv_event;  /**< event structure for recvs */
-    ompi_list_t  elan_procs;       /**< elan proc's */
+
+    struct mca_ptl_elan_proc_t *elan_local; 
+
     ompi_mutex_t elan_lock;        /**< lock for module state */
 };
 typedef struct mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module_1_0_0_t;

src/mca/ptl/elan/src/ptl_elan_frag.h

@@ -4,8 +4,8 @@
 /**
  * @file
  */
-#ifndef MCA_PTL_ELAN_FRAG_H
+#ifndef _MCA_PTL_ELAN_FRAG_H
-#define MCA_PTL_ELAN_FRAG_H
+#define _MCA_PTL_ELAN_FRAG_H
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -18,7 +18,7 @@
 #include "ptl_elan.h"
 
 extern ompi_class_t mca_ptl_elan_send_frag_t_class;
-struct mca_ptl_base_peer_t;
+struct mca_ptl_elan_peer_t;
 
 /**
  * ELAN send fragment derived type.
@@ -48,7 +48,7 @@ mca_ptl_elan_send_frag_handler (mca_ptl_elan_send_frag_t *, int sd);
 
 int 
 mca_ptl_elan_send_frag_init (mca_ptl_elan_send_frag_t *,
-                             struct mca_ptl_base_peer_t *,
+                             struct mca_ptl_elan_peer_t *,
                              struct mca_ptl_base_send_request_t *,
                              size_t offset, size_t * size, int flags);
 
@@ -71,7 +71,7 @@ mca_ptl_elan_recv_frag_handler (mca_ptl_elan_recv_frag_t *, int sd);
 
 void 
 mca_ptl_elan_recv_frag_init (mca_ptl_elan_recv_frag_t * frag,
-                              struct mca_ptl_base_peer_t *peer);
+                              struct mca_ptl_elan_peer_t *peer);
 
 bool 
 mca_ptl_elan_recv_frag_send_ack (mca_ptl_elan_recv_frag_t * frag); 
@@ -124,7 +124,7 @@ mca_ptl_elan_send_frag_progress (mca_ptl_elan_send_frag_t * frag)
 static inline void
 mca_ptl_elan_send_frag_init_ack (mca_ptl_elan_send_frag_t * ack,
                                  struct mca_ptl_t *ptl,
-                                 struct mca_ptl_base_peer_t *ptl_peer,
+                                 struct mca_ptl_elan_peer_t *ptl_peer,
                                  mca_ptl_elan_recv_frag_t * frag)
 {
     return;

src/mca/ptl/elan/src/ptl_elan_module.c

@@ -98,13 +98,12 @@ mca_ptl_elan_module_open (void)
     mca_ptl_elan.super.ptl_exclusivity =
         mca_ptl_elan_param_register_int ("exclusivity", 0);
     mca_ptl_elan.super.ptl_first_frag_size =
-        mca_ptl_elan_param_register_int ("first_frag_size", 1984/*magic*/);
+        mca_ptl_elan_param_register_int ("first_frag_size", 2048/*magic*/);
     mca_ptl_elan.super.ptl_min_frag_size =
         mca_ptl_elan_param_register_int ("min_frag_size", 320);
     mca_ptl_elan.super.ptl_max_frag_size =
         mca_ptl_elan_param_register_int ("max_frag_size", -1);
 
-#if 0 /* These parameters not very useful */
     /* register ELAN module parameters */
     mca_ptl_elan_module.elan_free_list_num =
         mca_ptl_elan_param_register_int ("free_list_num", 64);
@@ -117,11 +116,19 @@ mca_ptl_elan_module_open (void)
     mca_ptl_elan_module.elan_state = NULL;
     mca_ptl_elan_module.elan_ptls = NULL;
     mca_ptl_elan_module.elan_num_ptls = 0;
+    mca_ptl_elan_module.elan_local = NULL; 
 
     /* initialize objects */
-    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_lock, ompi_mutex_t);
+    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_reqs, ompi_list_t);
+    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_prog_events, ompi_list_t);
+    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_comp_events, ompi_list_t);
     OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_procs, ompi_list_t);
-#endif
+    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_pending_acks, ompi_list_t);
+
+    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_events_free, ompi_free_list_t);
+    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_reqs, ompi_free_list_t);
+
+    OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_lock, ompi_mutex_t);
 
     return OMPI_SUCCESS;
 }
@@ -130,15 +137,17 @@ mca_ptl_elan_module_open (void)
 int
 mca_ptl_elan_module_close (void)
 {
-    if (mca_ptl_elan_module.elan_reqs_free.fl_num_allocated !=
+#ifdef CLEAN_ELAN_MEM
-        mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length) {
+    elan_state_close(mca_ptl_elan_module.elan_state);
-        ompi_output (0, "elan requests: %d allocated %d returned\n",
+    elan_ptls_close(mca_ptl_elan_module.elan_ptls, 
-                     mca_ptl_elan_module.elan_reqs_free.fl_num_allocated,
+            mca_ptl_elan_module.elan_num_ptls);
-                     mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length);
+    elan_localproc_close(mca_ptl_elan_module.elan_local);
-    }
+#else
-
+    /* Free the dynamic memory for aggregated variables */
-    if (mca_ptl_elan_module.elan_events_free.fl_num_allocated !=
+    if ( mca_ptl_elan_module.elan_state ) {
-        mca_ptl_elan_module.elan_events_free.super.ompi_list_length) {
+        /* Make sure elan_state is the actual pointer,
+         * Avoid duplicated free memory calls */
+        free(mca_ptl_elan_module.elan_state);
     }
 
     if (NULL != mca_ptl_elan_module.elan_ptls) {
@@ -149,17 +158,40 @@ mca_ptl_elan_module_close (void)
         free (mca_ptl_elan_module.elan_ptls);
     }
 
+    /* TODO: Make sure this is not just an alias pointer */
+    if ( mca_ptl_elan_module.elan_local) {
+        free(mca_ptl_elan_module.elan_local);
+    }
+#endif
+
+    /* Check whether all the entries are return to the free list */
+    if (mca_ptl_elan_module.elan_reqs_free.fl_num_allocated !=
+        mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length) {
+        ompi_output (0, "elan requests: %d allocated %d returned\n",
+                     mca_ptl_elan_module.elan_reqs_free.fl_num_allocated,
+                     mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length);
+    }
+
+    if (mca_ptl_elan_module.elan_events_free.fl_num_allocated !=
+        mca_ptl_elan_module.elan_events_free.super.ompi_list_length) {
+        ompi_output (0, "elan events: %d allocated %d returned\n",
+                     mca_ptl_elan_module.elan_reqs_free.fl_num_allocated,
+                     mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length);
+    }
+
+
     /* Free the empty list holders */
     OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_reqs));
     OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_prog_events));
     OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_comp_events));
     OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_procs));
-    OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_lock));
+    OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_pending_acks));
 
-    /* TODO: check if this is needed. 
+    /* Destruct the free lists */
-     *       Free the free lists */
-    OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_reqs_free));
     OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_events_free));
+    OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_reqs_free));
+
+    OBJ_DESTRUCT (&mca_ptl_elan_module.elan_lock);
 
     return OMPI_SUCCESS;
 }
@@ -182,25 +214,28 @@ mca_ptl_elan_module_init (int *num_ptls,
     *allow_multi_user_threads = true;
     *have_hidden_threads = OMPI_HAVE_THREADS;
 
+    /* need to set ompi_using_threads() as ompi_event_init() 
+     * will spawn a thread if supported */
+    if(OMPI_HAVE_THREADS)
+        ompi_set_using_threads(true);
+
+    /* duplicated actions are avoid with a static variable, 
+     * inited in ompi_event_init() */
     if ((rc = ompi_event_init ()) != OMPI_SUCCESS) {
-        ompi_output (0,
+        ompi_output (0, "mca_ptl_elan_module_init: "
-                     "mca_ptl_elan_module_init: "
                      "unable to initialize event dispatch thread: %d\n",
                      rc);
         return NULL;
     }
 
     /* initialize free lists */
-#if 1
     ompi_free_list_init (&(mca_ptl_elan_module.elan_reqs_free),
                          sizeof (mca_ptl_elan_send_request_t),
                          OBJ_CLASS (mca_ptl_elan_send_request_t),
                          mca_ptl_elan_module.elan_free_list_num,
                          mca_ptl_elan_module.elan_free_list_max,
                          mca_ptl_elan_module.elan_free_list_inc, NULL);
-#endif
 
-    /* use default allocator */
     ompi_free_list_init (&mca_ptl_elan_module.elan_events_free,
                          sizeof (mca_ptl_elan_send_frag_t),
                          OBJ_CLASS (mca_ptl_elan_send_frag_t),
@@ -208,6 +243,8 @@ mca_ptl_elan_module_init (int *num_ptls,
                          mca_ptl_elan_module.elan_free_list_max,
                          mca_ptl_elan_module.elan_free_list_inc, NULL);
 
+    /* use default allocator */
+
     /* open basic elan device */
     if (OMPI_SUCCESS != ompi_mca_ptl_elan_init(&mca_ptl_elan_module)) {
         return NULL;
@@ -236,8 +273,13 @@ mca_ptl_elan_module_init (int *num_ptls,
         return NULL;
     }
 
+    /* FIXME: 
+     * Why use memcopy to create two instances of the same 
+     * structures, do they need to be defined them as constants,
+     * or coherency on two replicas will be a potential problem  */
     memcpy (ptls, mca_ptl_elan_module.elan_ptls,
             mca_ptl_elan_module.elan_num_ptls * sizeof (mca_ptl_elan_t *));
+
     *num_ptls = mca_ptl_elan_module.elan_num_ptls;
 
     return ptls;

src/mca/ptl/elan/src/ptl_elan_priv.h

@@ -42,8 +42,6 @@
 #include "elan/sys/misc_sys.h"
 #include "elan/sys/init_sys.h"
 
-int elan4_block_inputter (ELAN4_CTX *ctx, unsigned blocked);
-
 struct ompi_elan_railtable_t {
     int        rt_nrails;  
     int        rt_rail;   
@@ -73,14 +71,22 @@ struct ompi_elan_rail_t {
 };
 typedef struct ompi_elan_rail_t ompi_elan_rail_t;
 
-
 /* Initialization and finalization routines */
 int ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t *mp);
 int ompi_mca_ptl_elan_setup (mca_ptl_elan_module_1_0_0_t *mp);
 int ompi_mca_ptl_elan_fin (mca_ptl_elan_module_1_0_0_t *mp);
+/* Accessory functions to deallocate the memory */
+void elan_state_close(struct mca_ptl_elan_state_t *);
+void elan_ptls_close(struct mca_ptl_elan_t **, size_t elan_num_ptls);
+void elan_localproc_close(struct mca_ptl_elan_proc_t *); 
 
 /* communication prototypes */
 
 /* control, synchronization and state prototypes */
 
+/* Just to get rid of a warning from elan4 libraies,
+ * Many more needed but who cares. */
+int elan4_block_inputter (ELAN4_CTX *ctx, unsigned blocked);
+
+
 #endif

src/mca/ptl/elan/src/ptl_elan_req.h

@@ -11,13 +11,24 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+
 #include "ompi_config.h"
+#include "class/ompi_object.h"
+
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "mca/base/mca_base_param.h"
+#include "mca/base/mca_base_module_exchange.h"
+
 #include "mca/ptl/base/ptl_base_sendreq.h"
 #include "mca/ptl/base/ptl_base_recvreq.h"
+#include "ptl_elan.h"
 #include "ptl_elan_frag.h"
 
-/* Again, what is this! */
+OBJ_CLASS_DECLARATION(mca_ptl_elan_send_request_t);
- extern ompi_class_t mca_ptl_elan_send_request_t_class;
+OBJ_CLASS_DECLARATION(mca_ptl_elan_recv_request_t);
+/*extern ompi_class_t mca_ptl_elan_send_request_t_class;*/
+/*extern ompi_class_t mca_ptl_elan_recv_request_t_class;*/
 
 /**
  * ELAN send request derived type. The send request contains both the
@@ -30,6 +41,5 @@ struct mca_ptl_elan_send_request_t {
    mca_ptl_elan_send_frag_t req_frag; /* first fragment */
 };
 typedef struct mca_ptl_elan_send_request_t mca_ptl_elan_send_request_t;
-
 #endif