* add all the run-time tweakable parameters

* start trying to figure out the sendfrag code
* start of code to initialize recv buffer

This commit was SVN r5569.

src/mca/ptl/portals/Makefile.am

@@ -37,10 +37,12 @@ EXTRA_DIST = \
 portals_SOURCES = \
 	ptl_portals.h \
 	ptl_portals_compat.h \
+	ptl_portals_sendfrag.h \
 	ptl_portals.c \
 	ptl_portals_component.c \
 	ptl_portals_stubs.c \
-	ptl_portals_compat_utcp.c
+	ptl_portals_compat_utcp.c \
+	ptl_portals_sendfrag.c
 
 
 mcacomponentdir = $(libdir)/openmpi

src/mca/ptl/portals/configure.stub

@@ -15,6 +15,37 @@
 # $HEADER$
 #
 
+#
+# quicky function to set #defines based on argument values
+#
+# ARGUMENTS:
+#  1    configure name (first argument to ARG_WITH, minus the ptl-portals-)
+#  2    define name
+#  3    default value
+#  4    description (used for both ARG_WITH and DEFINE)
+AC_DEFUN([MCA_PTL_PORTALS_CONFIG_VAL],
+[
+    AC_ARG_WITH([ptl-portals-$1], AC_HELP_STRING([--with-ptl-portals-$1], 
+                [$4 (default: $3)]))
+    AC_MSG_CHECKING([for $1 value])
+    case "[$with_]m4_bpatsubst([ptl-portals-$1], -, _)" in
+        "")
+            $2=$3
+            AC_MSG_RESULT([[$]$2 (default)])
+            ;;
+        "no")
+            AC_MSG_RESULT([error])
+            AC_MSG_ERROR([--without-ptl-portals-$1 is invalid argument])
+            ;;
+        *)
+            $2="$with_m4_bpatsubst([ptl-portals-$1], -, _)"
+            AC_MSG_RESULT([[$]$2])
+            ;;
+    esac
+    AC_DEFINE_UNQUOTED([$2], [[$]$2], [$4])
+])dnl
+
+
 #
 # Main function.  This will be invoked in the middle of the templated
 # configure script.
@@ -113,6 +144,46 @@ AC_DEFUN([MCA_CONFIGURE_STUB],[
                        [Use the Red Storm implementation or Portals])
     AM_CONDITIONAL([PTL_PORTALS_REDSTORM], [test "$PTL_PORTALS_REDSTORM" = "1"])
 
+    MCA_PTL_PORTALS_CONFIG_VAL([first-frag-table-id],
+                      [PTL_PORTALS_FIRST_FRAG_TABLE_ID], [1],
+                      [Portals table id to use for first fragment receive queue])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-table-id],
+                      [PTL_PORTALS_RNDV_FRAG_TABLE_ID], [2],
+                      [Portals table id to use for rndv fragment receive queue])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([ack-table-id],
+                      [PTL_PORTALS_ACK_TABLE_ID], [3],
+                      [Portals table id to use for ack queue])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([debug-level],
+                      [PTL_PORTALS_DEFAULT_DEBUG_LEVEL], [1000],
+                      [Default debugging level for portals ptl])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([request-cache-size],
+                      [PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE], [1],
+                      [Default request cache size for portals ptl])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([first-frag-size],
+                      [PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE], [16384],
+                      [Default first frag size for portals ptl])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([first-frag-num-entries],
+                      [PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES], [3],
+                      [Default number of memory descriptors for first fragments])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([first-frag-entry-size],
+                      [PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE], [1048576],
+                      [Default size of memory associeted with first fag md])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-min-size],
+                      [PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE], [0],
+                      [Default minimum size of rndv fragments])
+
+    MCA_PTL_PORTALS_CONFIG_VAL([rndv-frag-max-size],
+                      [PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE], [32768],
+                      [Default maximum size of rndv fragments])
+
     #
     # Save extra compiler/linker flags so that they can be added in
     # the wrapper compilers, if necessary

src/mca/ptl/portals/ptl_portals.c

@@ -27,17 +27,18 @@
 
 #include "ptl_portals.h"
 #include "ptl_portals_compat.h"
+#include "ptl_portals_sendfrag.h"
 
 mca_ptl_portals_module_t mca_ptl_portals_module = {
     {
         &mca_ptl_portals_component.super,
-        1,   /* max size of request cache */
+        0,   /* max size of request cache */
-        1,   /* byest required by ptl for a request */
+        sizeof(mca_ptl_portals_send_frag_t),   /* byes required by ptl for a request */
-        128, /* max size of first frag */
+        0,   /* max size of first frag */
         0,   /* min size of frag */
-        128, /* max size of frag */
+        0,   /* max size of frag */
         0,   /* exclusivity */
-        50,  /* latency */
+        0,   /* latency */
         0,   /* bandwidth */
         0,   /* ptl flags */
 
@@ -59,7 +60,9 @@ mca_ptl_portals_module_t mca_ptl_portals_module = {
         NULL   /* PML use */
     },
 
-    0,
+    0, /* num first frag mds */
+    0, /* size first frag md */
+    0, /* ni handle */
 };
 
 
@@ -75,6 +78,8 @@ mca_ptl_portals_add_procs(struct mca_ptl_base_module_t* ptl,
     struct ompi_proc_t *curr_proc;
     ptl_process_id_t *portals_procs;
     size_t i;
+    unsigned long distance;
+    struct mca_ptl_portals_module_t *myptl = (struct mca_ptl_portals_module_t*) ptl;
 
     /* make sure our environment is fully initialized.  At end of this
        call, we have a working network handle on our module and
@@ -90,7 +95,31 @@ mca_ptl_portals_add_procs(struct mca_ptl_base_module_t* ptl,
         /* BWB - do we want to send to self?  No for now */
         if (curr_proc == local_proc) continue;
 
+        /* make sure we can reach the process */
+        ret = PtlNIDist(myptl->ni_handle,
+                        portals_procs[i],
+                        &distance);
+        if (ret != PTL_OK) {
+            ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
+                                "Could not find distance to process %d", i);
+            continue;
+        }
+
+        /* set the peer as a pointer to the address */
+        peers[i] = (struct mca_ptl_base_peer_t*) &(portals_procs[i]);
+
+        /* and here we can reach */
+        ompi_bitmap_set_bit(reachable, i);
     }
 
     return OMPI_SUCCESS;
 }
+
+
+int
+mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
+                              int value)
+{
+    /* need to do all the portals cleanup code here... */
+    return OMPI_SUCCESS;
+}

src/mca/ptl/portals/ptl_portals.h

@@ -49,14 +49,19 @@ struct mca_ptl_portals_component_t {
      */
     int portals_output;
 
+#if PTL_PORTALS_UTCP
     /** ethernet interface to use - only has meaning with utcp
         reference */
     char *portals_ifname;
+#endif
 
     /** Number of currently active portals modules */
     uint32_t portals_num_modules;
-    /** List of currently running modules */
+    /** List of currently available modules */
     struct mca_ptl_portals_module_t **portals_modules;
+
+    /** lock for accessing component */
+    ompi_mutex_t portals_lock;
 };
 
 struct mca_ptl_portals_recv_frag_t;
@@ -142,6 +147,11 @@ extern int mca_ptl_portals_component_progress(
 struct mca_ptl_portals_module_t {
     mca_ptl_base_module_t super;         /**< base PTL module interface */
 
+    /* number of mds for first frags */
+    int first_frag_num_mds;
+    /* size of each md for first frags */
+    int first_frag_md_size;
+
     /** our portals network interface */
     ptl_handle_ni_t ni_handle;
     /** the limits returned from PtlNIInit for interface */
@@ -357,4 +367,7 @@ extern int mca_ptl_portals_send_continue(
 );
 
 
+extern int mca_ptl_portals_module_enable(struct mca_ptl_portals_module_t *ptl,
+                                         int value);
+
 #endif

src/mca/ptl/portals/ptl_portals_component.c

@@ -115,6 +115,7 @@ mca_ptl_portals_component_open(void)
     mca_ptl_portals_component.portals_modules = NULL;
 
     /* initialize objects */
+    OBJ_CONSTRUCT(&mca_ptl_portals_component.portals_lock, ompi_mutex_t);
 
     /* register portals module parameters */
 #if PTL_PORTALS_UTCP
@@ -122,7 +123,28 @@ mca_ptl_portals_component_open(void)
         mca_ptl_portals_param_register_string("ifname", "eth0");
 #endif
     portals_output_stream.lds_verbose_level = 
-        mca_ptl_portals_param_register_int("debug_level", 1000);
+        mca_ptl_portals_param_register_int("debug_level",
+                                           PTL_PORTALS_DEFAULT_DEBUG_LEVEL);
+
+    mca_ptl_portals_module.super.ptl_cache_size =
+        mca_ptl_portals_param_register_int("request_cache_size",
+                                           PTL_PORTALS_DEFAULT_REQUEST_CACHE_SIZE);
+    mca_ptl_portals_module.super.ptl_first_frag_size =
+        mca_ptl_portals_param_register_int("first_frag_size",
+                                           PTL_PORTALS_DEFAULT_FIRST_FRAG_SIZE);
+    mca_ptl_portals_module.super.ptl_min_frag_size =
+        mca_ptl_portals_param_register_int("rndv_frag_min_size",
+                                           PTL_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE);
+    mca_ptl_portals_module.super.ptl_max_frag_size =
+        mca_ptl_portals_param_register_int("rndv_frag_max_size",
+                                           PTL_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE);
+
+    mca_ptl_portals_module.first_frag_num_mds = 
+        mca_ptl_portals_param_register_int("first_frag_num_entries",
+                                           PTL_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES);
+    mca_ptl_portals_module.first_frag_md_size = 
+        mca_ptl_portals_param_register_int("first_frag_entry_size",
+                                           PTL_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE);
 
     /* finish with objects */
     mca_ptl_portals_component.portals_output = 
@@ -149,6 +171,8 @@ mca_ptl_portals_component_close(void)
     /* print out debugging if anything is pending */
 
     /* release resources */
+    OBJ_DESTRUCT(&mca_ptl_portals_component.portals_lock);
+
     if (NULL != mca_ptl_portals_component.portals_ifname) {
         free(mca_ptl_portals_component.portals_ifname);
     }
@@ -201,20 +225,27 @@ mca_ptl_portals_component_init(int *num_ptls,
 int
 mca_ptl_portals_component_control(int param, void* value, size_t size)
 {
+    uint32_t i;
+    int ret = OMPI_SUCCESS;
+
     ompi_output_verbose(100, mca_ptl_portals_component.portals_output,
-                        "mca_ptl_portals_component_control(%d)", param);
+                        "mca_ptl_portals_component_control(%d, %d)", 
+                        param, (*(int*) value));
 
     switch(param) {
         case MCA_PTL_ENABLE:
-            if(*(int*)value) {
+            OMPI_THREAD_LOCK(&mca_ptl_portals_component.portals_lock);
-                /* BWB - enable the ptl */
+            for (i = 0 ; i < mca_ptl_portals_component.portals_num_modules ; ++i) {
-            } else
+                ret = mca_ptl_portals_module_enable(mca_ptl_portals_component.portals_modules[i],
-                /* BWB - disable the ptl */
+                                                    *(int*)value);                  
+                if (ret != OMPI_SUCCESS) break;
+            }
+            OMPI_THREAD_UNLOCK(&mca_ptl_portals_component.portals_lock);
             break;
         default:
             break;
     }
-    return OMPI_SUCCESS;
+    return ret;
 }
 
 

src/mca/ptl/portals/ptl_portals_sendfrag.c

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "portals_config.h"
+
+#include "ptl_portals.h"
+#include "ptl_portals_compat.h"
+#include "ptl_portals_sendfrag.h"
+
+OBJ_CLASS_INSTANCE(mca_ptl_portals_send_frag_t,
+                   mca_ptl_base_send_frag_t,
+                   NULL, NULL);

src/mca/ptl/portals/ptl_portals_sendfrag.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#ifndef MCA_PTL_PORTALS_SENDFRAG_H_
+#define MCA_PTL_PORTALS_SENDFRAG_H_
+
+#include "mca/ptl/base/ptl_base_sendfrag.h"
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+    struct mca_ptl_portals_send_frag_t {
+        mca_ptl_base_send_frag_t frag_send;
+        ptl_md_t memory_descriptor; /* BWB - not sure if we need this */
+        ptl_handle_eq_t eq_handle;
+        int status;
+    };
+    typedef struct mca_ptl_portals_send_frag_t mca_ptl_portals_send_frag_t;
+
+    OBJ_CLASS_DECLARATION (mca_ptl_portals_send_frag_t);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+
+#endif /* MCA_PTL_PORTALS_SENDFRAG_H_ */