Initial commit of QLogic PSM MTL.

This provides support for the Infinipath interconnect using the PSM API. 

Of note: 
This version has a "hackaround" we always return 1 or greater from
the MTL PSM progress function, this should be examined further.

This commit was SVN r11655.

acinclude.m4

@@ -67,6 +67,7 @@ m4_include(config/ompi_check_xcpu.m4)
 m4_include(config/ompi_check_mvapi.m4)
 m4_include(config/ompi_check_openib.m4)
 m4_include(config/ompi_check_portals.m4)
+m4_include(config/ompi_check_psm.m4)
 m4_include(config/ompi_check_udapl.m4)
 m4_include(config/ompi_check_package.m4)
 m4_include(config/ompi_check_slurm.m4)

config/ompi_check_psm.m4

@@ -0,0 +1,70 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2006 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006      QLogic Corp. All rights reserved.
+#
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# OMPI_CHECK_PSM(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if PSM support can be found.  sets prefix_{CPPFLAGS, 
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+AC_DEFUN([OMPI_CHECK_PSM],[
+    AC_ARG_WITH([psm],
+        [AC_HELP_STRING([--with-psm(=DIR)],
+             [Build PSM (QLogic InfiniPath PSM) support, searching for libraries in DIR])])
+    AC_ARG_WITH([psm-libdir],
+        [AC_HELP_STRING([--with-psm-libdir=DIR],
+             [Search for PSM (QLogic InfiniPath PSM) libraries in DIR])])
+
+    ompi_check_psm_$1_save_CPPFLAGS="$CPPFLAGS"
+    ompi_check_psm_$1_save_LDFLAGS="$LDFLAGS"
+    ompi_check_psm_$1_save_LIBS="$LIBS"
+
+    AS_IF([test "$with_psm" != "no"],
+          [AS_IF([test ! -z "$with_psm" -a "$with_psm" != "yes"],
+                 [ompi_check_psm_dir="$with_psm"])
+           AS_IF([test ! -z "$with_psm_libdir" -a "$with_psm_libdir" != "yes"],
+                 [ompi_check_psm_libdir="$with_psm_libdir"])
+
+           OMPI_CHECK_PACKAGE([$1],
+                              [psm.h],
+                              [psm_infinipath],
+                              [psm_finalize],
+			      [],
+                              [$ompi_check_psm_dir],
+                              [$ompi_check_psm_libdir],
+                              [ompi_check_psm_happy="yes"],
+                              [ompi_check_psm_happy="no"])],
+          [ompi_check_psm_happy="no"])
+
+    CPPFLAGS="$ompi_check_psm_$1_save_CPPFLAGS"
+    LDFLAGS="$ompi_check_psm_$1_save_LDFLAGS"
+    LIBS="$ompi_check_psm_$1_save_LIBS"
+
+    AS_IF([test "$ompi_check_psm_happy" = "yes" -a "$enable_progress_threads" = "yes"],
+          [AC_MSG_WARN([PSM driver does not currently support progress threads.  Disabling BTL.])
+           ompi_check_psm_happy="no"])
+
+    AS_IF([test "$ompi_check_psm_happy" = "yes"],
+          [$2],
+          [AS_IF([test ! -z "$with_psm" -a "$with_psm" != "no"],
+                 [AC_MSG_ERROR([PSM support requested but not found.  Aborting])])
+           $3])
+])
+

ompi/mca/mtl/psm/Makefile.am

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2006 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+EXTRA_DIST = post_configure.sh
+
+AM_CPPFLAGS = $(mtl_psm_CPPFLAGS)
+
+mtl_psm_sources = \
+	       mtl_psm.c \
+	       mtl_psm.h \
+	       mtl_psm_cancel.c \
+	       mtl_psm_component.c \
+	       mtl_psm_endpoint.c \
+	       mtl_psm_endpoint.h \
+	       mtl_psm_probe.c \
+	       mtl_psm_recv.c \
+	       mtl_psm_request.h \
+	       mtl_psm_send.c \
+	       mtl_psm_types.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_mtl_psm_DSO
+component_noinst =
+component_install = mca_mtl_psm.la
+else
+component_noinst = libmca_mtl_psm.la
+component_install =
+endif
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_mtl_psm_la_SOURCES = $(mtl_psm_sources)
+mca_mtl_psm_la_LIBADD = \
+    $(mtl_psm_LIBS) \
+    $(top_ompi_builddir)/ompi/libmpi.la \
+    $(top_ompi_builddir)/orte/liborte.la \
+    $(top_ompi_builddir)/opal/libopal.la
+mca_mtl_psm_la_LDFLAGS = -module -avoid-version $(mtl_psm_LDFLAGS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_mtl_psm_la_SOURCES = $(mtl_psm_sources)
+libmca_mtl_psm_la_LIBADD = $(mtl_psm_LIBS)
+libmca_mtl_psm_la_LDFLAGS = -module -avoid-version $(mtl_psm_LDFLAGS)

ompi/mca/mtl/psm/configure.m4

@@ -0,0 +1,41 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+# MCA_mtl_psm_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_mtl_psm_CONFIG],[
+    OMPI_CHECK_PSM([mtl_psm],
+                     [mtl_psm_happy="yes"],
+                     [mtl_psm_happy="no"])
+
+    AS_IF([test "$mtl_psm_happy" = "yes"],
+          [mtl_psm_WRAPPER_EXTRA_LDFLAGS="$mtl_psm_LDFLAGS"
+           mtl_psm_WRAPPER_EXTRA_LIBS="$mtl_psm_LIBS"
+           $1],
+          [$2])
+
+    # substitute in the things needed to build psm
+    AC_SUBST([mtl_psm_CFLAGS])
+    AC_SUBST([mtl_psm_CPPFLAGS])
+    AC_SUBST([mtl_psm_LDFLAGS])
+    AC_SUBST([mtl_psm_LIBS])
+])dnl
+

ompi/mca/mtl/psm/configure.params

@@ -0,0 +1,21 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+PARAM_CONFIG_FILES="Makefile"

ompi/mca/mtl/psm/mtl_psm.c

@@ -0,0 +1,382 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mca/mtl/mtl.h"
+#include "ompi/communicator/communicator.h"
+#include "opal/class/opal_list.h"
+#include "ompi/mca/pml/base/pml_base_module_exchange.h"
+#include "ompi/mca/mtl/base/mtl_base_datatype.h"
+
+#include "mtl_psm.h"
+#include "mtl_psm_types.h"
+#include "mtl_psm_endpoint.h"
+#include "mtl_psm_request.h"
+
+mca_mtl_psm_module_t ompi_mtl_psm = {
+    {
+        8191,         /* max cid - 2^13 - 1 */
+        (1UL << 30),  /* max tag value - must allow negatives */
+        0,            /* request reserve space */
+        0,            /* flags */
+        
+        ompi_mtl_psm_add_procs,
+        ompi_mtl_psm_del_procs,
+        ompi_mtl_psm_finalize,
+        
+        ompi_mtl_psm_send,
+        ompi_mtl_psm_isend,
+        
+        ompi_mtl_psm_irecv,
+        ompi_mtl_psm_iprobe,
+        
+        ompi_mtl_psm_cancel
+    }    
+};
+
+static
+psm_error_t
+ompi_mtl_psm_errhandler(psm_ep_t ep, const psm_error_t error, 
+			const char *error_string, psm_error_token_t token)
+{
+    switch (error) {
+	/* We don't want PSM to default to exiting when the following errors occur */
+	case PSM_EP_DEVICE_FAILURE:
+	case PSM_EP_NO_DEVICE:
+	case PSM_EP_NO_PORTS_AVAIL:
+	case PSM_EP_NO_NETWORK:
+	case PSM_EP_INVALID_UUID_KEY:
+	    opal_output(0, "Open MPI failed to open a PSM endpoint: %s\n", error_string);
+	    break;
+
+	/* We can't handle any other errors than the ones above */
+	default:
+	    opal_output(0, "Open MPI detected an unexpected PSM error in opening "
+			"an endpoint: %s\n", error_string);
+	    return psm_error_defer(token);
+	    break;
+    }
+    return error;
+}
+
+int ompi_mtl_psm_progress( void );
+
+int ompi_mtl_psm_module_init() { 
+    psm_error_t err;
+    psm_ep_t	ep; /* endpoint handle */
+    psm_mq_t	mq;
+    psm_epid_t	epid; /* unique lid+port identifier */
+    psm_uuid_t  unique_job_key;
+    uint64_t	*uu = (uint64_t *) unique_job_key;
+    char *generated_key;
+    
+    generated_key = getenv("OMPI_MCA_orte_precondition_transports");
+    
+    if (!generated_key || (strlen(generated_key) != 33) ||
+        sscanf(generated_key, "%016x-%016x", &uu[0], &uu[1]) != 2)
+    {
+        opal_output(0, "Error obtaining unique transport key from ORTE "
+                       "(orte_precondition_transpots %s the environment)\n", 
+			generated_key ? "could not be parsed from" :
+		       "not present in");
+        return OMPI_ERROR;
+    
+    }
+
+
+    /* Handle our own errors for opening endpoints */
+    psm_error_register_handler(ompi_mtl_psm.ep, ompi_mtl_psm_errhandler);
+         
+    err = psm_ep_open(unique_job_key, NULL, &ep, &epid);
+    if (err) {
+        opal_output(0, "Error in psm_ep_open (error %s)\n", 
+		    psm_error_get_string(err));
+        return OMPI_ERROR;
+    }
+
+    /* Future errors are handled by the default error handler */
+    psm_error_register_handler(ompi_mtl_psm.ep, PSM_ERRHANDLER_DEFAULT);
+    
+    err = psm_mq_init(ep, 
+		      0xffff000000000000ULL, 
+		      NULL,
+		      0,
+		      &mq);
+    if (err) {
+        opal_output(0, "Error in psm_mq_init (error %s)\n", 
+		    psm_error_get_string(err));
+        return OMPI_ERROR;
+    }
+
+    ompi_mtl_psm.ep   = ep;
+    ompi_mtl_psm.epid = epid;
+    ompi_mtl_psm.mq   = mq;
+
+    if (OMPI_SUCCESS != 
+	mca_pml_base_modex_send( &mca_mtl_psm_component.super.mtl_version, 
+                             &ompi_mtl_psm.epid, 
+			     sizeof(psm_epid_t))) {
+	opal_output(0, "Open MPI couldn't send PSM epid to head node process"); 
+	return OMPI_ERROR;
+    }
+    
+    /* register the psm progress function */
+    opal_progress_register(ompi_mtl_psm_progress);
+        
+    return OMPI_SUCCESS;
+}
+
+int
+ompi_mtl_psm_finalize(struct mca_mtl_base_module_t* mtl) { 
+    psm_error_t err;
+
+    opal_progress_unregister(ompi_mtl_psm_progress);
+
+    /* free resources */
+    err = psm_mq_finalize(ompi_mtl_psm.mq);
+    if (err) {
+        opal_output(0, "Error in psm_mq_finalize (error %s)\n", 
+		    psm_error_get_string(err));
+        return OMPI_ERROR;
+    }
+
+    err = psm_ep_close(ompi_mtl_psm.ep, PSM_EP_CLOSE_GRACEFUL, 1*1e9);
+    if (err) {
+        opal_output(0, "Error in psm_ep_close (error %s)\n", 
+		    psm_error_get_string(err));
+        return OMPI_ERROR;
+    }
+
+    err = psm_finalize();
+    if (err) {
+        opal_output(0, "Error in psm_finalize (error %s)\n", 
+		    psm_error_get_string(err));
+        return OMPI_ERROR;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static
+const char *
+ompi_mtl_psm_connect_error_msg(psm_error_t err)
+{
+    switch (err) { /* See if we expect the error */
+	case PSM_EPID_UNREACHABLE:
+	case PSM_EPID_INVALID_NODE:
+	case PSM_EPID_INVALID_MTU:
+	case PSM_EPID_INVALID_UUID_KEY:
+	case PSM_EPID_INVALID_VERSION:
+	case PSM_EPID_INVALID_CONNECT:
+	    return psm_error_get_string(err);
+	    break;
+	case PSM_EPID_UNKNOWN:
+	    return "Connect status could not be determined "
+		   "because of other errors";
+	default:
+	    return NULL;
+    }
+}
+
+#ifndef min
+#  define min(a,b) ((a) < (b) ? (a) : (b))
+#endif
+
+int
+ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t *mtl,
+                      size_t nprocs,
+                      struct ompi_proc_t** procs, /*const*/
+                      struct mca_mtl_base_endpoint_t **mtl_peer_data)
+{
+    int i,j; 
+    int rc;
+    psm_epid_t   *epids_in = NULL;
+    psm_epid_t	 *epid;
+    psm_epaddr_t *epaddrs_out = NULL;
+    psm_error_t  *errs_out = NULL, err;
+    size_t size;
+    int proc_errors[PSM_ERROR_LAST] = { 0 };
+    int timeout_in_secs;
+    
+    assert(mtl == &ompi_mtl_psm.super);
+    rc = OMPI_ERR_OUT_OF_RESOURCE;
+
+    errs_out = (psm_error_t *) malloc(nprocs * sizeof(psm_error_t));
+    if (errs_out == NULL)
+	goto bail;
+    epids_in = (psm_epid_t *) malloc(nprocs * sizeof(psm_epid_t));
+    if (epids_in == NULL)
+	goto bail;
+    epaddrs_out = (psm_epaddr_t *) malloc(nprocs * sizeof(psm_epaddr_t));
+    if (epaddrs_out == NULL)
+	goto bail;
+
+    rc = OMPI_SUCCESS;
+
+    /* Get the epids for all the processes from modex */
+    for (i = 0; i < (int) nprocs; i++) {
+	rc = mca_pml_base_modex_recv(&mca_mtl_psm_component.super.mtl_version, 
+				     procs[i], (void**)&epid, &size);
+	if (rc != OMPI_SUCCESS || size != sizeof(psm_epid_t))
+	    return OMPI_ERROR;
+	epids_in[i] = *epid;
+#if 0
+	printf("... connecting to epid=%llu, lid=%d,port=%d\n",
+		(unsigned long long) epids_in[i],
+		(int) psm_epid_nid(epids_in[i]),
+		(int) psm_epid_port(epids_in[i]));
+#endif
+    }
+
+    timeout_in_secs = min(180, 0.5 * nprocs);
+    if (ompi_mtl_psm.connect_timeout  < timeout_in_secs) 
+	timeout_in_secs = ompi_mtl_psm.connect_timeout;
+
+    psm_error_register_handler(ompi_mtl_psm.ep, PSM_ERRHANDLER_NOP);
+
+    err = psm_ep_connect(ompi_mtl_psm.ep,
+			 nprocs,
+			 epids_in,
+			 NULL, /* connect all */
+			 errs_out,
+			 epaddrs_out,
+			 timeout_in_secs * 1e9);
+    if (err) {
+	char *errstr = (char *) ompi_mtl_psm_connect_error_msg(err);
+	char *opalerr = NULL;
+	if (errstr == NULL) {
+	    opal_output(0, "PSM returned unhandled/unknown connect error: %s\n",
+			psm_error_get_string(err));
+	    return OMPI_ERROR;
+	}
+	for (i = 0; i < (int) nprocs; i++) {
+	    psm_error_t thiserr = errs_out[i];
+	    errstr = (char *) ompi_mtl_psm_connect_error_msg(thiserr);
+	    if (proc_errors[thiserr] == 0) {
+		proc_errors[thiserr] = 1;
+		opal_output(0, "PSM EP connect error (%s):", 
+			    errstr ? errstr : "unknown connect error");
+		for (j = 0; j < (int) nprocs; j++) {
+		    if (errs_out[j] == thiserr)
+			opal_output(0, " %s", procs[j]->proc_hostname);
+		}
+		opal_output(0, "\n");
+	    }
+	}
+    }
+
+    /* Default error handling is enabled, errors will not be returned to user.
+     * PSM prints the error and the offending endpoint's hostname and exits
+     * with -1 */
+    psm_error_register_handler(ompi_mtl_psm.ep, PSM_ERRHANDLER_DEFAULT);
+		
+    /* Fill in endpoint data */
+    for (i = 0; i < (int) nprocs; i++) { 
+	mtl_peer_data[i] =
+	    (mca_mtl_psm_endpoint_t *) OBJ_NEW(mca_mtl_psm_endpoint_t);
+	mtl_peer_data[i]->peer_epid = epids_in[i];
+	mtl_peer_data[i]->peer_addr = epaddrs_out[i];
+    }
+    
+bail:
+    if (epids_in != NULL)
+	free(epids_in);
+    if (errs_out != NULL)
+	free(errs_out);
+    if (epaddrs_out != NULL) 
+	free(epaddrs_out);
+    return rc;
+}
+
+int
+ompi_mtl_psm_del_procs(struct mca_mtl_base_module_t *mtl,
+                      size_t nprocs,
+                      struct ompi_proc_t** procs, 
+                      struct mca_mtl_base_endpoint_t **mtl_peer_data)
+{
+    return OMPI_SUCCESS;
+}
+
+
+int ompi_mtl_psm_progress( void ) { 
+    psm_error_t err;
+    mca_mtl_psm_request_t* mtl_psm_request;
+    psm_mq_status_t psm_status;
+    psm_mq_req_t req;
+    int completed = 1;
+
+    do {
+        err = psm_mq_ipeek(ompi_mtl_psm.mq, &req, NULL);
+	if (err == PSM_MQ_INCOMPLETE)
+	    return completed;
+	else if (err != PSM_OK)
+	    goto error;
+
+	completed++;
+
+	err = psm_mq_test(&req, &psm_status);
+	if (err != PSM_OK)
+	    goto error;
+
+        mtl_psm_request = (mca_mtl_psm_request_t*) psm_status.context;
+
+	if (mtl_psm_request->type == OMPI_MTL_PSM_IRECV) {
+            ompi_mtl_datatype_unpack(mtl_psm_request->convertor, 
+                                     mtl_psm_request->buf,
+                                     psm_status.msg_length);
+
+	    mtl_psm_request->super.ompi_req->req_status.MPI_SOURCE =
+		    PSM_GET_MQRANK(psm_status.msg_tag);
+	    mtl_psm_request->super.ompi_req->req_status.MPI_TAG =
+		    PSM_GET_MQUTAG(psm_status.msg_tag);
+            mtl_psm_request->super.ompi_req->req_status._count = 
+                    psm_status.nbytes;
+	}
+	
+	if(mtl_psm_request->type == OMPI_MTL_PSM_ISEND) { 
+	  if (mtl_psm_request->free_after) 
+	    free(mtl_psm_request->buf);
+	}
+
+	switch (psm_status.error_code) {
+	    case PSM_OK:
+		mtl_psm_request->super.ompi_req->req_status.MPI_ERROR = 
+		    OMPI_SUCCESS;
+		break;
+	    case PSM_MQ_TRUNCATION:
+		mtl_psm_request->super.ompi_req->req_status.MPI_ERROR = 
+		    MPI_ERR_TRUNCATE;
+		break;
+	    default:
+		mtl_psm_request->super.ompi_req->req_status.MPI_ERROR = 
+                        MPI_ERR_INTERN;
+	}
+
+	mtl_psm_request->super.completion_callback(&mtl_psm_request->super);
+
+    }
+    while (1);
+
+ error: 
+    opal_output(0, "Error in psm progress function: %s\n", 
+		psm_error_get_string(err));
+    return 1;
+}
+

ompi/mca/mtl/psm/mtl_psm.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MTL_PSM_H_HAS_BEEN_INCLUDED
+#define MTL_PSM_H_HAS_BEEN_INCLUDED
+
+#include "opal/threads/threads.h"
+#include "opal/threads/condition.h"
+#include "ompi/class/ompi_free_list.h"
+#include "opal/util/cmd_line.h"
+#include "ompi/request/request.h"
+#include "ompi/mca/mtl/mtl.h"
+#include "ompi/mca/mtl/base/base.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/datatype/convertor.h"
+#include <psm.h>
+#include <psm_mq.h>
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/* MTL interface functions */
+extern int ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t* mtl, 
+                          size_t nprocs,
+                          struct ompi_proc_t** procs, 
+                          struct mca_mtl_base_endpoint_t **mtl_peer_data);
+    
+extern int ompi_mtl_psm_del_procs(struct mca_mtl_base_module_t* mtl, 
+                                 size_t nprocs,
+                                 struct ompi_proc_t** procs, 
+                                 struct mca_mtl_base_endpoint_t **mtl_peer_data);
+
+int
+ompi_mtl_psm_send(struct mca_mtl_base_module_t* mtl, 
+                 struct ompi_communicator_t* comm,
+                 int dest,
+                 int tag,
+                 struct ompi_convertor_t *convertor,
+                 mca_pml_base_send_mode_t mode);
+
+extern int ompi_mtl_psm_isend(struct mca_mtl_base_module_t* mtl, 
+                             struct ompi_communicator_t* comm,
+                             int dest,
+                             int tag,
+                             struct ompi_convertor_t *convertor,
+                             mca_pml_base_send_mode_t mode,
+                             bool blocking,
+                             mca_mtl_request_t * mtl_request);
+
+extern int ompi_mtl_psm_irecv(struct mca_mtl_base_module_t* mtl,
+                             struct ompi_communicator_t *comm,
+                             int src,
+                             int tag,
+                             struct ompi_convertor_t *convertor,
+                             struct mca_mtl_request_t *mtl_request);
+    
+    
+extern int ompi_mtl_psm_iprobe(struct mca_mtl_base_module_t* mtl, 
+                              struct ompi_communicator_t *comm,
+                              int src,
+                              int tag,
+                              int *flag,
+                              struct ompi_status_public_t *status);
+
+extern int ompi_mtl_psm_cancel(struct mca_mtl_base_module_t* mtl,
+                              struct mca_mtl_request_t *mtl_request, 
+                              int flag);
+    
+extern int ompi_mtl_psm_finalize(struct mca_mtl_base_module_t* mtl);
+
+int ompi_mtl_psm_module_init(void);
+    
+
+   
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif  /* MTL_PSM_H_HAS_BEEN_INCLUDED */
+

ompi/mca/mtl/psm/mtl_psm_cancel.c

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "ompi/request/request.h"
+#include "mtl_psm.h"
+#include "mtl_psm_request.h"
+
+int ompi_mtl_psm_cancel(struct mca_mtl_base_module_t* mtl, 
+                       struct mca_mtl_request_t *mtl_request, 
+                       int flag) {
+
+  psm_error_t err;
+  psm_mq_status_t status;
+  
+  mca_mtl_psm_request_t *mtl_psm_request = 
+    (mca_mtl_psm_request_t*) mtl_request; 
+  
+  err = psm_mq_cancel(&mtl_psm_request->psm_request); 
+  if(PSM_OK == err) { 
+    err = psm_mq_test(&mtl_psm_request->psm_request, &status);
+    if(PSM_OK == err) { 
+      mtl_request->ompi_req->req_status._cancelled = true;
+      mtl_psm_request->super.completion_callback(&mtl_psm_request->super);
+      return OMPI_SUCCESS;
+    } else { 
+      return OMPI_ERROR;
+    }
+  } else if(PSM_MQ_INCOMPLETE == err) { 
+    return OMPI_SUCCESS; 
+  } 
+  
+  return OMPI_ERROR;
+}

ompi/mca/mtl/psm/mtl_psm_component.c

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "opal/event/event.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "opal/util/output.h"
+#include "ompi/datatype/convertor.h"
+
+#include "mtl_psm.h"
+#include "mtl_psm_types.h"
+#include "mtl_psm_request.h"
+
+#include "psm.h"
+
+static int ompi_mtl_psm_component_open(void);
+static int ompi_mtl_psm_component_close(void);
+
+static mca_mtl_base_module_t* ompi_mtl_psm_component_init( bool enable_progress_threads, 
+                                                          bool enable_mpi_threads );
+
+mca_mtl_psm_component_t mca_mtl_psm_component = {
+
+    {
+        /* First, the mca_base_component_t struct containing meta
+         * information about the component itself */
+        
+        {
+            /* Indicate that we are a mtl v1.0.0 component (which also implies
+             *          a specific MCA version) */
+            
+            MCA_MTL_BASE_VERSION_1_0_0,
+            
+            "psm", /* MCA component name */
+            OMPI_MAJOR_VERSION,  /* MCA component major version */
+            OMPI_MINOR_VERSION,  /* MCA component minor version */
+            OMPI_RELEASE_VERSION,  /* MCA component release version */
+            ompi_mtl_psm_component_open,  /* component open */
+            ompi_mtl_psm_component_close  /* component close */
+        },
+        
+        /* Next the MCA v1.0.0 component meta data */
+        
+        {
+            /* Whether the component is checkpointable or not */
+            false
+        },
+        
+        ompi_mtl_psm_component_init  /* component init */
+    }
+};
+
+    
+static int
+ompi_mtl_psm_component_open(void)
+{
+    
+    mca_base_param_reg_int(&mca_mtl_psm_component.super.mtl_version, "connect_timeout",
+                           "PSM connection timeout value in seconds",
+                           false, false, 30, &ompi_mtl_psm.connect_timeout);
+
+    return OMPI_SUCCESS;
+    
+}
+
+
+static int
+ompi_mtl_psm_component_close(void)
+{
+    return OMPI_SUCCESS;
+}
+
+
+static mca_mtl_base_module_t*
+ompi_mtl_psm_component_init(bool enable_progress_threads,
+                           bool enable_mpi_threads)
+{
+    psm_error_t	err;
+    int	verno_major = PSM_VERNO_MAJOR;
+    int verno_minor = PSM_VERNO_MINOR;
+
+    err = psm_error_register_handler(NULL /* no ep */,
+			             PSM_ERRHANDLER_NOP);
+    if (err) {
+        opal_output(0, "Error in psm_error_register_handler (error %s)\n", 
+		    psm_error_get_string(err));
+	return NULL;
+    }
+
+    err = psm_init(&verno_major, &verno_minor);
+    if (err) {
+        opal_output(0, "Error in psm_init (error %s)\n", 
+		    psm_error_get_string(err));
+        return NULL;
+    }
+        
+    ompi_mtl_psm_module_init();
+    
+    ompi_mtl_psm.super.mtl_request_size = 
+        sizeof(mca_mtl_psm_request_t) - 
+        sizeof(struct mca_mtl_request_t);
+    
+    return &ompi_mtl_psm.super;
+}
+

ompi/mca/mtl/psm/mtl_psm_endpoint.c

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include <sys/time.h>
+#include <time.h>
+#include "ompi/types.h"
+#include "orte/mca/ns/base/base.h"
+#include "orte/mca/oob/base/base.h"
+#include "orte/mca/rml/rml.h"
+#include "orte/mca/errmgr/errmgr.h"
+#include "opal/util/output.h"
+#include "mtl_psm.h"
+#include "mtl_psm_types.h"
+#include "mtl_psm_endpoint.h" 
+#include "ompi/mca/pml/base/pml_base_module_exchange.h"
+
+/*
+ * Initialize state of the endpoint instance.
+ *
+ */
+
+static void mca_mtl_psm_endpoint_construct(mca_mtl_psm_endpoint_t* endpoint)
+{
+    endpoint->mtl_psm_module = NULL;
+}
+
+/*
+ * Destroy a endpoint
+ *
+ */
+
+static void mca_mtl_psm_endpoint_destruct(mca_mtl_psm_endpoint_t* endpoint)
+{
+}
+
+
+OBJ_CLASS_INSTANCE(
+    mca_mtl_psm_endpoint_t, 
+    opal_list_item_t, 
+    mca_mtl_psm_endpoint_construct, 
+    mca_mtl_psm_endpoint_destruct);
+

ompi/mca/mtl/psm/mtl_psm_endpoint.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef MCA_MTL_PSM_ENDPOINT_H
+#define MCA_MTL_PSM_ENDPOINT_H
+
+#include "opal/class/opal_list.h"
+#include "opal/event/event.h"
+#include "ompi/proc/proc.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/mtl/mtl.h"
+#include "mtl_psm.h"
+
+#include "psm.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+OBJ_CLASS_DECLARATION(mca_mtl_psm_endpoint_t);
+
+/**
+ * An abstraction that represents a connection to a endpoint process.
+ * An instance of mca_mtl_base_endpoint_t is associated w/ each process
+ * and MTL pair at startup. However, connections to the endpoint
+ * are established dynamically on an as-needed basis:
+ */
+
+struct mca_mtl_base_endpoint_t {
+    opal_list_item_t super;
+   
+    struct mca_mtl_psm_module_t* mtl_psm_module;
+    /**< MTL instance that created this connection */
+    
+    psm_epid_t	    peer_epid;
+    /**< The unique epid for the opened port */
+
+    psm_epaddr_t    peer_addr;
+    /**< The connected endpoint handle*/
+};
+
+typedef struct mca_mtl_base_endpoint_t mca_mtl_base_endpoint_t;
+typedef mca_mtl_base_endpoint_t  mca_mtl_psm_endpoint_t;
+OBJ_CLASS_DECLARATION(mca_mtl_psm_endpoint);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif

ompi/mca/mtl/psm/mtl_psm_probe.c

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "ompi/request/request.h"
+#include "mtl_psm.h"
+#include "mtl_psm_types.h"
+#include "psm.h"
+#include "ompi/communicator/communicator.h"
+
+
+int ompi_mtl_psm_iprobe(struct mca_mtl_base_module_t* mtl, 
+                              struct ompi_communicator_t *comm,
+                              int src,
+                              int tag,
+                              int *flag,
+                              struct ompi_status_public_t *status)
+{
+    uint64_t mqtag, tagsel;
+    psm_mq_status_t mqstat;
+    psm_error_t err;
+
+    PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
+
+    err = psm_mq_iprobe(ompi_mtl_psm.mq, mqtag, tagsel, &mqstat);
+    if (err == PSM_OK) {
+	*flag = 1;
+	status->MPI_SOURCE = PSM_GET_MQRANK(mqstat.msg_tag);
+	status->MPI_TAG = PSM_GET_MQUTAG(mqstat.msg_tag);
+        status->_count = mqstat.nbytes;
+
+	switch (mqstat.error_code) {
+	    case PSM_OK:
+		status->MPI_ERROR = OMPI_SUCCESS;
+		break;
+	    case PSM_MQ_TRUNCATION:
+		status->MPI_ERROR = MPI_ERR_TRUNCATE;
+		break;
+	    default:
+		status->MPI_ERROR = MPI_ERR_INTERN;
+	}
+	return OMPI_SUCCESS;
+    }
+    else if (err == PSM_MQ_INCOMPLETE) {
+	*flag = 0;
+	return OMPI_SUCCESS;
+    }
+    else
+	return OMPI_ERROR;
+}

ompi/mca/mtl/psm/mtl_psm_recv.c

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "ompi/request/request.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/datatype/convertor.h"
+#include "ompi/mca/mtl/base/mtl_base_datatype.h"
+
+#include "mtl_psm.h"
+#include "mtl_psm_types.h"
+#include "mtl_psm_request.h"
+
+int
+ompi_mtl_psm_irecv(struct mca_mtl_base_module_t* mtl,
+                  struct ompi_communicator_t *comm,
+                  int src,
+                  int tag,
+                  struct ompi_convertor_t *convertor,
+                  struct mca_mtl_request_t *mtl_request)
+{
+    int ret;    
+    psm_error_t err;
+    mca_mtl_psm_request_t * mtl_psm_request = (mca_mtl_psm_request_t*) mtl_request;
+    uint64_t mqtag;
+    uint64_t tagsel;
+    size_t length;
+    
+    ret = ompi_mtl_datatype_recv_buf(convertor,
+                                     &mtl_psm_request->buf,
+                                     &length, 
+                                     &mtl_psm_request->free_after);
+    
+    if (OMPI_SUCCESS != ret) return ret;
+
+    mtl_psm_request->length = length;
+    mtl_psm_request->convertor = convertor;
+    mtl_psm_request->type = OMPI_MTL_PSM_IRECV;
+
+    PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
+
+#if 0
+    printf("recv bits:   0x%016llx 0x%016llx\n", mqtag, tagsel);
+#endif
+    err = psm_mq_irecv(ompi_mtl_psm.mq, 
+		       mqtag,
+		       tagsel,
+		       0,
+		       mtl_psm_request->buf,
+		       length,
+		       mtl_psm_request,
+		       &mtl_psm_request->psm_request);
+    
+    if (err) {
+        opal_output(0, "Error in psm_mq_irecv (error %s)\n", psm_error_get_string(err));
+        return OMPI_ERROR;
+    }
+
+    return OMPI_SUCCESS;
+}
+

ompi/mca/mtl/psm/mtl_psm_request.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef OMPI_MTL_PSM_REQUEST_H
+#define OMPI_MTL_PSM_REQUEST_H
+
+#include "ompi/datatype/convertor.h"
+
+
+typedef enum {
+    OMPI_MTL_PSM_ISEND,
+    OMPI_MTL_PSM_IRECV
+} mca_mtl_psm_request_type_t;
+
+struct mca_mtl_psm_request_t { 
+    struct mca_mtl_request_t super;
+    mca_mtl_psm_request_type_t type;
+    psm_mq_req_t psm_request;
+    /* psm_segment_t psm_segment[1]; */
+    void *buf;
+    size_t length;
+    struct ompi_convertor_t *convertor;
+    bool free_after;
+}; 
+typedef struct mca_mtl_psm_request_t mca_mtl_psm_request_t;
+
+#endif

ompi/mca/mtl/psm/mtl_psm_send.c

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/datatype/convertor.h"
+
+#include "mtl_psm.h"
+#include "mtl_psm_types.h"
+#include "mtl_psm_request.h"
+#include "ompi/mca/mtl/base/mtl_base_datatype.h"
+
+int
+ompi_mtl_psm_send(struct mca_mtl_base_module_t* mtl, 
+                 struct ompi_communicator_t* comm,
+                 int dest,
+                 int tag,
+                 struct ompi_convertor_t *convertor,
+                 mca_pml_base_send_mode_t mode)
+{
+    psm_error_t err;
+    mca_mtl_psm_request_t mtl_psm_request;
+    uint64_t mqtag;
+    uint32_t flags = 0;
+    int ret;
+    size_t length;
+        
+    mca_mtl_psm_endpoint_t* psm_endpoint = 
+        (mca_mtl_psm_endpoint_t*) comm->c_pml_procs[dest]->proc_ompi->proc_pml;
+
+    assert(mtl == &ompi_mtl_psm.super);
+
+    mqtag = PSM_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag);
+    
+    ret = ompi_mtl_datatype_pack(convertor, 
+                                 &mtl_psm_request.buf,
+                                 &length, 
+                                 &mtl_psm_request.free_after);
+
+    
+    mtl_psm_request.length = length;
+    mtl_psm_request.convertor = convertor;
+    mtl_psm_request.type = OMPI_MTL_PSM_ISEND;
+
+    if (OMPI_SUCCESS != ret) return ret;
+
+    if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS)
+	flags |= PSM_MQ_FLAG_SENDSYNC;
+
+    err = psm_mq_send(ompi_mtl_psm.mq,
+		      psm_endpoint->peer_addr,
+		      flags,
+		      mqtag,
+		      mtl_psm_request.buf,
+		      length);
+#if 0
+        printf("send bits: 0x%016llx\n", mqtag);
+#endif
+
+    return err == PSM_OK ? OMPI_SUCCESS : OMPI_ERROR;
+}
+
+int
+ompi_mtl_psm_isend(struct mca_mtl_base_module_t* mtl, 
+                  struct ompi_communicator_t* comm,
+                  int dest,
+                  int tag,
+                  struct ompi_convertor_t *convertor,
+                  mca_pml_base_send_mode_t mode,
+                  bool blocking,
+                  mca_mtl_request_t * mtl_request)
+{
+    psm_error_t psm_error;
+    uint64_t mqtag;
+    uint32_t flags = 0;
+    int ret;
+    mca_mtl_psm_request_t * mtl_psm_request = (mca_mtl_psm_request_t*) mtl_request;
+    size_t length;
+
+    mca_mtl_psm_endpoint_t* psm_endpoint = 
+        (mca_mtl_psm_endpoint_t*) comm->c_pml_procs[dest]->proc_ompi->proc_pml;
+
+    assert(mtl == &ompi_mtl_psm.super);
+
+    mqtag = PSM_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag);
+
+    
+    ret = ompi_mtl_datatype_pack(convertor, 
+                                 &mtl_psm_request->buf,
+                                 &length, 
+                                 &mtl_psm_request->free_after);
+
+    mtl_psm_request->length= length;
+    mtl_psm_request->convertor = convertor;
+    mtl_psm_request->type = OMPI_MTL_PSM_ISEND;
+
+    if (OMPI_SUCCESS != ret) return ret;
+
+    if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS)
+	flags |= PSM_MQ_FLAG_SENDSYNC;
+    
+#if 0
+        printf("isend bits: 0x%016llx\n", mqtag);
+#endif
+
+    psm_error = psm_mq_isend(ompi_mtl_psm.mq,
+			     psm_endpoint->peer_addr,
+			     flags,
+			     mqtag,
+			     mtl_psm_request->buf,
+			     length,
+			     mtl_psm_request,
+			     &mtl_psm_request->psm_request);
+    
+    return psm_error == PSM_OK ? OMPI_SUCCESS : OMPI_ERROR;
+}

ompi/mca/mtl/psm/mtl_psm_types.h

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MTL_PSM_TYPES_H_HAS_BEEN_INCLUDED
+#define MTL_PSM_TYPS_H_HAS_BEEN_INCLUDED
+
+#include "mtl_psm.h"
+
+#include "opal/threads/threads.h"
+#include "opal/threads/condition.h"
+#include "ompi/class/ompi_free_list.h"
+#include "opal/util/cmd_line.h"
+#include "ompi/request/request.h"
+#include "ompi/mca/mtl/mtl.h"
+#include "ompi/mca/mtl/base/base.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/datatype/convertor.h"
+#include "mtl_psm_endpoint.h" 
+
+#include "psm.h"
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/** 
+ * MTL Module Interface
+ */
+struct mca_mtl_psm_module_t { 
+    mca_mtl_base_module_t super; /**< base MTL interface */
+
+    int32_t connect_timeout;
+
+    psm_ep_t	 ep;
+    psm_mq_t	 mq;
+    psm_epid_t	 epid;
+    psm_epaddr_t epaddr;
+}; 
+
+typedef struct mca_mtl_psm_module_t mca_mtl_psm_module_t;
+
+extern mca_mtl_psm_module_t ompi_mtl_psm;
+
+struct mca_mtl_psm_component_t{ 
+    mca_mtl_base_component_1_0_0_t          super;  /**< base MTL component */ 
+};
+typedef struct mca_mtl_psm_component_t mca_mtl_psm_component_t;
+
+extern mca_mtl_psm_component_t mca_mtl_psm_component;
+    
+#define PSM_MAKE_MQTAG(ctxt,rank,utag)		    \
+        ( (((ctxt)&0xffffULL)<<48)| (((rank)&0xffffULL)<<32)| \
+	  (((utag)&0xffffffffULL)) )
+
+#define PSM_GET_MQRANK(tag_u64)	((int)(((tag_u64)>>32)&0xffff))
+#define PSM_GET_MQUTAG(tag_u64)	((int)((tag_u64)&0xffffffffULL))
+
+#define PSM_MAKE_TAGSEL(user_rank, user_tag, user_ctxt, tag, tagsel)		\
+	do {									\
+	    if ((user_tag) == MPI_ANY_TAG) {					\
+		if ((user_rank) == MPI_ANY_SOURCE) {				\
+		    (tagsel) = PSM_MAKE_MQTAG(0xffff,0,0);			\
+		    (tag) = PSM_MAKE_MQTAG((user_ctxt),0,0);			\
+		}								\
+		else {								\
+		    (tagsel) = PSM_MAKE_MQTAG(0xffff,0xffff,0);			\
+		    (tag) = PSM_MAKE_MQTAG((user_ctxt),(user_rank),0);		\
+		}								\
+	    }									\
+	    else {								\
+		if ((user_rank) == MPI_ANY_SOURCE) {				\
+		    (tagsel) = PSM_MAKE_MQTAG(0xffff,0,0xffffffff);		\
+		    (tag) = PSM_MAKE_MQTAG((user_ctxt),0,(user_tag));		\
+		}								\
+		else {								\
+		    (tagsel) = PSM_MAKE_MQTAG(0xffff,0xffff,0xffffffff);	\
+		    (tag) = PSM_MAKE_MQTAG((user_ctxt),(user_rank),(user_tag));	\
+		}								\
+	    }									\
+	} while (0)
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif  /* MTL_PSM_TYPES_H_HAS_BEEN_INCLUDED */
+

ompi/mca/mtl/psm/post_configure.sh

@@ -0,0 +1 @@
+DIRECT_CALL_HEADER="ompi/mca/mtl/psm/mtl_psm.h"