Convert a few opal_output() calls to instead use orte_show_help() as well as do some minor cosmetic changes dealing with tab spacing and c-blocks being enclosed with \{\}. There was also a long standing bug with the PSM mtl if the number of hardware contexts on adapter were less than the number of cores on a node (The default case is they are the same hence no issues were reported). For completeness we take care of this case as well but it requires us to tell PSM how many local processes are running on a node and the local rank of the process on a node so it can allocate the available hardware contexts appropriately.

This commit was SVN r21745.

ompi/mca/mtl/psm/Makefile.am

@@ -20,6 +20,8 @@ EXTRA_DIST = post_configure.sh
 
 AM_CPPFLAGS = $(mtl_psm_CPPFLAGS)
 
+dist_pkgdata_DATA = help-mtl-psm.txt
+
 mtl_psm_sources = \
 	       mtl_psm.c \
 	       mtl_psm.h \

ompi/mca/mtl/psm/help-mtl-psm.txt

@@ -0,0 +1,40 @@
+# -*- text -*-
+#
+# Copyright (C) 2009. QLogic Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+[psm init]
+Initialization of PSM library failed.
+
+  Error: %s
+#
+[debug level]
+Unable to set PSM debug level.
+
+  Error: %s
+#
+[unable to open endpoint]
+PSM was unable to open an endpoint. Please make sure that the network link is
+active on the node and the hardware is functioning. 
+
+  Error: %s
+#
+[no uuid present]
+Error obtaining unique transport key from ORTE (orte_precondition_transports %s
+the environment).
+
+  Local host: %s
+#
+[error polling network]
+Error %s occurred in attempting to make network progress (psm_mq_ipeek).
+#
+[error posting receive]
+Unable to post application receive buffer (psm_mq_irecv).
+
+  Error:      %s
+  Buffer:     %p
+  Length:     %d

ompi/mca/mtl/psm/mtl_psm.c

@@ -19,6 +19,8 @@
 
 #include "ompi_config.h"
 
+#include "orte/util/show_help.h"
+#include "orte/util/proc_info.h"
 #include "ompi/mca/mtl/mtl.h"
 #include "ompi/runtime/ompi_module_exchange.h"
 #include "ompi/mca/mtl/base/mtl_base_datatype.h"
@@ -62,7 +64,9 @@ ompi_mtl_psm_errhandler(psm_ep_t ep, const psm_error_t error,
 	case PSM_EP_NO_PORTS_AVAIL:
 	case PSM_EP_NO_NETWORK:
 	case PSM_EP_INVALID_UUID_KEY:
-	    opal_output(0, "Open MPI failed to open a PSM endpoint: %s\n", error_string);
+	  orte_show_help("help-mtl-psm.txt",
+			 "unable to open endpoint", true,
+			 psm_error_get_string(error));
 	    break;
 
 	/* We can't handle any other errors than the ones above */
@@ -77,7 +81,7 @@ ompi_mtl_psm_errhandler(psm_ep_t ep, const psm_error_t error,
 
 int ompi_mtl_psm_progress( void );
 
-int ompi_mtl_psm_module_init() { 
+int ompi_mtl_psm_module_init(int local_rank, int num_local_procs) { 
     psm_error_t err;
     psm_ep_t	ep; /* endpoint handle */
     psm_mq_t	mq;
@@ -86,6 +90,7 @@ int ompi_mtl_psm_module_init() {
     struct psm_ep_open_opts ep_opt;
     unsigned long long *uu = (unsigned long long *) unique_job_key;
     char *generated_key;
+    char env_string[256];
     
     generated_key = getenv("OMPI_MCA_orte_precondition_transports");
     memset(uu, 0, sizeof(psm_uuid_t));
@@ -93,10 +98,10 @@ int ompi_mtl_psm_module_init() {
     if (!generated_key || (strlen(generated_key) != 33) ||
         sscanf(generated_key, "%016llx-%016llx", &uu[0], &uu[1]) != 2)
     {
-        opal_output(0, "Error obtaining unique transport key from ORTE "
+      orte_show_help("help-mtl-psm.txt",
-                       "(orte_precondition_transpots %s the environment)\n", 
+		     "no uuid present", true,
 		     generated_key ? "could not be parsed from" :
-		       "not present in");
+		     "not present in", orte_process_info.nodename);
       return OMPI_ERROR;
       
     }
@@ -104,6 +109,15 @@ int ompi_mtl_psm_module_init() {
     /* Handle our own errors for opening endpoints */
     psm_error_register_handler(ompi_mtl_psm.ep, ompi_mtl_psm_errhandler);
     
+    /* Setup MPI_LOCALRANKID and MPI_LOCALNRANKS so PSM can allocate hardware
+     * contexts correctly.
+     */
+    snprintf(env_string, sizeof(env_string), "%d", local_rank);
+    setenv("MPI_LOCALRANKID", env_string, 0);
+    snprintf(env_string, sizeof(env_string), "%d", num_local_procs);
+    setenv("MPI_LOCALNRANKS", env_string, 0);
+    
+    /* Setup the endpoint options. */
     bzero((void*) &ep_opt, sizeof(ep_opt));
     ep_opt.timeout = ompi_mtl_psm.connect_timeout * 1e9;
     ep_opt.unit = ompi_mtl_psm.ib_unit;
@@ -121,7 +135,8 @@ int ompi_mtl_psm_module_init() {
     /* Open PSM endpoint */
     err = psm_ep_open(unique_job_key, &ep_opt, &ep, &epid);
     if (err) {
-        opal_output(0, "Error in psm_ep_open (error %s)\n", 
+      orte_show_help("help-mtl-psm.txt",
+		     "unable to open endpoint", true,
 		     psm_error_get_string(err));
       return OMPI_ERROR;
     }
@@ -135,7 +150,8 @@ int ompi_mtl_psm_module_init() {
 		      0,
 		      &mq);
     if (err) {
-        opal_output(0, "Error in psm_mq_init (error %s)\n", 
+      orte_show_help("help-mtl-psm.txt",
+		     "psm init", true,
 		     psm_error_get_string(err));
       return OMPI_ERROR;
     }
@@ -228,35 +244,39 @@ ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t *mtl,
     psm_error_t  *errs_out = NULL, err;
     size_t size;
     int proc_errors[PSM_ERROR_LAST] = { 0 };
-    int timeout_in_secs;
+    int proc, my_local_rank = -1, num_local_procs = 0, timeout_in_secs;
     
     assert(mtl == &ompi_mtl_psm.super);
     rc = OMPI_ERR_OUT_OF_RESOURCE;
 
     errs_out = (psm_error_t *) malloc(nprocs * sizeof(psm_error_t));
-    if (errs_out == NULL)
+    if (errs_out == NULL) {
 	goto bail;
+    }
     epids_in = (psm_epid_t *) malloc(nprocs * sizeof(psm_epid_t));
-    if (epids_in == NULL)
+    if (epids_in == NULL) {
 	goto bail;
+    }
     epaddrs_out = (psm_epaddr_t *) malloc(nprocs * sizeof(psm_epaddr_t));
-    if (epaddrs_out == NULL)
+    if (epaddrs_out == NULL) {
 	goto bail;
-
+    }
     rc = OMPI_SUCCESS;
 
     /* Get the epids for all the processes from modex */
     for (i = 0; i < (int) nprocs; i++) {
 	rc = ompi_modex_recv(&mca_mtl_psm_component.super.mtl_version, 
 				     procs[i], (void**)&epid, &size);
-	if (rc != OMPI_SUCCESS || size != sizeof(psm_epid_t))
+	if (rc != OMPI_SUCCESS || size != sizeof(psm_epid_t)) {
 	  return OMPI_ERROR;
+	}
 	epids_in[i] = *epid;
     }
 
     timeout_in_secs = min(180, 0.5 * nprocs);
-    if (ompi_mtl_psm.connect_timeout  < timeout_in_secs) 
+    if (ompi_mtl_psm.connect_timeout  < timeout_in_secs)  {
 	timeout_in_secs = ompi_mtl_psm.connect_timeout;
+    }
 
     psm_error_register_handler(ompi_mtl_psm.ep, PSM_ERRHANDLER_NOP);
 
@@ -281,9 +301,10 @@ ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t *mtl,
 		opal_output(0, "PSM EP connect error (%s):", 
 			    errstr ? errstr : "unknown connect error");
 		for (j = 0; j < (int) nprocs; j++) {
-		    if (errs_out[j] == thiserr)
+		  if (errs_out[j] == thiserr) {
 		    opal_output(0, " %s", procs[j]->proc_hostname);
 		  }
+		}
 		opal_output(0, "\n");
 	    }
 	}
@@ -308,12 +329,16 @@ ompi_mtl_psm_add_procs(struct mca_mtl_base_module_t *mtl,
     }
     
 bail:
-    if (epids_in != NULL)
+    if (epids_in != NULL) {
 	free(epids_in);
-    if (errs_out != NULL)
+    }
+    if (errs_out != NULL) {
 	free(errs_out);
-    if (epaddrs_out != NULL) 
+    }
+    if (epaddrs_out != NULL) {
 	free(epaddrs_out);
+    }
+
     return rc;
 }
 
@@ -336,16 +361,18 @@ int ompi_mtl_psm_progress( void ) {
 
     do {
         err = psm_mq_ipeek(ompi_mtl_psm.mq, &req, NULL);
-	if (err == PSM_MQ_INCOMPLETE)
+	if (err == PSM_MQ_INCOMPLETE) {
 	    return completed;
-	else if (err != PSM_OK)
+	} else if (err != PSM_OK) {
 	    goto error;
+	}
 	
 	completed++;
 
 	err = psm_mq_test(&req, &psm_status);
-	if (err != PSM_OK)
+	if (err != PSM_OK) {
 	    goto error;
+	}
 
         mtl_psm_request = (mca_mtl_psm_request_t*) psm_status.context;
 
@@ -363,9 +390,10 @@ int ompi_mtl_psm_progress( void ) {
 	}
 	
 	if(mtl_psm_request->type == OMPI_MTL_PSM_ISEND) { 
-	  if (mtl_psm_request->free_after) 
+	  if (mtl_psm_request->free_after) {
 	    free(mtl_psm_request->buf);
 	  }
+	}
 
 	switch (psm_status.error_code) {
 	    case PSM_OK:
@@ -387,7 +415,8 @@ int ompi_mtl_psm_progress( void ) {
     while (1);
 
  error: 
-    opal_output(0, "Error in psm progress function: %s\n", 
+    orte_show_help("help-mtl-psm.txt",
+		   "error polling network", true,
 		   psm_error_get_string(err));
     return 1;
 }

ompi/mca/mtl/psm/mtl_psm.h

@@ -81,7 +81,7 @@ extern int ompi_mtl_psm_cancel(struct mca_mtl_base_module_t* mtl,
     
 extern int ompi_mtl_psm_finalize(struct mca_mtl_base_module_t* mtl);
 
-int ompi_mtl_psm_module_init(void);
+int ompi_mtl_psm_module_init(int local_rank, int num_local_procs);
     
 
    

ompi/mca/mtl/psm/mtl_psm_component.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2006      QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006-2009 QLogic Corporation. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,9 +19,11 @@
 
 #include "ompi_config.h"
 
+#include "orte/util/show_help.h"
 #include "opal/event/event.h"
 #include "opal/util/output.h"
 #include "opal/mca/base/mca_base_param.h"
+#include "ompi/proc/proc.h"
 
 #include "mtl_psm.h"
 #include "mtl_psm_types.h"
@@ -101,10 +103,11 @@ ompi_mtl_psm_component_open(void)
 			   false, false, 0x7fffUL, 
 			   &ompi_mtl_psm.ib_pkey);
   
-  if (ompi_mtl_psm.ib_service_level < 0) 
+    if (ompi_mtl_psm.ib_service_level < 0)  {
       ompi_mtl_psm.ib_service_level = 0;
-  else if (ompi_mtl_psm.ib_service_level > 15)
+    } else if (ompi_mtl_psm.ib_service_level > 15) {
       ompi_mtl_psm.ib_service_level = 15;
+    }
   
   return OMPI_SUCCESS;
     
@@ -123,8 +126,39 @@ ompi_mtl_psm_component_init(bool enable_progress_threads,
                            bool enable_mpi_threads)
 {
     psm_error_t	err;
+    int rc;
     int	verno_major = PSM_VERNO_MAJOR;
     int verno_minor = PSM_VERNO_MINOR;
+    ompi_proc_t *my_proc, **procs;
+    size_t num_total_procs;
+    int local_rank = -1, num_local_procs = 0, proc;
+    
+    /* Compute the total number of processes on this host and our local rank
+     * on that node. We need to provide PSM with these values so it can 
+     * allocate hardware contexts appropriately across processes.
+     */
+    if ((rc = ompi_proc_refresh()) != OMPI_SUCCESS) {
+      return NULL;
+    }
+    
+    my_proc = ompi_proc_local();
+    if (NULL == (procs = ompi_proc_world(&num_total_procs))) {
+      return NULL;
+    }
+    
+    for (proc = 0; proc < num_total_procs; proc++) {
+      if (my_proc == procs[proc]) {
+	local_rank = num_local_procs++;
+	continue;
+      }
+      
+      if (OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) {
+	num_local_procs++;
+      }
+    }
+    
+    assert(local_rank >= 0 && num_local_procs > 0);
+    free(procs);
     
     err = psm_error_register_handler(NULL /* no ep */,
 			             PSM_ERRHANDLER_NOP);
@@ -141,7 +175,8 @@ ompi_mtl_psm_component_init(bool enable_progress_threads,
 		     sizeof(unsigned));
     if (err) {
       /* Non fatal error. Can continue */
-      opal_output(0, "Unable to set infinipath debug level (error %s)\n",
+      orte_show_help("help-mtl-psm.txt",
+		     "psm init", false,
 		     psm_error_get_string(err));
     }
 #endif
@@ -149,22 +184,24 @@ ompi_mtl_psm_component_init(bool enable_progress_threads,
     /* Only allow for shm and ipath devices in 2.0 and earlier releases 
      * (unless the user overrides the setting).
      */
+    
+    if (PSM_VERNO >= 0x0104) {
+      setenv("PSM_DEVICES", "self,shm,ipath", 0);
+    }
+    else {
       setenv("PSM_DEVICES", "shm,ipath", 0);
+    }
     
     err = psm_init(&verno_major, &verno_minor);
     if (err) {
-        opal_output(0, "Error in psm_init (error %s)\n", 
+      orte_show_help("help-mtl-psm.txt",
+		     "psm init", true,
 		     psm_error_get_string(err));
       return NULL;
     }
     
-    /*
+    /* Complete PSM initialization */
-     * Enable 'self' device only in a post-2.0 release(s)
+    ompi_mtl_psm_module_init(local_rank, num_local_procs);
-     */
-    if (PSM_VERNO >= 0x0104)
-      setenv("PSM_DEVICES", "self,shm,ipath", 0);
-
-    ompi_mtl_psm_module_init();
 
     ompi_mtl_psm.super.mtl_request_size = 
       sizeof(mca_mtl_psm_request_t) - 

ompi/mca/mtl/psm/mtl_psm_recv.c

@@ -68,7 +68,10 @@ ompi_mtl_psm_irecv(struct mca_mtl_base_module_t* mtl,
 		       &mtl_psm_request->psm_request);
     
     if (err) {
-        opal_output(0, "Error in psm_mq_irecv (error %s)\n", psm_error_get_string(err));
+      orte_show_help("help-mtl-psm.txt",
+		     "error posting receive", true,
+		     psm_error_get_string(err),
+		     mtl_psm_request->buf, length);
       return OMPI_ERROR;
     }
 

ompi/mca/mtl/psm/mtl_psm_send.c

@@ -70,8 +70,9 @@ ompi_mtl_psm_send(struct mca_mtl_base_module_t* mtl,
 		      mtl_psm_request.buf,
 		      length);
 
-    if (mtl_psm_request.free_after)
+    if (mtl_psm_request.free_after) {
 	free(mtl_psm_request.buf);
+    }
 
     return err == PSM_OK ? OMPI_SUCCESS : OMPI_ERROR;
 }