Paul Hargrove has pointed out that some big SMP systems (e.g., from SGI) configure Torque differently - instead of listing each node name once/slot in the nodefile, they list the node only once and set an envar to indicate the number of procs/node being allocated. Add an MCA param users can set to indicate we are in such an environment, and then use the envar to set the slots. Error out if the mode flag is given, but (a) we don't find the PBS_PPN envar, or (b) we find a node actually listed more than once in the PBS_Nodefile.

cmr=v1.7.5:reviewer=jsquyres:subject=Support SMP mode in Torque

This commit was SVN r30568.

orte/mca/ras/tm/help-ras-tm.txt

@@ -12,6 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2006      Sun Microsystems, Inc.  All rights reserved.
 #                         Use is subject to license terms.
+# Copyright (c) 2014      Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -25,3 +26,14 @@ No nodes were found in the PBS_NODEFILE:
 
 This is probably a setup issue, could be due to a corrupted file, or
 - most probably - is caused by having failed to obtain an allocation.
+#
+[smp-error]
+The SMP mode flag was given, but the required PBS_PPN environmental
+variable was not found. Please check that your system actually supports
+the indicated mode and try again.
+#
+[smp-multi]
+The SMP mode flag was given, but a node was listed in the PBS_Nodefile
+more than once. This usually indicates that the system is not
+actually operating in SMP mode. Please check your system configuration
+and try again.

orte/mca/ras/tm/ras_tm.h

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -29,14 +30,15 @@
 
 BEGIN_C_DECLS
 
-    struct orte_ras_tm_component_t {
-        orte_ras_base_component_t super;
-        char *nodefile_dir;
-    };
-    typedef struct orte_ras_tm_component_t orte_ras_tm_component_t;
+struct orte_ras_tm_component_t {
+    orte_ras_base_component_t super;
+    char *nodefile_dir;
+    bool smp_mode;
+};
+typedef struct orte_ras_tm_component_t orte_ras_tm_component_t;
     
-    ORTE_DECLSPEC extern orte_ras_tm_component_t mca_ras_tm_component;
-    ORTE_DECLSPEC extern orte_ras_base_module_t orte_ras_tm_module;
+ORTE_DECLSPEC extern orte_ras_tm_component_t mca_ras_tm_component;
+ORTE_DECLSPEC extern orte_ras_base_module_t orte_ras_tm_module;
 
 END_C_DECLS
 

orte/mca/ras/tm/ras_tm_component.c

@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -99,6 +100,21 @@ static int ras_tm_register(void)
                                             MCA_BASE_VAR_SCOPE_READONLY,
                                             &mca_ras_tm_component.nodefile_dir);
 
+    /* for big SMP machines (e.g., those from SGI), listing the nodes
+     * once/slot in the nodefile is extreme. In those cases, they may
+     * choose to list each node once, but then provide an envar that
+     * tells us how many cpus/node were allocated. Allow the user to
+     * inform us that we are in such an environment
+     */
+    mca_ras_tm_component.smp_mode = false;
+    (void) mca_base_component_var_register (c, "smp",
+                                            "The Torque system is configured in SMP mode "
+                                            "with the number of cpus/node given in the environment",
+                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                            OPAL_INFO_LVL_9,
+                                            MCA_BASE_VAR_SCOPE_READONLY,
+                                            &mca_ras_tm_component.smp_mode);
+
     return ORTE_SUCCESS;
 }
 

orte/mca/ras/tm/ras_tm_module.c

@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -124,7 +125,8 @@ static int discover(opal_list_t* nodelist, char *pbs_jobid)
     orte_node_t *node;
     opal_list_item_t* item;
     FILE *fp;
-    char *hostname;
+    char *hostname, *cppn;
+    int ppn;
 
     /* Ignore anything that the user already specified -- we're
        getting nodes only from TM. */
@@ -136,6 +138,19 @@ static int discover(opal_list_t* nodelist, char *pbs_jobid)
        slightly inefficient, but no big deal); just mentioned for
        completeness... */
 
+    /* if we are in SMP mode, then read the environment to get the
+     * number of cpus for each node read in the file
+     */
+    if (mca_ras_tm_component.smp_mode) {
+        if (NULL == (cppn = getenv("PBS_PPN"))) {
+            orte_show_help("help-ras-tm.txt", "smp-error", true);
+            return ORTE_ERR_NOT_FOUND;
+        }
+        ppn = strtol(cppn, NULL, 10);
+    } else {
+        ppn = 1;
+    }
+
     /* setup the full path to the PBS file */
     filename = opal_os_path(false, mca_ras_tm_component.nodefile_dir,
                             pbs_jobid, NULL);
@@ -166,6 +181,11 @@ static int discover(opal_list_t* nodelist, char *pbs_jobid)
              item = opal_list_get_next(item)) {
             node = (orte_node_t*) item;
             if (0 == strcmp(node->name, hostname)) {
+                if (mca_ras_tm_component.smp_mode) {
+                    /* this cannot happen in smp mode */
+                    orte_show_help("help-ras-tm.txt", "smp-multi", true);
+                    return ORTE_ERR_BAD_PARAM;
+                }
                 ++node->slots;
 
                 OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
@@ -191,7 +211,7 @@ static int discover(opal_list_t* nodelist, char *pbs_jobid)
             node->launch_id = nodeid;
             node->slots_inuse = 0;
             node->slots_max = 0;
-            node->slots = 1;
+            node->slots = ppn;
             opal_list_append(nodelist, &node->super);
         } else {