Add a new "generic" ess module that supports direct launched MPI jobs so long as certain minimum envars are provided, regardless of launch environment.

This commit was SVN r23478.

orte/mca/ess/generic/Makefile.am

@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+EXTRA_DIST = .windows
+
+sources = \
+        ess_generic.h \
+        ess_generic_component.c \
+        ess_generic_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_ess_generic_DSO
+component_noinst =
+component_install = mca_ess_generic.la
+else
+component_noinst = libmca_ess_generic.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_ess_generic_la_SOURCES = $(sources)
+mca_ess_generic_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_ess_generic_la_SOURCES =$(sources)
+libmca_ess_generic_la_LDFLAGS = -module -avoid-version

orte/mca/ess/generic/configure.m4

@@ -0,0 +1,13 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2007      Sandia National Laboratories. All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# MCA_ess_generic_CONFIG([action-if-found], [action-if-not-found])
+# -----------------------------------------------------------
+AC_DEFUN([MCA_ess_generic_CONFIG], [$1])

orte/mca/ess/generic/configure.params

@@ -0,0 +1,27 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+#                         reserved. 
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+PARAM_CONFIG_FILES="Makefile"
+#
+# Set the config priority so that we build
+# whenever someone using daemons does
+
+PARAM_CONFIG_PRIORITY=10

orte/mca/ess/generic/ess_generic.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef ORTE_ESS_GENERIC_H
+#define ORTE_ESS_GENERIC_H
+
+BEGIN_C_DECLS
+
+/*
+ * Module open / close
+ */
+int orte_ess_generic_component_open(void);
+int orte_ess_generic_component_close(void);
+int orte_ess_generic_component_query(mca_base_module_t **module, int *priority);
+
+
+ORTE_MODULE_DECLSPEC extern orte_ess_base_component_t mca_ess_generic_component;
+
+END_C_DECLS
+
+#endif /* ORTE_ESS_GENERIC_H */

orte/mca/ess/generic/ess_generic_component.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "orte_config.h"
+#include "orte/constants.h"
+
+#include "opal/mca/base/mca_base_param.h"
+
+#include "orte/util/proc_info.h"
+
+#include "orte/mca/ess/ess.h"
+#include "orte/mca/ess/generic/ess_generic.h"
+
+extern orte_ess_base_module_t orte_ess_generic_module;
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+orte_ess_base_component_t mca_ess_generic_component = {
+    {
+        ORTE_ESS_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "generic",
+        ORTE_MAJOR_VERSION,
+        ORTE_MINOR_VERSION,
+        ORTE_RELEASE_VERSION,
+
+        /* Component open and close functions */
+        orte_ess_generic_component_open,
+        orte_ess_generic_component_close,
+        orte_ess_generic_component_query
+    },
+    {
+        /* The component is checkpoint ready */
+        MCA_BASE_METADATA_PARAM_CHECKPOINT
+    }
+};
+
+
+int
+orte_ess_generic_component_open(void)
+{
+    return ORTE_SUCCESS;
+}
+
+int orte_ess_generic_component_query(mca_base_module_t **module, int *priority)
+{
+    char *pick;
+
+    /* only pick us if directed to do so */
+    if (NULL != (pick = getenv("OMPI_MCA_env")) &&
+                 0 == strcmp(pick, "generic")) {
+        *priority = 1000;
+        *module = (mca_base_module_t *)&orte_ess_generic_module;
+        return ORTE_SUCCESS;
+    }
+    
+    /* if not, then return NULL - we cannot be selected */
+    *priority = -1;
+    *module = NULL;
+    return ORTE_ERROR;
+}
+
+
+int
+orte_ess_generic_component_close(void)
+{
+    return ORTE_SUCCESS;
+}
+

orte/mca/ess/generic/ess_generic_module.c

@@ -0,0 +1,399 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2009 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ */
+
+#include "orte_config.h"
+#include "orte/constants.h"
+
+#include <sys/types.h>
+#include <stdio.h>
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdlib.h>
+
+#include "opal/event/event.h"
+#include "opal/runtime/opal.h"
+
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "opal/mca/paffinity/paffinity.h"
+#include "opal/util/output.h"
+#include "opal/util/opal_sos.h"
+#include "opal/util/malloc.h"
+#include "opal/util/argv.h"
+
+#include "orte/util/show_help.h"
+#include "orte/mca/rml/base/base.h"
+#include "orte/mca/rml/rml_types.h"
+#include "orte/mca/routed/base/base.h"
+#include "orte/mca/routed/routed.h"
+#include "orte/mca/errmgr/base/base.h"
+#include "orte/mca/grpcomm/base/base.h"
+#include "orte/mca/iof/base/base.h"
+#include "orte/mca/ess/base/base.h"
+#include "orte/mca/ess/ess.h"
+#include "orte/mca/ras/base/base.h"
+#include "orte/mca/plm/base/base.h"
+
+#include "orte/mca/rmaps/base/base.h"
+#if OPAL_ENABLE_FT_CR == 1
+#include "orte/mca/snapc/base/base.h"
+#endif
+#include "orte/mca/filem/base/base.h"
+#include "orte/util/proc_info.h"
+#include "orte/util/session_dir.h"
+#include "orte/util/name_fns.h"
+#include "orte/util/nidmap.h"
+
+#include "orte/runtime/runtime.h"
+#include "orte/runtime/orte_wait.h"
+#include "orte/runtime/orte_globals.h"
+
+#include "orte/runtime/orte_cr.h"
+#include "orte/mca/ess/ess.h"
+#include "orte/mca/ess/base/base.h"
+#include "orte/mca/ess/generic/ess_generic.h"
+
+static int rte_init(void);
+static int rte_finalize(void);
+static uint8_t proc_get_locality(orte_process_name_t *proc);
+static orte_vpid_t proc_get_daemon(orte_process_name_t *proc);
+static char* proc_get_hostname(orte_process_name_t *proc);
+static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc);
+static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc);
+static int update_pidmap(opal_byte_object_t *bo);
+static int update_nidmap(opal_byte_object_t *bo);
+
+orte_ess_base_module_t orte_ess_generic_module = {
+    rte_init,
+    rte_finalize,
+    orte_ess_base_app_abort,
+    proc_get_locality,
+    proc_get_daemon,
+    proc_get_hostname,
+    proc_get_local_rank,
+    proc_get_node_rank,
+    update_pidmap,
+    update_nidmap,
+    orte_ess_base_query_sys_info,
+    NULL
+};
+
+static int rte_init(void)
+{
+    int ret;
+    char *error = NULL;
+    char **nodes = NULL;
+    char *envar;
+    int32_t jobfam;
+    int i, j, ppn;
+    orte_nid_t *node;
+    orte_jmap_t *jmap;
+    orte_pmap_t *pmap;
+    orte_vpid_t vpid;
+
+    /* run the prolog */
+    if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
+        error = "orte_ess_base_std_prolog";
+        goto error;
+    }
+    
+    /* Only application procs can use this module. Since we
+     * were directly launched by someone, we need to bootstrap
+     * our own global info so we can startup.
+     */
+    
+    /* declare ourselves to be standalone - i.e., not launched by orted */
+    orte_standalone_operation = true;
+    
+    /* extract a jobid from the environment - can be totally
+     * arbitrary. if one isn't provided, just fake it
+     */
+    if (NULL != (envar = getenv("OMPI_MCA_orte_jobid"))) {
+        jobfam = strtol(envar, NULL, 10);
+    } else {
+        jobfam = 1;
+    }
+    ORTE_PROC_MY_NAME->jobid = ORTE_CONSTRUCT_LOCAL_JOBID(0, jobfam);
+    
+    /* extract a rank from the environment */
+    if (NULL == (envar = getenv("OMPI_MCA_orte_rank"))) {
+        error = "could not get process rank";
+        goto error;
+    }
+    ORTE_PROC_MY_NAME->vpid = strtol(envar, NULL, 10);
+
+    OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output,
+                         "%s completed name definition",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
+
+    /* get the number of procs in this job */
+    if (NULL == (envar = getenv("OMPI_MCA_orte_num_procs"))) {
+        error = "could not get number of processes in job";
+        goto error;
+    }
+    orte_process_info.num_procs = strtol(envar, NULL, 10);
+    
+    /* set the app_num so that MPI attributes get set correctly */
+    orte_process_info.app_num = 1;
+
+    /* get the list of nodes */
+    if (NULL == (envar = getenv("OMPI_MCA_orte_nodes"))) {
+        error = "could not get list of nodes";
+        goto error;
+    }
+    /* break this down */
+    nodes = opal_argv_split(envar, ',');
+    orte_process_info.num_nodes = opal_argv_count(nodes);
+
+    /* get the ppn */
+    if (NULL == (envar = getenv("OMPI_MCA_orte_ppn"))) {
+        error = "could not get ppn";
+        goto error;
+    }
+    ppn = strtol(envar, NULL, 10);
+
+    /* setup the nidmap arrays */
+    if (ORTE_SUCCESS != (ret = orte_util_nidmap_init(NULL))) {
+        ORTE_ERROR_LOG(ret);
+        error = "orte_util_nidmap_init";
+        goto error;
+    }
+    
+    /* set the size of the nidmap storage so we minimize realloc's */
+    if (ORTE_SUCCESS != (ret = opal_pointer_array_set_size(&orte_nidmap, orte_process_info.num_nodes))) {
+        error = "could not set pointer array size for nidmap";
+        goto error;
+    }
+    
+    /* create a job map for this job */
+    jmap = OBJ_NEW(orte_jmap_t);
+    jmap->job = ORTE_PROC_MY_NAME->jobid;
+    opal_pointer_array_add(&orte_jobmap, jmap);
+    /* update the num procs */
+    jmap->num_procs = orte_process_info.num_procs;
+    /* set the size of the pidmap storage so we minimize realloc's */
+    if (ORTE_SUCCESS != (ret = opal_pointer_array_set_size(&jmap->pmap, jmap->num_procs))) {
+        ORTE_ERROR_LOG(ret);
+        error = "could not set value array size for pidmap";
+        goto error;
+    }
+
+    /* construct the nidmap and pidmap */
+    vpid = 0;
+    for (i=0; NULL != nodes[i]; i++) {
+        node = OBJ_NEW(orte_nid_t);
+        node->name = strdup(nodes[i]);
+        node->daemon = i;
+        node->index = i;
+        opal_pointer_array_set_item(&orte_nidmap, i, node);
+        /* for each node, cycle through the ppn */
+        for (j=0; j < ppn; j++) {
+            pmap = OBJ_NEW(orte_pmap_t);
+            pmap->node = node->index;
+            pmap->local_rank = j;
+            pmap->node_rank = j;
+            if (ORTE_SUCCESS != (ret = opal_pointer_array_set_item(&jmap->pmap, vpid, pmap))) {
+                ORTE_ERROR_LOG(ret);
+                error = "could not set pmap values";
+                goto error;
+            }
+            OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output,
+                                 "%s node %d name %s rank %s",
+                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                                 (int) node->index, node->name, ORTE_VPID_PRINT(vpid)));
+            vpid++;
+        }
+    }
+    opal_argv_free(nodes);
+
+    /* ensure we pick the correct critical components */
+    putenv("OMPI_MCA_grpcomm=hier");
+    putenv("OMPI_MCA_routed=direct");
+
+    /* use the default procedure to finish my setup */
+    if (ORTE_SUCCESS != (ret = orte_ess_base_app_setup())) {
+        ORTE_ERROR_LOG(ret);
+        error = "orte_ess_base_app_setup";
+        goto error;
+    }
+
+    return ORTE_SUCCESS;
+
+ error:
+    orte_show_help("help-orte-runtime.txt",
+                   "orte_init:startup:internal-failure",
+                   true, error, ORTE_ERROR_NAME(ret), ret);
+    
+    return ret;
+}
+
+static int rte_finalize(void)
+{
+    int ret;
+    
+    /* use the default procedure to finish */
+    if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) {
+        ORTE_ERROR_LOG(ret);
+    }
+    
+    /* remove the envars that we pushed into environ
+     * so we leave that structure intact
+     */
+    unsetenv("OMPI_MCA_grpcomm");
+    unsetenv("OMPI_MCA_routed");
+
+    /* deconstruct my nidmap and jobmap arrays - this
+     * function protects itself from being called
+     * before things were initialized
+     */
+    orte_util_nidmap_finalize();
+
+    return ret;    
+}
+
+static uint8_t proc_get_locality(orte_process_name_t *proc)
+{
+    orte_nid_t *nid;
+    
+    if (NULL == (nid = orte_util_lookup_nid(proc))) {
+        ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
+        return OPAL_PROC_NON_LOCAL;
+    }
+    
+    if (nid->daemon == ORTE_PROC_MY_DAEMON->vpid) {
+        OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
+                             "%s ess:generic: proc %s on LOCAL NODE",
+                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                             ORTE_NAME_PRINT(proc)));
+        return (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
+    }
+
+    OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
+                         "%s ess:generic: proc %s is REMOTE",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_NAME_PRINT(proc)));
+    
+    return OPAL_PROC_NON_LOCAL;
+    
+}
+
+static orte_vpid_t proc_get_daemon(orte_process_name_t *proc)
+{
+    orte_nid_t *nid;
+
+    if( ORTE_JOBID_IS_DAEMON(proc->jobid) ) {
+        return proc->vpid;
+    }
+
+    if (NULL == (nid = orte_util_lookup_nid(proc))) {
+        return ORTE_VPID_INVALID;
+    }
+    
+    OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
+                         "%s ess:generic: proc %s is hosted by daemon %s",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_NAME_PRINT(proc),
+                         ORTE_VPID_PRINT(nid->daemon)));
+    
+    return nid->daemon;
+}
+
+static char* proc_get_hostname(orte_process_name_t *proc)
+{
+    orte_nid_t *nid;
+        
+    if (NULL == (nid = orte_util_lookup_nid(proc))) {
+        ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
+        return NULL;
+    }
+    
+    OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
+                         "%s ess:generic: proc %s is on host %s",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_NAME_PRINT(proc),
+                         nid->name));
+    
+    return nid->name;
+}
+
+static orte_local_rank_t proc_get_local_rank(orte_process_name_t *proc)
+{
+    orte_pmap_t *pmap;
+    
+    if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
+        ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
+        return ORTE_LOCAL_RANK_INVALID;
+    }    
+    
+    OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
+                         "%s ess:generic: proc %s has local rank %d",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_NAME_PRINT(proc),
+                         (int)pmap->local_rank));
+    
+    return pmap->local_rank;
+}
+
+static orte_node_rank_t proc_get_node_rank(orte_process_name_t *proc)
+{
+    orte_pmap_t *pmap;
+    
+    if (NULL == (pmap = orte_util_lookup_pmap(proc))) {
+        return ORTE_NODE_RANK_INVALID;
+    }    
+    
+    OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
+                         "%s ess:generic: proc %s has node rank %d",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_NAME_PRINT(proc),
+                         (int)pmap->node_rank));
+    
+    return pmap->node_rank;
+}
+
+static int update_pidmap(opal_byte_object_t *bo)
+{
+    int ret;
+    
+    OPAL_OUTPUT_VERBOSE((2, orte_ess_base_output,
+                         "%s ess:generic: updating pidmap",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
+    
+    /* build the pmap */
+    if (ORTE_SUCCESS != (ret = orte_util_decode_pidmap(bo))) {
+        ORTE_ERROR_LOG(ret);
+    }
+    
+    return ret;
+}
+
+static int update_nidmap(opal_byte_object_t *bo)
+{
+    int rc;
+    /* decode the nidmap - the util will know what to do */
+    if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
+        ORTE_ERROR_LOG(rc);
+    }    
+    return rc;
+}