Merge pull request #3218 from rhc54/topic/pmix2

Update to include the PMIx 2.0 APIs for monitoring and job control.

opal/include/opal/constants.h

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -94,7 +94,9 @@ enum {
     OPAL_ERR_PROC_RESTART                   = (OPAL_ERR_BASE - 63),
     OPAL_ERR_PROC_CHECKPOINT                = (OPAL_ERR_BASE - 64),
     OPAL_ERR_PROC_MIGRATE                   = (OPAL_ERR_BASE - 65),
-    OPAL_ERR_EVENT_REGISTRATION             = (OPAL_ERR_BASE - 66)
+    OPAL_ERR_EVENT_REGISTRATION             = (OPAL_ERR_BASE - 66),
+    OPAL_ERR_HEARTBEAT_ALERT                = (OPAL_ERR_BASE - 67),
+    OPAL_ERR_FILE_ALERT                     = (OPAL_ERR_BASE - 68)
 };
 
 #define OPAL_ERR_MAX                (OPAL_ERR_BASE - 100)

opal/mca/pmix/ext2x/pmix2x.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
@@ -352,7 +352,7 @@ static void _event_hdlr(int sd, short args, void *cbdata)
     if (NULL != chain->final_cbfunc) {
         chain->final_cbfunc(PMIX_SUCCESS, chain->final_cbdata);
     }
-    
+
     OBJ_RELEASE(chain);
 
     return;

opal/mca/pmix/pmix2x/pmix/include/pmix.h

@@ -473,6 +473,59 @@ pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t directive,
                                          pmix_info_t *info, size_t ninfo,
                                          pmix_info_cbfunc_t cbfunc, void *cbdata);
 
+/* Request a job control action. The targets array identifies the
+ * processes to which the requested job control action is to be applied.
+ * A NULL value can be used to indicate all processes in the caller's
+ * nspace. The use of PMIX_RANK_WILDARD can also be used to indicate
+ * that all processes in the given nspace are to be included.
+ *
+ * The directives are provided as pmix_info_t structs in the directives
+ * array. The callback function provides a status to indicate whether or
+ * not the request was granted, and to provide some information as to
+ * the reason for any denial in the pmix_info_cbfunc_t array of pmix_info_t
+ * structures. If non-NULL, then the specified release_fn must be called
+ * when the callback function completes - this will be used to release
+ * any provided pmix_info_t array.
+ */
+pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_t ntargets,
+                                  const pmix_info_t directives[], size_t ndirs,
+                                  pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Request that something be monitored - e.g., that the server monitor
+ * this process for periodic heartbeats as an indication that the process
+ * has not become "wedged". When a monitor detects the specified alarm
+ * condition, it will generate an event notification using the provided
+ * error code and passing along any available relevant information. It is
+ * up to the caller to register a corresponding event handler.
+ *
+ * Params:
+ *
+ * monitor: attribute indicating the type of monitor being requested - e.g.,
+ *          PMIX_MONITOR_FILE to indicate that the requestor is asking that
+ *          a file be monitored.
+ *
+ * error: the status code to be used when generating an event notification
+ *        alerting that the monitor has been triggered. The range of the
+ *        notification defaults to PMIX_RANGE_NAMESPACE - this can be
+ *        changed by providing a PMIX_RANGE directive
+ *
+ * directives: characterize the monitoring request (e.g., monitor file size)
+ *             and frequency of checking to be done
+ *
+ * cbfunc: provides a status to indicate whether or not the request was granted,
+ *         and to provide some information as to the reason for any denial in
+ *         the pmix_info_cbfunc_t array of pmix_info_t structures.
+ *
+ * Note: a process can send a heartbeat to the server using the PMIx_Heartbeat
+ * macro provided below*/
+pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pmix_status_t error,
+                                      const pmix_info_t directives[], size_t ndirs,
+                                      pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* define a special macro to simplify sending of a heartbeat */
+#define PMIx_Heartbeat() \
+    PMIx_Process_monitor_nb(PMIX_SEND_HEARTBEAT, NULL, 0, NULL, NULL)
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

opal/mca/pmix/pmix2x/pmix/include/pmix_common.h

@@ -123,6 +123,8 @@ typedef uint32_t pmix_rank_t;
                                                                     //        a local system-level PMIx server
 #define PMIX_CONNECT_SYSTEM_FIRST           "pmix.cnct.sys.first"   // (bool) Preferentially look for a system-level PMIx server first
 #define PMIX_REGISTER_NODATA                "pmix.reg.nodata"       // (bool) Registration is for nspace only, do not copy job data
+#define PMIX_SERVER_ENABLE_MONITORING       "pmix.srv.monitor"      // (bool) Enable PMIx internal monitoring by server
+
 
 /* identification attributes */
 #define PMIX_USERID                         "pmix.euid"             // (uint32_t) effective user id
@@ -218,8 +220,9 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_COLLECTIVE_ALGO                "pmix.calgo"            // (char*) comma-delimited list of algorithms to use for collective
 #define PMIX_COLLECTIVE_ALGO_REQD           "pmix.calreqd"          // (bool) if true, indicates that the requested choice of algo is mandatory
 #define PMIX_NOTIFY_COMPLETION              "pmix.notecomp"         // (bool) notify parent process upon termination of child job
-#define PMIX_RANGE                          "pmix.range"            // (int) pmix_data_range_t value for calls to publish/lookup/unpublish
-#define PMIX_PERSISTENCE                    "pmix.persist"          // (int) pmix_persistence_t value for calls to publish
+#define PMIX_RANGE                          "pmix.range"            // (pmix_data_range_t) value for calls to publish/lookup/unpublish or for
+                                                                    //        monitoring event notifications
+#define PMIX_PERSISTENCE                    "pmix.persist"          // (pmix_persistence_t) value for calls to publish
 #define PMIX_OPTIONAL                       "pmix.optional"         // (bool) look only in the immediate data store for the requested value - do
                                                                     //        not request data from the server if not found
 #define PMIX_EMBED_BARRIER                  "pmix.embed.barrier"    // (bool) execute a blocking fence operation before executing the
@@ -259,66 +262,72 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_EVENT_ACTION_TIMEOUT           "pmix.evtimeout"        // (int) time in sec before RM will execute error response
 
 /* attributes used to describe "spawn" attributes */
-#define PMIX_PERSONALITY                    "pmix.pers"              // (char*) name of personality to use
-#define PMIX_HOST                           "pmix.host"              // (char*) comma-delimited list of hosts to use for spawned procs
-#define PMIX_HOSTFILE                       "pmix.hostfile"          // (char*) hostfile to use for spawned procs
-#define PMIX_ADD_HOST                       "pmix.addhost"           // (char*) comma-delimited list of hosts to add to allocation
-#define PMIX_ADD_HOSTFILE                   "pmix.addhostfile"       // (char*) hostfile to add to existing allocation
-#define PMIX_PREFIX                         "pmix.prefix"            // (char*) prefix to use for starting spawned procs
-#define PMIX_WDIR                           "pmix.wdir"              // (char*) working directory for spawned procs
-#define PMIX_MAPPER                         "pmix.mapper"            // (char*) mapper to use for placing spawned procs
-#define PMIX_DISPLAY_MAP                    "pmix.dispmap"           // (bool) display process map upon spawn
-#define PMIX_PPR                            "pmix.ppr"               // (char*) #procs to spawn on each identified resource
-#define PMIX_MAPBY                          "pmix.mapby"             // (char*) mapping policy
-#define PMIX_RANKBY                         "pmix.rankby"            // (char*) ranking policy
-#define PMIX_BINDTO                         "pmix.bindto"            // (char*) binding policy
-#define PMIX_PRELOAD_BIN                    "pmix.preloadbin"        // (bool) preload binaries
-#define PMIX_PRELOAD_FILES                  "pmix.preloadfiles"      // (char*) comma-delimited list of files to pre-position
-#define PMIX_NON_PMI                        "pmix.nonpmi"            // (bool) spawned procs will not call PMIx_Init
-#define PMIX_STDIN_TGT                      "pmix.stdin"             // (uint32_t) spawned proc rank that is to receive stdin
-#define PMIX_FWD_STDIN                      "pmix.fwd.stdin"         // (bool) forward my stdin to the designated proc
-#define PMIX_FWD_STDOUT                     "pmix.fwd.stdout"        // (bool) forward stdout from spawned procs to me
-#define PMIX_FWD_STDERR                     "pmix.fwd.stderr"        // (bool) forward stderr from spawned procs to me
-#define PMIX_DEBUGGER_DAEMONS               "pmix.debugger"          // (bool) spawned app consists of debugger daemons
-#define PMIX_COSPAWN_APP                    "pmix.cospawn"           // (bool) designated app is to be spawned as a disconnected
-                                                                     //     job - i.e., not part of the "comm_world" of the job
+#define PMIX_PERSONALITY                    "pmix.pers"             // (char*) name of personality to use
+#define PMIX_HOST                           "pmix.host"             // (char*) comma-delimited list of hosts to use for spawned procs
+#define PMIX_HOSTFILE                       "pmix.hostfile"         // (char*) hostfile to use for spawned procs
+#define PMIX_ADD_HOST                       "pmix.addhost"          // (char*) comma-delimited list of hosts to add to allocation
+#define PMIX_ADD_HOSTFILE                   "pmix.addhostfile"      // (char*) hostfile to add to existing allocation
+#define PMIX_PREFIX                         "pmix.prefix"           // (char*) prefix to use for starting spawned procs
+#define PMIX_WDIR                           "pmix.wdir"             // (char*) working directory for spawned procs
+#define PMIX_MAPPER                         "pmix.mapper"           // (char*) mapper to use for placing spawned procs
+#define PMIX_DISPLAY_MAP                    "pmix.dispmap"          // (bool) display process map upon spawn
+#define PMIX_PPR                            "pmix.ppr"              // (char*) #procs to spawn on each identified resource
+#define PMIX_MAPBY                          "pmix.mapby"            // (char*) mapping policy
+#define PMIX_RANKBY                         "pmix.rankby"           // (char*) ranking policy
+#define PMIX_BINDTO                         "pmix.bindto"           // (char*) binding policy
+#define PMIX_PRELOAD_BIN                    "pmix.preloadbin"       // (bool) preload binaries
+#define PMIX_PRELOAD_FILES                  "pmix.preloadfiles"     // (char*) comma-delimited list of files to pre-position
+#define PMIX_NON_PMI                        "pmix.nonpmi"           // (bool) spawned procs will not call PMIx_Init
+#define PMIX_STDIN_TGT                      "pmix.stdin"            // (uint32_t) spawned proc rank that is to receive stdin
+#define PMIX_FWD_STDIN                      "pmix.fwd.stdin"        // (bool) forward my stdin to the designated proc
+#define PMIX_FWD_STDOUT                     "pmix.fwd.stdout"       // (bool) forward stdout from spawned procs to me
+#define PMIX_FWD_STDERR                     "pmix.fwd.stderr"       // (bool) forward stderr from spawned procs to me
+#define PMIX_DEBUGGER_DAEMONS               "pmix.debugger"         // (bool) spawned app consists of debugger daemons
+#define PMIX_COSPAWN_APP                    "pmix.cospawn"          // (bool) designated app is to be spawned as a disconnected
+                                                                    //     job - i.e., not part of the "comm_world" of the job
 
 /* query attributes */
-#define PMIX_QUERY_NAMESPACES               "pmix.qry.ns"            // (char*) request a comma-delimited list of active nspaces
-#define PMIX_QUERY_JOB_STATUS               "pmix.qry.jst"           // (pmix_status_t) status of a specified currently executing job
-#define PMIX_QUERY_QUEUE_LIST               "pmix.qry.qlst"          // (char*) request a comma-delimited list of scheduler queues
-#define PMIX_QUERY_QUEUE_STATUS             "pmix.qry.qst"           // (TBD) status of a specified scheduler queue
-#define PMIX_QUERY_PROC_TABLE               "pmix.qry.ptable"        // (char*) input nspace of job whose info is being requested
-                                                                     //     returns (pmix_data_array_t) an array of pmix_proc_info_t
-#define PMIX_QUERY_LOCAL_PROC_TABLE         "pmix.qry.lptable"       // (char*) input nspace of job whose info is being requested
-                                                                     //     returns (pmix_data_array_t) an array of pmix_proc_info_t for
-                                                                     //     procs in job on same node
-#define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"         // return operations tool is authorized to perform
-#define PMIX_QUERY_SPAWN_SUPPORT            "pmix.qry.spawn"         // return a comma-delimited list of supported spawn attributes
-#define PMIX_QUERY_DEBUG_SUPPORT            "pmix.qry.debug"         // return a comma-delimited list of supported debug attributes
-#define PMIX_QUERY_MEMORY_USAGE             "pmix.qry.mem"           // return info on memory usage for the procs indicated in the qualifiers
-#define PMIX_QUERY_LOCAL_ONLY               "pmix.qry.local"         // constrain the query to local information only
-#define PMIX_QUERY_REPORT_AVG               "pmix.qry.avg"           // report average values
-#define PMIX_QUERY_REPORT_MINMAX            "pmix.qry.minmax"        // report minimum and maximum value
-#define PMIX_QUERY_ALLOC_STATUS             "pmix.query.alloc"       // (char*) string identifier of the allocation whose status
-                                                                     //         is being requested
+#define PMIX_QUERY_NAMESPACES               "pmix.qry.ns"           // (char*) request a comma-delimited list of active nspaces
+#define PMIX_QUERY_JOB_STATUS               "pmix.qry.jst"          // (pmix_status_t) status of a specified currently executing job
+#define PMIX_QUERY_QUEUE_LIST               "pmix.qry.qlst"         // (char*) request a comma-delimited list of scheduler queues
+#define PMIX_QUERY_QUEUE_STATUS             "pmix.qry.qst"          // (TBD) status of a specified scheduler queue
+#define PMIX_QUERY_PROC_TABLE               "pmix.qry.ptable"       // (char*) input nspace of job whose info is being requested
+                                                                    //         returns (pmix_data_array_t) an array of pmix_proc_info_t
+#define PMIX_QUERY_LOCAL_PROC_TABLE         "pmix.qry.lptable"      // (char*) input nspace of job whose info is being requested
+                                                                    //         returns (pmix_data_array_t) an array of pmix_proc_info_t for
+                                                                    //         procs in job on same node
+#define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"        // (bool) return operations tool is authorized to perform
+#define PMIX_QUERY_SPAWN_SUPPORT            "pmix.qry.spawn"        // (bool) return a comma-delimited list of supported spawn attributes
+#define PMIX_QUERY_DEBUG_SUPPORT            "pmix.qry.debug"        // (bool) return a comma-delimited list of supported debug attributes
+#define PMIX_QUERY_MEMORY_USAGE             "pmix.qry.mem"          // (bool) return info on memory usage for the procs indicated in the qualifiers
+#define PMIX_QUERY_LOCAL_ONLY               "pmix.qry.local"        // (bool) constrain the query to local information only
+#define PMIX_QUERY_REPORT_AVG               "pmix.qry.avg"          // (bool) report average values
+#define PMIX_QUERY_REPORT_MINMAX            "pmix.qry.minmax"       // (bool) report minimum and maximum value
+#define PMIX_QUERY_ALLOC_STATUS             "pmix.query.alloc"      // (char*) string identifier of the allocation whose status
+                                                                    //         is being requested
+#define PMIX_TIME_REMAINING                 "pmix.time.remaining"   // (char*) query number of seconds (uint32_t) remaining in allocation
+                                                                    //         for the specified nspace
 
 /* log attributes */
-#define PMIX_LOG_STDERR                    "pmix.log.stderr"         // (char*) log string to stderr
-#define PMIX_LOG_STDOUT                    "pmix.log.stdout"         // (char*) log string to stdout
-#define PMIX_LOG_SYSLOG                    "pmix.log.syslog"         // (char*) log data to syslog - defaults to ERROR priority unless
-#define PMIX_LOG_MSG                       "pmix.log.msg"            // (pmix_byte_object_t) message blob to be sent somewhere
+#define PMIX_LOG_STDERR                     "pmix.log.stderr"       // (char*) log string to stderr
+#define PMIX_LOG_STDOUT                     "pmix.log.stdout"       // (char*) log string to stdout
+#define PMIX_LOG_SYSLOG                     "pmix.log.syslog"       // (char*) log data to syslog - defaults to ERROR priority unless
+#define PMIX_LOG_MSG                        "pmix.log.msg"          // (pmix_byte_object_t) message blob to be sent somewhere
+#define PMIX_LOG_EMAIL                      "pmix.log.email"        // (pmix_data_array_t) log via email based on pmix_info_t containing directives
+#define PMIX_LOG_EMAIL_ADDR                 "pmix.log.emaddr"       // (char*) comma-delimited list of email addresses that are to recv msg
+#define PMIX_LOG_EMAIL_SUBJECT              "pmix.log.emsub"        // (char*) subject line for email
+#define PMIX_LOG_EMAIL_MSG                  "pmix.log.emmsg"        // (char*) msg to be included in email
 
 /* debugger attributes */
-#define PMIX_DEBUG_STOP_ON_EXEC             "pmix.dbg.exec"          // (bool) job is being spawned under debugger - instruct it to pause on start
-#define PMIX_DEBUG_STOP_IN_INIT             "pmix.dbg.init"          // (bool) instruct job to stop during PMIx init
-#define PMIX_DEBUG_WAIT_FOR_NOTIFY          "pmix.dbg.notify"        // (bool) block at desired point until receiving debugger release notification
-#define PMIX_DEBUG_JOB                      "pmix.dbg.job"           // (char*) nspace of the job to be debugged - the RM/PMIx server are
-#define PMIX_DEBUG_WAITING_FOR_NOTIFY       "pmix.dbg.waiting"       // (bool) job to be debugged is waiting for a release
+#define PMIX_DEBUG_STOP_ON_EXEC             "pmix.dbg.exec"         // (bool) job is being spawned under debugger - instruct it to pause on start
+#define PMIX_DEBUG_STOP_IN_INIT             "pmix.dbg.init"         // (bool) instruct job to stop during PMIx init
+#define PMIX_DEBUG_WAIT_FOR_NOTIFY          "pmix.dbg.notify"       // (bool) block at desired point until receiving debugger release notification
+#define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) nspace of the job to be debugged - the RM/PMIx server are
+#define PMIX_DEBUG_WAITING_FOR_NOTIFY       "pmix.dbg.waiting"      // (bool) job to be debugged is waiting for a release
 
 /* Resource Manager identification */
-#define PMIX_RM_NAME                        "pmix.rm.name"           // (char*) string name of the resource manager
-#define PMIX_RM_VERSION                     "pmix.rm.version"        // (char*) RM version string
+#define PMIX_RM_NAME                        "pmix.rm.name"          // (char*) string name of the resource manager
+#define PMIX_RM_VERSION                     "pmix.rm.version"       // (char*) RM version string
 
 /* attributes for setting envars */
 #define PMIX_SET_ENVAR                      "pmix.set.envar"        // (char*) string "key=value" value shall be put into the environment
@@ -327,7 +336,6 @@ typedef uint32_t pmix_rank_t;
 /* attributes relating to allocations */
 #define PMIX_ALLOC_ID                       "pmix.alloc.id"         // (char*) provide a string identifier for this allocation request
                                                                     //         which can later be used to query status of the request
-#define PMIX_TIME_REMAINING                 "pmix.time.remaining"   // (uint32_t) get number of seconds remaining in allocation
 #define PMIX_ALLOC_NUM_NODES                "pmix.alloc.nnodes"     // (uint64_t) number of nodes
 #define PMIX_ALLOC_NODE_LIST                "pmix.alloc.nlist"      // (char*) regex of specific nodes
 #define PMIX_ALLOC_NUM_CPUS                 "pmix.alloc.ncpus"      // (uint64_t) number of cpus
@@ -343,6 +351,38 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_ALLOC_NETWORK_QOS              "pmix.alloc.netqos"     // (char*) quality of service level
 #define PMIX_ALLOC_TIME                     "pmix.alloc.time"       // (uint32_t) time in seconds
 
+/* job control attributes */
+#define PMIX_JOB_CTRL_ID                    "pmix.jctrl.id"         // (char*) provide a string identifier for this request
+#define PMIX_JOB_CTRL_PAUSE                 "pmix.jctrl.pause"      // (bool) pause the specified processes
+#define PMIX_JOB_CTRL_RESUME                "pmix.jctrl.resume"     // (bool) "un-pause" the specified processes
+#define PMIX_JOB_CTRL_CANCEL                "pmix.jctrl.cancel"     // (char*) cancel the specified request
+                                                                    //         (NULL => cancel all requests from this requestor)
+#define PMIX_JOB_CTRL_KILL                  "pmix.jctrl.kill"       // (bool) forcibly terminate the specified processes and cleanup
+#define PMIX_JOB_CTRL_RESTART               "pmix.jctrl.restart"    // (char*) restart the specified processes using the given checkpoint ID
+#define PMIX_JOB_CTRL_CHECKPOINT            "pmix.jctrl.ckpt"       // (char*) checkpoint the specified processes and assign the given ID to it
+#define PMIX_JOB_CTRL_CHECKPOINT_EVENT      "pmix.jctrl.ckptev"     // (bool) use event notification to trigger process checkpoint
+#define PMIX_JOB_CTRL_CHECKPOINT_SIGNAL     "pmix.jctrl.ckptsig"    // (int) use the given signal to trigger process checkpoint
+#define PMIX_JOB_CTRL_CHECKPOINT_TIMEOUT    "pmix.jctrl.ckptsig"    // (int) time in seconds to wait for checkpoint to complete
+#define PMIX_JOB_CTRL_SIGNAL                "pmix.jctrl.sig"        // (int) send given signal to specified processes
+#define PMIX_JOB_CTRL_PROVISION             "pmix.jctrl.pvn"        // (char*) regex identifying nodes that are to be provisioned
+#define PMIX_JOB_CTRL_PROVISION_IMAGE       "pmix.jctrl.pvnimg"     // (char*) name of the image that is to be provisioned
+#define PMIX_JOB_CTRL_PREEMPTIBLE           "pmix.jctrl.preempt"    // (bool) job can be pre-empted
+
+/* monitoring attributes */
+#define PMIX_MONITOR_HEARTBEAT              "pmix.monitor.mbeat"    // (void) register to have the server monitor the requestor for heartbeats
+#define PMIX_SEND_HEARTBEAT                 "pmix.monitor.beat"     // (void) send heartbeat to local server
+#define PMIX_MONITOR_HEARTBEAT_TIME         "pmix.monitor.btime"    // (uint32_t) time in seconds before declaring heartbeat missed
+#define PMIX_MONITOR_HEARTBEAT_DROPS        "pmix.monitor.bdrop"    // (uint32_t) number of heartbeats that can be missed before taking
+                                                                    //            specified action
+#define PMIX_MONITOR_FILE                   "pmix.monitor.fmon"     // (char*) register to monitor file for signs of life
+#define PMIX_MONITOR_FILE_SIZE              "pmix.monitor.fsize"    // (bool) monitor size of given file is growing to determine app is running
+#define PMIX_MONITOR_FILE_ACCESS            "pmix.monitor.faccess"  // (char*) monitor time since last access of given file to determine app is running
+#define PMIX_MONITOR_FILE_MODIFY            "pmix.monitor.fmod"     // (char*) monitor time since last modified of given file to determine app is running
+#define PMIX_MONITOR_FILE_CHECK_TIME        "pmix.monitor.ftime"    // (uint32_t) time in seconds between checking file
+#define PMIX_MONITOR_FILE_DROPS             "pmix.monitor.fdrop"    // (uint32_t) number of file checks that can be missed before taking
+                                                                    //            specified action
+
+
 /****    PROCESS STATE DEFINITIONS    ****/
 typedef uint8_t pmix_proc_state_t;
 #define PMIX_PROC_STATE_UNDEF                    0  /* undefined process state */
@@ -455,7 +495,14 @@ typedef int pmix_status_t;
 #define PMIX_ERR_LOST_CONNECTION_TO_CLIENT      (PMIX_ERR_V2X_BASE - 3)
 /* used by the query system */
 #define PMIX_QUERY_PARTIAL_SUCCESS              (PMIX_ERR_V2X_BASE - 4)
+/* request responses */
 #define PMIX_NOTIFY_ALLOC_COMPLETE              (PMIX_ERR_V2X_BASE - 5)
+/* job control */
+#define PMIX_JCTRL_CHECKPOINT                   (PMIX_ERR_V2X_BASE - 6)
+#define PMIX_JCTRL_PREEMPT_ALERT                (PMIX_ERR_V2X_BASE - 7)
+/* monitoring */
+#define PMIX_MONITOR_HEARTBEAT_ALERT            (PMIX_ERR_V2X_BASE - 8)
+#define PMIX_MONITOR_FILE_ALERT                 (PMIX_ERR_V2X_BASE - 9)
 
 /* define a starting point for operational error constants so
  * we avoid renumbering when making additions */

opal/mca/pmix/pmix2x/pmix/include/pmix_server.h

@@ -328,6 +328,17 @@ typedef pmix_status_t (*pmix_server_alloc_fn_t)(const pmix_proc_t *client,
                                                 const pmix_info_t data[], size_t ndata,
                                                 pmix_info_cbfunc_t cbfunc, void *cbdata);
 
+/* Execute a job control action on behalf of a client */
+typedef pmix_status_t (*pmix_server_job_control_fn_t)(const pmix_proc_t *requestor,
+                                                      const pmix_proc_t targets[], size_t ntargets,
+                                                      const pmix_info_t directives[], size_t ndirs,
+                                                      pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Request that a client be monitored for activity */
+typedef pmix_status_t (*pmix_server_monitor_fn_t)(const pmix_proc_t *requestor, pmix_status_t error,
+                                                  const pmix_info_t directives[], size_t ndirs,
+                                                  pmix_info_cbfunc_t cbfunc, void *cbdata);
+
 typedef struct pmix_server_module_2_0_0_t {
     /* v1x interfaces */
     pmix_server_client_connected_fn_t   client_connected;
@@ -350,12 +361,14 @@ typedef struct pmix_server_module_2_0_0_t {
     pmix_server_tool_connection_fn_t    tool_connected;
     pmix_server_log_fn_t                log;
     pmix_server_alloc_fn_t              allocate;
+    pmix_server_job_control_fn_t        job_control;
+    pmix_server_monitor_fn_t            monitor;
 } pmix_server_module_t;
 
 /****    SERVER SUPPORT INIT/FINALIZE FUNCTIONS    ****/
 
 /* Initialize the server support library, and provide a
- *  pointer to a pmix_server_module_t structure
+ * pointer to a pmix_server_module_t structure
  * containing the caller's callback functions. The
  * array of pmix_info_t structs is used to pass
  * additional info that may be required by the server

opal/mca/pmix/pmix2x/pmix/src/common/Makefile.include

@@ -1,6 +1,6 @@
 # -*- makefile -*-
 #
-# Copyright (c) 2015 Intel, Inc.  All rights reserved.
+# Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
 # Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -13,4 +13,5 @@ sources += \
         common/pmix_query.c \
         common/pmix_strings.c \
         common/pmix_log.c \
-        common/pmix_jobdata.c
+        common/pmix_jobdata.c \
+        common/pmix_control.c

opal/mca/pmix/pmix2x/pmix/src/common/pmix_control.c

@@ -0,0 +1,269 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include <src/include/pmix_config.h>
+
+#include <src/include/types.h>
+#include <src/include/pmix_stdint.h>
+#include <src/include/pmix_socket_errno.h>
+
+#include <pmix.h>
+#include <pmix_common.h>
+#include <pmix_server.h>
+#include <pmix_rename.h>
+
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/buffer_ops/buffer_ops.h"
+#include "src/mca/ptl/ptl.h"
+
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/include/pmix_globals.h"
+
+static void relcbfunc(void *cbdata)
+{
+    pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:query release callback");
+
+    if (NULL != cd->info) {
+        PMIX_INFO_FREE(cd->info, cd->ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+static void query_cbfunc(struct pmix_peer_t *peer,
+                         pmix_ptl_hdr_t *hdr,
+                         pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+    pmix_status_t rc;
+    pmix_shift_caddy_t *results;
+    int cnt;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:query cback from server");
+
+    results = PMIX_NEW(pmix_shift_caddy_t);
+
+    /* unpack the status */
+    cnt = 1;
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &results->status, &cnt, PMIX_STATUS))) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (PMIX_SUCCESS != results->status) {
+        goto complete;
+    }
+
+    /* unpack any returned data */
+    cnt = 1;
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &results->ninfo, &cnt, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (0 < results->ninfo) {
+        PMIX_INFO_CREATE(results->info, results->ninfo);
+        cnt = results->ninfo;
+        if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, results->info, &cnt, PMIX_INFO))) {
+            PMIX_ERROR_LOG(rc);
+            goto complete;
+        }
+    }
+
+  complete:
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:query cback from server releasing");
+    /* release the caller */
+    if (NULL != cd->cbfunc) {
+        cd->cbfunc(results->status, results->info, results->ninfo, cd->cbdata, relcbfunc, results);
+    }
+    PMIX_RELEASE(cd);
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_t ntargets,
+                                              const pmix_info_t directives[], size_t ndirs,
+                                              pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_JOB_CONTROL_CMD;
+    pmix_status_t rc;
+    pmix_query_caddy_t *cb;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix: job control called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we are the server, then we just issue the request and
+     * return the response */
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
+        if (NULL == pmix_host_server.job_control) {
+            /* nothing we can do */
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "pmix:job_control handed to RM");
+        rc = pmix_host_server.job_control(&pmix_globals.myid,
+                                          targets, ntargets,
+                                          directives, ndirs,
+                                          cbfunc, cbdata);
+        return rc;
+    }
+
+    /* if we are a client, then relay this request to the server */
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        return PMIX_ERR_UNREACH;
+    }
+
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* pack the number of targets */
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ntargets, 1, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+    /* remember, the targets can be NULL to indicate that the operation
+     * is to be done against all members of our nspace */
+    if (0 < ntargets) {
+        /* pack the targets */
+        if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, targets, ntargets, PMIX_PROC))) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            return rc;
+        }
+    }
+
+    /* pack the directives */
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ndirs, 1, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+    if (0 < ndirs) {
+        if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, directives, ndirs, PMIX_INFO))) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            return rc;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_query_caddy_t);
+    cb->cbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* push the message into our event base to send to the server */
+    if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, query_cbfunc, (void*)cb))){
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+    }
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pmix_status_t error,
+                                                  const pmix_info_t directives[], size_t ndirs,
+                                                  pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_MONITOR_CMD;
+    pmix_status_t rc;
+    pmix_query_caddy_t *cb;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix: monitor called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we are the server, then we just issue the request and
+     * return the response */
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
+        if (NULL == pmix_host_server.monitor) {
+            /* nothing we can do */
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "pmix:monitor handed to RM");
+        rc = pmix_host_server.monitor(&pmix_globals.myid, error,
+                                      directives, ndirs, cbfunc, cbdata);
+        return rc;
+    }
+
+    /* if we are a client, then relay this request to the server */
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        return PMIX_ERR_UNREACH;
+    }
+
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* pack the error */
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &error, 1, PMIX_STATUS))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* pack the directives */
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ndirs, 1, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+    if (0 < ndirs) {
+        if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, directives, ndirs, PMIX_INFO))) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            return rc;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_query_caddy_t);
+    cb->cbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* push the message into our event base to send to the server */
+    if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, query_cbfunc, (void*)cb))){
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+    }
+
+    return rc;
+}

opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c

@@ -257,6 +257,8 @@ static void qcon(pmix_query_caddy_t *p)
 {
     p->queries = NULL;
     p->nqueries = 0;
+    p->targets = NULL;
+    p->ntargets = 0;
     p->info = NULL;
     p->ninfo = 0;
     p->cbfunc = NULL;

opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h

@@ -72,7 +72,9 @@ typedef enum {
     PMIX_DEREGEVENTS_CMD,
     PMIX_QUERY_CMD,
     PMIX_LOG_CMD,
-    PMIX_ALLOC_CMD
+    PMIX_ALLOC_CMD,
+    PMIX_JOB_CONTROL_CMD,
+    PMIX_MONITOR_CMD
 } pmix_cmd_t;
 
 /* provide a "pretty-print" function for cmds */
@@ -214,6 +216,8 @@ typedef struct {
     pmix_status_t status;
     pmix_query_t *queries;
     size_t nqueries;
+    pmix_proc_t *targets;
+    size_t ntargets;
     pmix_info_t *info;
     size_t ninfo;
     pmix_info_cbfunc_t cbfunc;

opal/mca/pmix/pmix2x/pmix/src/include/types.h

@@ -256,4 +256,13 @@ typedef struct event pmix_event_t;
 
 #define pmix_event_active(x, y, z) event_active((x), (y), (z))
 
+#define pmix_event_evtimer_new(b, cb, arg) pmix_event_new((b), -1, 0, (cb), (arg))
+
+#define pmix_event_evtimer_add(x, tv) pmix_event_add((x), (tv))
+
+#define pmix_event_evtimer_set(b, x, cb, arg) event_assign((x), (b), -1, 0, (event_callback_fn) (cb), (arg))
+
+#define pmix_event_evtimer_del(x) pmix_event_del((x))
+
+
 #endif /* PMIX_TYPES_H */

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/Makefile.am

@@ -3,26 +3,27 @@
 #
 # Copyright (c) 2017      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
-# 
+#
 # Additional copyrights may follow
-# 
+#
 # $HEADER$
 #
 
+AM_CPPFLAGS = $(LTDLINCL)
+
 # main library setup
-noinst_LTLIBRARIES = libmca_sensor.la
-libmca_sensor_la_SOURCES =
+noinst_LTLIBRARIES = libmca_psensor.la
+libmca_psensor_la_SOURCES =
 
 # local files
-headers = sensor.h \
-          sensor_types.h
+headers = psensor.h
 
-libmca_sensor_la_SOURCES += $(headers)
+libmca_psensor_la_SOURCES += $(headers)
 
 # Conditionally install the header files
 if WANT_INSTALL_HEADERS
-ortedir = $(ompiincludedir)/$(subdir)
-nobase_orte_HEADERS = $(headers)
+pmixdir = $(pmixincludedir)/$(subdir)
+nobase_pmix_HEADERS = $(headers)
 endif
 
 include base/Makefile.am

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/base/Makefile.am

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved.
 #
 # Copyright (c) 2017      Intel, Inc.  All rights reserved.
@@ -11,10 +11,9 @@
 #
 
 headers += \
-        base/base.h \
-        base/sensor_private.h
+        base/base.h
 
-libmca_sensor_la_SOURCES += \
-        base/sensor_base_frame.c \
-        base/sensor_base_select.c \
-        base/sensor_base_fns.c
+libmca_psensor_la_SOURCES += \
+        base/psensor_base_frame.c \
+        base/psensor_base_select.c \
+        base/psensor_base_stubs.c

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/base/base.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
+ *
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/** @file:
+ */
+
+#ifndef PMIX_PSENSOR_BASE_H_
+#define PMIX_PSENSOR_BASE_H_
+
+#include <src/include/pmix_config.h>
+
+#include "src/class/pmix_list.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+#include "src/mca/psensor/psensor.h"
+
+BEGIN_C_DECLS
+
+/*
+ * MCA Framework
+ */
+PMIX_EXPORT extern pmix_mca_base_framework_t pmix_psensor_base_framework;
+
+PMIX_EXPORT int pmix_psensor_base_select(void);
+
+/* define a struct to hold framework-global values */
+typedef struct {
+    pmix_list_t actives;
+    pmix_event_base_t *evbase;
+} pmix_psensor_base_t;
+
+typedef struct {
+    pmix_list_item_t super;
+    pmix_psensor_base_component_t *component;
+    pmix_psensor_base_module_t *module;
+    int priority;
+} pmix_psensor_active_module_t;
+PMIX_CLASS_DECLARATION(pmix_psensor_active_module_t);
+
+PMIX_EXPORT extern pmix_psensor_base_t pmix_psensor_base;
+
+PMIX_EXPORT pmix_status_t pmix_psensor_base_start(pmix_peer_t *requestor, pmix_status_t error,
+                                                  const pmix_info_t *monitor,
+                                                  const pmix_info_t directives[], size_t ndirs);
+
+PMIX_EXPORT pmix_status_t pmix_psensor_base_stop(pmix_peer_t *requestor,
+                                                 char *id);
+
+END_C_DECLS
+#endif

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/base/psensor_base_frame.c

@@ -0,0 +1,103 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved.
+ *
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include <src/include/pmix_config.h>
+
+#include <pmix_common.h>
+
+#include <pthread.h>
+#include PMIX_EVENT_HEADER
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+#include "src/class/pmix_list.h"
+#include "src/runtime/pmix_progress_threads.h"
+#include "src/include/types.h"
+
+#include "src/mca/psensor/base/base.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "src/mca/psensor/base/static-components.h"
+
+/*
+ * Global variables
+ */
+pmix_psensor_base_module_t pmix_psensor = {
+    pmix_psensor_base_start,
+    pmix_psensor_base_stop
+};
+pmix_psensor_base_t pmix_psensor_base = {{{0}}};;
+
+static bool use_separate_thread = false;
+
+static int pmix_psensor_register(pmix_mca_base_register_flag_t flags)
+{
+    (void) pmix_mca_base_var_register("pmix", "psensor", "base", "use_separate_thread",
+                                      "Use a separate thread for monitoring local procs",
+                                      PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                      PMIX_INFO_LVL_9,
+                                      PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                      &use_separate_thread);
+    return PMIX_SUCCESS;
+}
+
+
+static int pmix_psensor_base_close(void)
+{
+    PMIX_LIST_DESTRUCT(&pmix_psensor_base.actives);
+
+    if (use_separate_thread && NULL != pmix_psensor_base.evbase) {
+        (void)pmix_progress_thread_stop("PSENSOR");
+    }
+
+    /* Close all remaining available components */
+    return pmix_mca_base_framework_components_close(&pmix_psensor_base_framework, NULL);
+}
+
+/**
+ * Function for finding and opening either all MCA components, or the one
+ * that was specifically requested via a MCA parameter.
+ */
+static int pmix_psensor_base_open(pmix_mca_base_open_flag_t flags)
+{
+    /* construct the list of modules */
+    PMIX_CONSTRUCT(&pmix_psensor_base.actives, pmix_list_t);
+
+    if (use_separate_thread) {
+        /* create an event base and progress thread for us */
+        if (NULL == (pmix_psensor_base.evbase = pmix_progress_thread_init("PSENSOR"))) {
+            return PMIX_ERROR;
+        }
+
+    } else {
+        pmix_psensor_base.evbase = pmix_globals.evbase;
+    }
+
+    /* Open up all available components */
+    return pmix_mca_base_framework_components_open(&pmix_psensor_base_framework, flags);
+}
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, psensor, "PMIx Monitoring Sensors",
+                                pmix_psensor_register,
+                                pmix_psensor_base_open, pmix_psensor_base_close,
+                                mca_psensor_base_static_components, 0);
+
+PMIX_CLASS_INSTANCE(pmix_psensor_active_module_t,
+                    pmix_list_item_t,
+                    NULL, NULL);

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/base/psensor_base_select.c

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include <string.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+#include "src/mca/psensor/base/base.h"
+
+static bool selected = false;
+
+/* Function for selecting a prioritized list of components
+ * from all those that are available. */
+int pmix_psensor_base_select(void)
+{
+    pmix_mca_base_component_list_item_t *cli = NULL;
+    pmix_psensor_base_component_t *component = NULL;
+    pmix_psensor_active_module_t *newactive, *active;
+    pmix_mca_base_module_t *mod;
+    int pri;
+    bool inserted;
+
+    if (selected) {
+        /* ensure we don't do this twice */
+        return PMIX_SUCCESS;
+    }
+    selected = true;
+
+    /* Query all available components and ask if they have a module */
+    PMIX_LIST_FOREACH(cli, &pmix_psensor_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
+        component = (pmix_psensor_base_component_t *) cli->cli_component;
+
+        pmix_output_verbose(5, pmix_psensor_base_framework.framework_output,
+                            "mca:psensor:select: checking available component %s",
+                            component->base.pmix_mca_component_name);
+
+        /* get the module for this component */
+        if (PMIX_SUCCESS != component->base.pmix_mca_query_component(&mod, &pri)) {
+            continue;
+        }
+
+        /* add to our prioritized list of available actives */
+        newactive = PMIX_NEW(pmix_psensor_active_module_t);
+        newactive->priority = pri;
+        newactive->component = component;
+        newactive->module = (pmix_psensor_base_module_t*)mod;
+
+        /* maintain priority order */
+        inserted = false;
+        PMIX_LIST_FOREACH(active, &pmix_psensor_base.actives, pmix_psensor_active_module_t) {
+            if (newactive->priority > active->priority) {
+                pmix_list_insert_pos(&pmix_psensor_base.actives,
+                                     (pmix_list_item_t*)active, &newactive->super);
+                inserted = true;
+                break;
+            }
+        }
+        if (!inserted) {
+            /* must be lowest priority - add to end */
+            pmix_list_append(&pmix_psensor_base.actives, &newactive->super);
+        }
+    }
+
+    if (4 < pmix_output_get_verbosity(pmix_psensor_base_framework.framework_output)) {
+        pmix_output(0, "Final PSENSOR priorities");
+        /* show the prioritized list */
+        PMIX_LIST_FOREACH(active, &pmix_psensor_base.actives, pmix_psensor_active_module_t) {
+            pmix_output(0, "\tPSENSOR: %s Priority: %d",
+                        active->component->base.pmix_mca_component_name, active->priority);
+        }
+    }
+
+    return PMIX_SUCCESS;;
+}

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/base/psensor_base_stubs.c

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include "src/util/error.h"
+
+#include "src/mca/psensor/base/base.h"
+
+static bool mods_active = false;
+
+pmix_status_t pmix_psensor_base_start(pmix_peer_t *requestor, pmix_status_t error,
+                                      const pmix_info_t *monitor,
+                                      const pmix_info_t directives[], size_t ndirs)
+{
+    pmix_psensor_active_module_t *mod;
+    pmix_status_t rc;
+
+    opal_output_verbose(5, pmix_psensor_base_framework.framework_output,
+                        "%s:%d sensor:base: starting sensors",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank);
+
+    /* call the start function of all modules in priority order */
+    PMIX_LIST_FOREACH(mod, &pmix_psensor_base.actives, pmix_psensor_active_module_t) {
+        if (NULL != mod->module->start) {
+            rc = mod->module->start(requestor, error, monitor, directives, ndirs);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_psensor_base_stop(pmix_peer_t *requestor,
+                                     char *id)
+{
+    pmix_psensor_active_module_t *mod;
+    pmix_status_t rc;
+
+    opal_output_verbose(5, pmix_psensor_base_framework.framework_output,
+                        "%s:%d sensor:base: stopping sensors",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank);
+
+    /* call the stop function of all modules in priority order */
+    PMIX_LIST_FOREACH(mod, &pmix_psensor_base.actives, pmix_psensor_active_module_t) {
+        if (NULL != mod->module->stop) {
+            rc = mod->module->stop(requestor, id);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/Makefile.am

@@ -1,37 +1,37 @@
 #
-# Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved. 
+# Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
-# 
+#
 # Additional copyrights may follow
-# 
+#
 # $HEADER$
 #
 
-dist_ompidata_DATA = help-orte-sensor-file.txt
+dist_pmixdata_DATA = help-pmix-psensor-file.txt
 
 sources = \
-        sensor_file.c \
-        sensor_file.h \
-        sensor_file_component.c
+        psensor_file.c \
+        psensor_file.h \
+        psensor_file_component.c
 
 # Make the output library in this directory, and name it either
 # mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
 # (for static builds).
 
-if MCA_BUILD_orte_sensor_file_DSO
+if MCA_BUILD_pmix_psensor_file_DSO
 component_noinst =
-component_install = mca_sensor_file.la
+component_install = mca_psensor_file.la
 else
-component_noinst = libmca_sensor_file.la
+component_noinst = libmca_psensor_file.la
 component_install =
 endif
 
-mcacomponentdir = $(ompilibdir)
+mcacomponentdir = $(pmixlibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
-mca_sensor_file_la_SOURCES = $(sources)
-mca_sensor_file_la_LDFLAGS = -module -avoid-version
+mca_psensor_file_la_SOURCES = $(sources)
+mca_psensor_file_la_LDFLAGS = -module -avoid-version
 
 noinst_LTLIBRARIES = $(component_noinst)
-libmca_sensor_file_la_SOURCES =$(sources)
-libmca_sensor_file_la_LDFLAGS = -module -avoid-version
+libmca_psensor_file_la_SOURCES =$(sources)
+libmca_psensor_file_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/help-pmix-psensor-file.txt

@@ -4,9 +4,9 @@
 #
 # Copyright (c) 2017      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
-# 
+#
 # Additional copyrights may follow
-# 
+#
 # $HEADER$
 #
 # This is the US/English general help file for the file sensor

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.c

@@ -0,0 +1,352 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ *
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <src/include/types.h>
+#include <pmix_common.h>
+
+#include <stdio.h>
+#include <stddef.h>
+#include <ctype.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#include <fcntl.h>
+#include <errno.h>
+#include <signal.h>
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "src/class/pmix_list.h"
+#include "src/include/pmix_globals.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/util/show_help.h"
+
+#include "src/mca/psensor/base/base.h"
+#include "psensor_file.h"
+
+/* declare the API functions */
+static pmix_status_t start(pmix_peer_t *requestor, pmix_status_t error,
+                           const pmix_info_t *monitor,
+                           const pmix_info_t directives[], size_t ndirs);
+static pmix_status_t stop(pmix_peer_t *requestor, char *id);
+
+/* instantiate the module */
+pmix_psensor_base_module_t pmix_psensor_file_module = {
+    .start = start,
+    .stop = stop
+};
+
+/* define a tracking object */
+typedef struct {
+    pmix_list_item_t super;
+    pmix_peer_t *requestor;
+    char *id;
+    bool event_active;
+    pmix_event_t ev;
+    pmix_event_t cdev;
+    struct timeval tv;
+    int tick;
+    char *file;
+    bool file_size;
+    bool file_access;
+    bool file_mod;
+    size_t last_size;
+    time_t last_access;
+    time_t last_mod;
+    uint32_t ndrops;
+    uint32_t nmisses;
+    pmix_status_t error;
+    pmix_data_range_t range;
+    pmix_info_t *info;
+    size_t ninfo;
+} file_tracker_t;
+static void ft_constructor(file_tracker_t *ft)
+{
+    ft->requestor = NULL;
+    ft->id = NULL;
+    ft->event_active = false;
+    ft->tv.tv_sec = 0;
+    ft->tv.tv_usec = 0;
+    ft->tick = 0;
+    ft->file_size = false;
+    ft->file_access = false;
+    ft->file_mod = false;
+    ft->last_size = 0;
+    ft->last_access = 0;
+    ft->last_mod = 0;
+    ft->ndrops = 0;
+    ft->nmisses = 0;
+    ft->error = PMIX_SUCCESS;
+    ft->range = PMIX_RANGE_NAMESPACE;
+    ft->info = NULL;
+    ft->ninfo = 0;
+}
+static void ft_destructor(file_tracker_t *ft)
+{
+    if (NULL != ft->requestor) {
+        PMIX_RELEASE(ft->requestor);
+    }
+    if (NULL != ft->id) {
+        free(ft->id);
+    }
+    if (event_active) {
+        pmix_event_del(&ft->ev);
+    }
+    if (NULL != ft->file) {
+        free(ft->file);
+    }
+    if (NULL != ft->info) {
+        PMIX_INFO_FREE(ft->info, ft->ninfo);
+    }
+}
+PMIX_CLASS_INSTANCE(file_tracker_t,
+                    pmix_list_item_t,
+                    ft_constructor, ft_destructor);
+
+/* define a local caddy */
+typedef struct {
+    pmix_object_t super;
+    pmix_event_t ev;
+    pmix_peer_t *requestor;
+    char *id;
+} file_caddy_t;
+static void cd_con(file_caddy_t *p)
+{
+    p->requestor = NULL;
+    p->id = NULL;
+}
+static void cd_des(file_caddy_t *p)
+{
+    if (NULL != (p->requestor)) {
+        PMIX_RELEASE(p->requestor);
+    }
+    if (NULL != p->id) {
+        free(p->id);
+    }
+}
+PMIX_CLASS_INSTANCE(file_caddy_t,
+                    pmix_object_t,
+                    cd_con, cd_des);
+
+static void file_sample(int sd, short args, void *cbdata);
+
+static void add_tracker(int sd, short flags, void *cbdata)
+{
+    file_tracker_t *ft = (file_tracker_t*)cbdata;
+
+    /* add the tracker to our list */
+    pmix_list_append(&mca_psensor_file_component.trackers, &ft->super);
+
+    /* setup the timer event */
+    pmix_event_evtimer_set(pmix_psensor_base.evbase, &ft->ev,
+                           file_sample, ft);
+    pmix_event_evtimer_add(&ft->ev, &ft->tv);
+    ft->event_active = true;
+}
+
+/*
+ * Start monitoring of local processes
+ */
+static pmix_status_t start(pmix_peer_t *requestor, pmix_status_t error,
+                           const pmix_info_t *monitor,
+                           const pmix_info_t directives[], size_t ndirs)
+{
+    file_tracker_t *ft;
+    pmix_info_t *ptr;
+    size_t n, n2;
+
+    PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                         "[%s:%d] checking file monitoring for requestor %s:%d",
+                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                         requestor->info->nptr->nspace, requestor->info->rank));
+
+    /* if they didn't ask to monitor a file, then nothing for us to do */
+    if (0 != strcmp(monitor->key, PMIX_MONITOR_FILE)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* setup to track this monitoring operation */
+    ft = PMIX_NEW(file_tracker_t);
+    PMIX_RETAIN(requestor);
+    ft->requestor = requestor;
+    ft->file = strdup(monitor->value.data.string);
+
+    /* check the directives to see if what they want monitored */
+    for (n=0; n < ndirs; n++) {
+        if (0 == strcmp(directives[n].key, PMIX_MONITOR_FILE_SIZE)) {
+            ft->file_size = directives[n].value.data.flag;
+        } else if (0 == strcmp(directives[n].key, PMIX_MONITOR_FILE_ACCESS)) {
+            ft->file_access = directives[n].value.data.flag;
+        } else if (0 == strcmp(directives[n].key, PMIX_MONITOR_FILE_MODIFY)) {
+            ft->file_mod = directives[n].value.data.flag;
+        } else if (0 == strcmp(directives[n].key, PMIX_MONITOR_FILE_DROPS)) {
+            ft->ndrops = directives[n].value.data.uint32;
+        } else if (0 == strcmp(directives[n].key, PMIX_MONITOR_FILE_CHECK_TIME)) {
+            ft->tv.tv_sec = directives[n].value.data.uint32;
+        } else if (0 == strcmp(directives[n].key, PMIX_RANGE)) {
+            ft->range = directives[n].value.data.range;
+        }
+    }
+
+    if (0 == ft->tv.tv_sec ||
+        (!ft->file_size && !ft->file_access && !ft->file_mod)) {
+        /* didn't specify a sample rate, or what should be sampled */
+        PMIX_RELEASE(ft);
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* need to push into our event base to add this to our trackers */
+    pmix_event_assign(&ft->cdev, pmix_psensor_base.evbase, -1,
+                      EV_WRITE, add_tracker, ft);
+    pmix_event_active(&ft->cdev, EV_WRITE, 1);
+
+    return PMIX_SUCCESS;
+}
+
+
+static void del_tracker(int sd, short flags, void *cbdata)
+{
+    file_caddy_t *cd = (file_caddy_t*)cbdata;
+    file_tracker_t *ft, *ftnext;
+
+    /* remove the tracker from our list */
+    PMIX_LIST_FOREACH_SAFE(ft, ftnext, &mca_psensor_file_component.trackers, file_tracker_t) {
+        if (ft->requestor != cd->requestor) {
+            continue;
+        }
+        if (NULL == cd->id ||
+            (NULL != ft->id && 0 == strcmp(ft->id, cd->id))) {
+            pmix_list_remove_item(&mca_psensor_file_component.trackers, &ft->super);
+            PMIX_RELEASE(ft);
+        }
+    }
+    PMIX_RELEASE(cd);
+}
+
+static pmix_status_t stop(pmix_peer_t *requestor, char *id)
+{
+    file_caddy_t *cd;
+
+    cd = PMIX_NEW(file_caddy_t);
+    PMIX_RETAIN(requestor);
+    cd->requestor = requestor;
+    cd->id = strdup(id);
+
+    /* need to push into our event base to add this to our trackers */
+    pmix_event_assign(&cd->ev, pmix_psensor_base.evbase, -1,
+                      EV_WRITE, del_tracker, cd);
+    pmix_event_active(&cd->ev, EV_WRITE, 1);
+
+    return PMIX_SUCCESS;
+}
+
+static void opcbfunc(pmix_status_t status, void *cbdata)
+{
+    file_tracker_t *ft = (file_tracker_t*)cbdata;
+
+    PMIX_RELEASE(ft);
+}
+
+static void file_sample(int sd, short args, void *cbdata)
+{
+    file_tracker_t *ft = (file_tracker_t*)cbdata;
+    struct stat buf;
+    pmix_status_t rc;
+    pmix_proc_t source;
+
+    OPAL_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                         "[%s:%d] sampling file %s",
+                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                         ft->file));
+
+    /* stat the file and get its info */
+    if (0 > stat(ft->file, &buf)) {
+        /* cannot stat file */
+        PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                             "[%s:%d] could not stat %s",
+                             pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                             ft->file));
+        /* re-add the timer, in case this file shows up */
+        pmix_event_evtimer_add(&ft->ev, &ft->tv);
+        return;
+    }
+
+    PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                         "[%s:%d] size %lu access %s\tmod %s",
+                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                         (unsigned long)buf.st_size, ctime(&buf.st_atime), ctime(&buf.st_mtime)));
+
+    if (ft->file_size) {
+        if (buf.st_size == ft->last_size) {
+            ft->nmisses++;
+        } else {
+            ft->nmisses = 0;
+            ft->last_size = buf.st_size;
+        }
+    } else if (ft->file_access) {
+        if (buf.st_atime == ft->last_access) {
+            ft->nmisses++;
+        } else {
+            ft->nmisses = 0;
+            ft->last_access = buf.st_atime;
+        }
+    } else if (ft->file_mod) {
+        if (buf.st_mtime == ft->last_mod) {
+            ft->nmisses++;
+        } else {
+            ft->nmisses = 0;
+            ft->last_mod = buf.st_mtime;
+        }
+    }
+
+  CHECK:
+    PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                         "[%s:%d] sampled file %s misses %d",
+                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                         ft->file, ft->nmisses));
+
+    if (ft->nmisses == ft->ndrops) {
+        if (4 < pmix_output_get_verbosity(pmix_psensor_base_framework.framework_output)) {
+            pmix_show_help("help-pmix-psensor-file.txt", "file-stalled", true,
+                           ft->file, ft->last_size, ctime(&ft->last_access), ctime(&ft->last_mod));
+        }
+        /* stop monitoring this client */
+        pmix_list_remove_item(&mca_psensor_file_component.trackers, &ft->super);
+        /* generate an event */
+        (void)strncpy(source.nspace, ft->requestor->info->nptr->nspace, PMIX_MAX_NSLEN);
+        source.rank = ft->requestor->info->rank;
+        rc = PMIx_Notify_event(PMIX_MONITOR_FILE_ALERT, &source,
+                               ft->range, ft->info, ft->ninfo, opcbfunc, ft);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+        return;
+    }
+
+    /* re-add the timer */
+    pmix_event_evtimer_add(&ft->ev, &ft->tv);
+}

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * File movement sensor
+ */
+#ifndef PMIX_PSENSOR_FILE_H
+#define PMIX_PSENSOR_FILE_H
+
+#include <src/include/pmix_config.h>
+
+#include "src/class/pmix_list.h"
+
+#include "src/mca/psensor/psensor.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    pmix_psensor_base_component_t super;
+    pmix_list_t trackers;
+} pmix_psensor_file_component_t;
+
+extern pmix_psensor_file_component_t mca_psensor_file_component;
+extern pmix_psensor_base_module_t pmix_psensor_file_module;
+
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file_component.c

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include "src/class/pmix_list.h"
+
+#include "src/mca/psensor/base/base.h"
+#include "src/mca/psensor/file/psensor_file.h"
+
+/*
+ * Local functions
+ */
+static int psensor_file_open(void);
+static int psensor_file_close(void);
+static int psensor_file_query(pmix_mca_base_module_t **module, int *priority);
+
+pmix_psensor_file_component_t mca_psensor_file_component = {
+    .super = {
+        .base = {
+            PMIX_PSENSOR_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+            .pmix_mca_component_name = "file",
+            PMIX_MCA_BASE_MAKE_VERSION(component,
+                                       PMIX_MAJOR_VERSION,
+                                       PMIX_MINOR_VERSION,
+                                       PMIX_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            psensor_file_open,  /* component open  */
+            psensor_file_close, /* component close */
+            psensor_file_query  /* component query */
+        },
+    }
+};
+
+
+static int psensor_file_open(void)
+{
+    PMIX_CONSTRUCT(&mca_psensor_file_component.trackers, pmix_list_t);
+    return PMIX_SUCCESS;
+}
+
+
+static int psensor_file_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 20;  /* irrelevant */
+    *module = (pmix_mca_base_module_t *)&pmix_psensor_file_module;
+    return PMIX_SUCCESS;
+}
+
+/**
+ *  Close all subsystems.
+ */
+
+static int psensor_file_close(void)
+{
+    PMIX_LIST_DESTRUCT(&mca_psensor_file_component.trackers);
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/Makefile.am

@@ -0,0 +1,38 @@
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+#
+# Copyright (c) 2017      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+dist_pmixdata_DATA = help-pmix-psensor-heartbeat.txt
+
+sources = \
+        psensor_heartbeat.c \
+        psensor_heartbeat.h \
+        psensor_heartbeat_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_psensor_heartbeat_DSO
+component_noinst =
+component_install = mca_psensor_heartbeat.la
+else
+component_noinst = libmca_psensor_heartbeat.la
+component_install =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_psensor_heartbeat_la_SOURCES = $(sources)
+mca_psensor_heartbeat_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_psensor_heartbeat_la_SOURCES =$(sources)
+libmca_psensor_heartbeat_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/help-pmix-psensor-heartbeat.txt

@@ -4,9 +4,9 @@
 #
 # Copyright (c) 2017      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
-# 
+#
 # Additional copyrights may follow
-# 
+#
 # $HEADER$
 #
 # This is the US/English general help file for the memory usage sensor
@@ -18,4 +18,3 @@ Node:         %s
 Process rank: %s
 Memory used:  %luGbytes
 Memory limit: %luGbytes
-

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c

@@ -0,0 +1,330 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2012 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+  *
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif  /* HAVE_UNISTD_H */
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif  /* HAVE_STRING_H */
+#include <stdio.h>
+#include <pthread.h>
+#include PMIX_EVENT_HEADER
+
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/util/show_help.h"
+#include "src/include/pmix_globals.h"
+#include "src/mca/ptl/ptl.h"
+
+#include "src/mca/psensor/base/base.h"
+#include "psensor_heartbeat.h"
+
+/* declare the API functions */
+static pmix_status_t heartbeat_start(pmix_peer_t *requestor, pmix_status_t error,
+                                     const pmix_info_t *monitor,
+                                     const pmix_info_t directives[], size_t ndirs);
+static pmix_status_t heartbeat_stop(pmix_peer_t *requestor, char *id);
+
+/* instantiate the module */
+pmix_psensor_base_module_t pmix_psensor_heartbeat_module = {
+    .start = heartbeat_start,
+    .stop = heartbeat_stop
+};
+
+/* tracker object */
+typedef struct {
+    pmix_list_item_t super;
+    pmix_peer_t *requestor;
+    char *id;
+    bool event_active;
+    pmix_event_t ev;
+    pmix_event_t cdev;
+    struct timeval tv;
+    uint32_t nbeats;
+    uint32_t ndrops;
+    uint32_t nmissed;
+    pmix_status_t error;
+    pmix_data_range_t range;
+    pmix_info_t *info;
+    size_t ninfo;
+} pmix_heartbeat_trkr_t;
+
+static void ft_constructor(pmix_heartbeat_trkr_t *ft)
+{
+    ft->requestor = NULL;
+    ft->id = NULL;
+    ft->event_active = false;
+    ft->tv.tv_sec = 0;
+    ft->tv.tv_usec = 0;
+    ft->nbeats = 0;
+    ft->ndrops = 0;
+    ft->nmissed = 0;
+    ft->error = PMIX_SUCCESS;
+    ft->range = PMIX_RANGE_NAMESPACE;
+    ft->info = NULL;
+    ft->ninfo = 0;
+}
+static void ft_destructor(pmix_heartbeat_trkr_t *ft)
+{
+    if (NULL != ft->requestor) {
+        PMIX_RELEASE(ft->requestor);
+    }
+    if (NULL != ft->id) {
+        free(ft->id);
+    }
+    if (event_active) {
+        pmix_event_del(&ft->ev);
+    }
+    if (NULL != ft->info) {
+        PMIX_INFO_FREE(ft->info, ft->ninfo);
+    }
+}
+PMIX_CLASS_INSTANCE(pmix_heartbeat_trkr_t,
+                    pmix_list_item_t,
+                    ft_constructor, ft_destructor);
+
+/* define a local caddy */
+typedef struct {
+    pmix_object_t super;
+    pmix_event_t ev;
+    pmix_peer_t *requestor;
+    char *id;
+} heartbeat_caddy_t;
+static void cd_con(heartbeat_caddy_t *p)
+{
+    p->requestor = NULL;
+    p->id = NULL;
+}
+static void cd_des(heartbeat_caddy_t *p)
+{
+    if (NULL != (p->requestor)) {
+        PMIX_RELEASE(p->requestor);
+    }
+    if (NULL != p->id) {
+        free(p->id);
+    }
+}
+PMIX_CLASS_INSTANCE(heartbeat_caddy_t,
+                    pmix_object_t,
+                    cd_con, cd_des);
+
+typedef struct {
+    pmix_object_t super;
+    pmix_event_t ev;
+    pmix_peer_t *peer;
+} pmix_psensor_beat_t;
+
+static void bcon(pmix_psensor_beat_t *p)
+{
+    p->peer = NULL;
+}
+static void bdes(pmix_psensor_beat_t *p)
+{
+    if (NULL != p->peer) {
+        PMIX_RELEASE(p->peer);
+    }
+}
+PMIX_CLASS_INSTANCE(pmix_psensor_beat_t,
+                    pmix_object_t,
+                    bcon, bdes);
+
+static void check_heartbeat(int fd, short dummy, void *arg);
+
+static void add_tracker(int sd, short flags, void *cbdata)
+{
+    pmix_heartbeat_trkr_t *ft = (pmix_heartbeat_trkr_t*)cbdata;
+
+    /* add the tracker to our list */
+    pmix_list_append(&mca_psensor_heartbeat_component.trackers, &ft->super);
+
+    /* setup the timer event */
+    pmix_event_evtimer_set(pmix_psensor_base.evbase, &ft->ev,
+                           check_heartbeat, ft);
+    pmix_event_evtimer_add(&ft->ev, &ft->tv);
+    ft->event_active = true;
+}
+
+static pmix_status_t heartbeat_start(pmix_peer_t *requestor, pmix_status_t error,
+                                     const pmix_info_t *monitor,
+                                     const pmix_info_t directives[], size_t ndirs)
+{
+    pmix_heartbeat_trkr_t *ft;
+    size_t n, n2;
+
+    PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                         "[%s:%d] checking heartbeat monitoring for requestor %s:%d",
+                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                         requestor->info->nptr->nspace, requestor->info->rank));
+
+    /* if they didn't ask for heartbeats, then nothing for us to do */
+    if (0 != strcmp(monitor->key, PMIX_MONITOR_HEARTBEAT)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* setup to track this monitoring operation */
+    ft = PMIX_NEW(pmix_heartbeat_trkr_t);
+    PMIX_RETAIN(requestor);
+    ft->requestor = requestor;
+    ft->error = error;
+
+    /* check the directives to see what they want monitored */
+    for (n=0; n < ndirs; n++) {
+        if (0 == strcmp(directives[n].key, PMIX_MONITOR_HEARTBEAT_TIME)) {
+            ft->tv.tv_sec = directives[n].value.data.uint32;
+        } else if (0 == strcmp(directives[n].key, PMIX_MONITOR_HEARTBEAT_DROPS)) {
+            ft->ndrops = directives[n].value.data.uint32;
+        } else if (0 == strcmp(directives[n].key, PMIX_RANGE)) {
+            ft->range = directives[n].value.data.range;
+        }
+    }
+
+    if (0 == ft->tv.tv_sec) {
+        /* didn't specify a sample rate, or what should be sampled */
+        PMIX_RELEASE(ft);
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* need to push into our event base to add this to our trackers */
+    pmix_event_assign(&ft->cdev, pmix_psensor_base.evbase, -1,
+                      EV_WRITE, add_tracker, ft);
+    pmix_event_active(&ft->cdev, EV_WRITE, 1);
+
+    return PMIX_SUCCESS;
+}
+
+static void del_tracker(int sd, short flags, void *cbdata)
+{
+    heartbeat_caddy_t *cd = (heartbeat_caddy_t*)cbdata;
+    pmix_heartbeat_trkr_t *ft, *ftnext;
+
+    /* remove the tracker from our list */
+    PMIX_LIST_FOREACH_SAFE(ft, ftnext, &mca_psensor_heartbeat_component.trackers, pmix_heartbeat_trkr_t) {
+        if (ft->requestor != cd->requestor) {
+            continue;
+        }
+        if (NULL == cd->id ||
+            (NULL != ft->id && 0 == strcmp(ft->id, cd->id))) {
+            pmix_list_remove_item(&mca_psensor_heartbeat_component.trackers, &ft->super);
+            PMIX_RELEASE(ft);
+        }
+    }
+    PMIX_RELEASE(cd);
+}
+
+static pmix_status_t heartbeat_stop(pmix_peer_t *requestor, char *id)
+{
+    heartbeat_caddy_t *cd;
+
+    cd = PMIX_NEW(heartbeat_caddy_t);
+    PMIX_RETAIN(requestor);
+    cd->requestor = requestor;
+    cd->id = strdup(id);
+
+    /* need to push into our event base to add this to our trackers */
+    pmix_event_assign(&cd->ev, pmix_psensor_base.evbase, -1,
+                      EV_WRITE, del_tracker, cd);
+    pmix_event_active(&cd->ev, EV_WRITE, 1);
+
+    return PMIX_SUCCESS;
+}
+
+static void opcbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_heartbeat_trkr_t *ft = (pmix_heartbeat_trkr_t*)cbdata;
+
+    PMIX_RELEASE(ft);
+}
+
+/* this function automatically gets periodically called
+ * by the event library so we can check on the state
+ * of the various procs we are monitoring
+ */
+static void check_heartbeat(int fd, short dummy, void *cbdata)
+{
+    pmix_heartbeat_trkr_t *ft = (pmix_heartbeat_trkr_t*)cbdata;
+    pmix_status_t rc;
+    pmix_proc_t source;
+
+    PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                         "[%s:%d] sensor:check_heartbeat for proc %s:%d",
+                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                        ft->requestor->info->nptr->nspace, ft->requestor->info->rank));
+
+    if (0 == ft->nbeats) {
+        /* no heartbeat recvd in last window */
+        PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                             "[%s:%d] sensor:check_heartbeat failed for proc %s:%d",
+                             pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                             ft->requestor->info->nptr->nspace, ft->requestor->info->rank));
+        /* stop monitoring this client */
+        pmix_list_remove_item(&mca_psensor_heartbeat_component.trackers, &ft->super);
+        /* generate an event */
+        (void)strncpy(source.nspace, ft->requestor->info->nptr->nspace, PMIX_MAX_NSLEN);
+        source.rank = ft->requestor->info->rank;
+        rc = PMIx_Notify_event(PMIX_MONITOR_HEARTBEAT_ALERT, &source,
+                               ft->range, ft->info, ft->ninfo, opcbfunc, ft);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+        return;
+    } else {
+        PMIX_OUTPUT_VERBOSE((1, pmix_psensor_base_framework.framework_output,
+                             "[%s:%d] sensor:check_heartbeat detected %d beats for proc %s:%d",
+                             pmix_globals.myid.nspace, pmix_globals.myid.rank, ft->nbeats,
+                             ft->requestor->info->nptr->nspace, ft->requestor->info->rank));
+    }
+    /* reset for next period */
+    ft->nbeats = 0;
+
+    /* reset the timer */
+    pmix_event_evtimer_add(&ft->ev, &ft->tv);
+}
+
+static void add_beat(int sd, short args, void *cbdata)
+{
+    pmix_psensor_beat_t *b = (pmix_psensor_beat_t*)cbdata;
+    pmix_heartbeat_trkr_t *ft;
+
+    /* find this peer in our trackers */
+    PMIX_LIST_FOREACH(ft, &mca_psensor_heartbeat_component.trackers, pmix_heartbeat_trkr_t) {
+        if (ft->requestor == b->peer) {
+            /* increment the beat count */
+            ++ft->nbeats;
+            break;
+        }
+    }
+
+    PMIX_RELEASE(b);
+}
+
+void pmix_psensor_heartbeat_recv_beats(struct pmix_peer_t *peer,
+                                       pmix_ptl_hdr_t *hdr,
+                                       pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_psensor_beat_t *b;
+
+    b = PMIX_NEW(pmix_psensor_beat_t);
+    PMIX_RETAIN(peer);
+    b->peer = peer;
+
+    /* shift this to our thread for processing */
+    pmix_event_assign(&b->ev, pmix_psensor_base.evbase, -1,
+                      EV_WRITE, add_beat, b);
+    pmix_event_active(&b->ev, EV_WRITE, 1);
+}

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
+ *
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * Heartbeat sensor
+ */
+#ifndef PMIX_PSENSOR_HEARTBEAT_H
+#define PMIX_PSENSOR_HEARTBEAT_H
+
+#include <src/include/pmix_config.h>
+#include <src/include/types.h>
+
+#include "src/class/pmix_list.h"
+#include "src/include/pmix_globals.h"
+#include "src/mca/psensor/psensor.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    pmix_psensor_base_component_t super;
+    pmix_list_t trackers;
+} pmix_psensor_heartbeat_component_t;
+
+PMIX_EXPORT extern pmix_psensor_heartbeat_component_t mca_psensor_heartbeat_component;
+extern pmix_psensor_base_module_t pmix_psensor_heartbeat_module;
+
+void pmix_psensor_heartbeat_recv_beats(struct pmix_peer_t *peer,
+                                       pmix_ptl_hdr_t *hdr,
+                                       pmix_buffer_t *buf, void *cbdata);
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat_component.c

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include "src/mca/ptl/ptl.h"
+#include "src/mca/psensor/base/base.h"
+#include "src/mca/psensor/heartbeat/psensor_heartbeat.h"
+
+/*
+ * Local functions
+ */
+
+static int heartbeat_open(void);
+static int heartbeat_close(void);
+static int heartbeat_query(pmix_mca_base_module_t **module, int *priority);
+
+pmix_psensor_heartbeat_component_t mca_psensor_heartbeat_component = {
+    .super = {
+        .base = {
+            PMIX_PSENSOR_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+            .pmix_mca_component_name = "heartbeat",
+            PMIX_MCA_BASE_MAKE_VERSION(component,
+                                       PMIX_MAJOR_VERSION,
+                                       PMIX_MINOR_VERSION,
+                                       PMIX_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            heartbeat_open,  /* component open  */
+            heartbeat_close, /* component close */
+            heartbeat_query  /* component query */
+        }
+    }
+};
+
+
+/**
+  * component open/close/init function
+  */
+static int heartbeat_open(void)
+{
+    PMIX_CONSTRUCT(&mca_psensor_heartbeat_component.trackers, pmix_list_t);
+
+    /* setup to receive heartbeats */
+    pmix_ptl.recv(pmix_globals.mypeer, pmix_psensor_heartbeat_recv_beats, PMIX_PTL_TAG_HEARTBEAT);
+
+    return PMIX_SUCCESS;
+}
+
+
+static int heartbeat_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 5;  // irrelevant
+    *module = (pmix_mca_base_module_t *)&pmix_psensor_heartbeat_module;
+    return PMIX_SUCCESS;
+}
+
+/**
+ *  Close all subsystems.
+ */
+
+static int heartbeat_close(void)
+{
+    /* cancel our persistent recv */
+    pmix_ptl.cancel(pmix_globals.mypeer, PMIX_PTL_TAG_HEARTBEAT);
+
+    PMIX_LIST_DESTRUCT(&mca_psensor_heartbeat_component.trackers);
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix2x/pmix/src/mca/psensor/psensor.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * @file:
+ *
+ */
+
+#ifndef PMIX_PSENSOR_H_
+#define PMIX_PSENSOR_H_
+
+#include <src/include/pmix_config.h>
+
+#include "src/class/pmix_list.h"
+#include "src/mca/mca.h"
+#include "src/include/pmix_globals.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Component functions - all MUST be provided!
+ */
+
+/* start a sensor operation:
+ *
+ * requestor - the process requesting this operation
+ *
+ * monitor - a PMIx attribute specifying what is to be monitored
+ *
+ * directives - an array of pmix_info_t specifying relevant limits on values, and action
+ *              to be taken when limits exceeded. Can include
+ *              user-provided "id" string */
+typedef pmix_status_t (*pmix_psensor_base_module_start_fn_t)(pmix_peer_t *requestor, pmix_status_t error,
+                                                             const pmix_info_t *monitor,
+                                                             const pmix_info_t directives[], size_t ndirs);
+
+/* stop a sensor operation:
+ *
+ * requestor - the process requesting this operation
+ *
+ * id - the "id" string provided by the user at the time the
+ *      affected monitoring operation was started. A NULL indicates
+ *      that all operations started by this requestor are to
+ *      be terminated */
+typedef pmix_status_t (*pmix_psensor_base_module_stop_fn_t)(pmix_peer_t *requestor,
+                                                            char *id);
+
+/* API module */
+/*
+ * Ver 1.0
+ */
+typedef struct pmix_psensor_base_module_1_0_0_t {
+    pmix_psensor_base_module_start_fn_t      start;
+    pmix_psensor_base_module_stop_fn_t       stop;
+} pmix_psensor_base_module_t;
+
+/*
+ * the standard component data structure
+ */
+typedef struct pmix_psensor_base_component_1_0_0_t {
+    pmix_mca_base_component_t base;
+    pmix_mca_base_component_data_t data;
+} pmix_psensor_base_component_t;
+
+
+
+/*
+ * Macro for use in components that are of type sensor v1.0.0
+ */
+#define PMIX_PSENSOR_BASE_VERSION_1_0_0 \
+  PMIX_MCA_BASE_VERSION_1_0_0("psensor", 1, 0, 0)
+
+/* Global structure for accessing sensor functions
+ */
+PMIX_EXPORT extern pmix_psensor_base_module_t pmix_psensor;  /* holds API function pointers */
+
+END_C_DECLS
+
+#endif /* MCA_SENSOR_H */

opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -73,6 +73,7 @@ struct pmix_ptl_globals_t {
     pmix_list_t actives;
     bool initialized;
     pmix_list_t posted_recvs;     // list of pmix_ptl_posted_recv_t
+    pmix_list_t unexpected_msgs;
     int stop_thread[2];
     bool listen_thread_active;
     pmix_list_t listeners;
@@ -93,6 +94,11 @@ PMIX_EXPORT pmix_status_t pmix_ptl_stub_send_oneway(struct pmix_peer_t *peer,
                                                     pmix_ptl_tag_t tag);
 PMIX_EXPORT pmix_status_t pmix_ptl_stub_connect_to_peer(struct pmix_peer_t *peer,
                                                         pmix_info_t info[], size_t ninfo);
+PMIX_EXPORT pmix_status_t pmix_ptl_stub_register_recv(struct pmix_peer_t *peer,
+                                                      pmix_ptl_cbfunc_t cbfunc,
+                                                      pmix_ptl_tag_t tag);
+PMIX_EXPORT pmix_status_t pmix_ptl_stub_cancel_recv(struct pmix_peer_t *peer,
+                                                    pmix_ptl_tag_t tag);
 
 PMIX_EXPORT pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo);
 PMIX_EXPORT void pmix_ptl_base_stop_listening(void);

opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c

@@ -61,6 +61,8 @@ pmix_ptl_API_t pmix_ptl = {
     .send_recv = pmix_ptl_stub_send_recv,
     .send_oneway = pmix_ptl_stub_send_oneway,
     .connect_to_peer = pmix_ptl_stub_connect_to_peer,
+    .recv = pmix_ptl_stub_register_recv,
+    .cancel = pmix_ptl_stub_cancel_recv,
     .start_listening = pmix_ptl_base_start_listening,
     .stop_listening = pmix_ptl_base_stop_listening
 };
@@ -88,6 +90,7 @@ static pmix_status_t pmix_ptl_close(void)
     /* the components will cleanup when closed */
     PMIX_LIST_DESTRUCT(&pmix_ptl_globals.actives);
     PMIX_LIST_DESTRUCT(&pmix_ptl_globals.posted_recvs);
+    PMIX_LIST_DESTRUCT(&pmix_ptl_globals.unexpected_msgs);
     PMIX_LIST_DESTRUCT(&pmix_ptl_globals.listeners);
 
     return pmix_mca_base_framework_components_close(&pmix_ptl_base_framework, NULL);
@@ -99,6 +102,7 @@ static pmix_status_t pmix_ptl_open(pmix_mca_base_open_flag_t flags)
     pmix_ptl_globals.initialized = true;
     PMIX_CONSTRUCT(&pmix_ptl_globals.actives, pmix_list_t);
     PMIX_CONSTRUCT(&pmix_ptl_globals.posted_recvs, pmix_list_t);
+    PMIX_CONSTRUCT(&pmix_ptl_globals.unexpected_msgs, pmix_list_t);
     pmix_ptl_globals.listen_thread_active = false;
     PMIX_CONSTRUCT(&pmix_ptl_globals.listeners, pmix_list_t);
     pmix_client_globals.myserver.sd = -1;

opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c

@@ -46,7 +46,7 @@
 
 #include "src/mca/ptl/base/base.h"
 
-static uint32_t current_tag = 1;  // 0 is reserved for system purposes
+static uint32_t current_tag = PMIX_PTL_TAG_DYNAMIC;
 
 static void _notify_complete(pmix_status_t status, void *cbdata)
 {
@@ -162,7 +162,7 @@ static pmix_status_t send_msg(int sd, pmix_ptl_send_t *msg)
     } else {
         iov_count = 1;
     }
-retry:
+  retry:
     rc = writev(sd, iov, iov_count);
     if (PMIX_LIKELY(rc == remain)) {
         /* we successfully sent the header and the msg data if any */
@@ -521,16 +521,16 @@ void pmix_ptl_base_send_recv(int fd, short args, void *cbdata)
         return;
     }
 
-    /* set the tag */
-    tag = current_tag++;
+    /* take the next tag in the sequence */
+    current_tag++;
+    if (UINT32_MAX == current_tag ) {
+        current_tag = PMIX_PTL_TAG_DYNAMIC;
+    }
+    tag = current_tag;
 
     if (NULL != ms->cbfunc) {
         /* if a callback msg is expected, setup a recv for it */
         req = PMIX_NEW(pmix_ptl_posted_recv_t);
-        /* take the next tag in the sequence */
-        if (UINT32_MAX == current_tag ) {
-            current_tag = 1;
-        }
         req->tag = tag;
         req->cbfunc = ms->cbfunc;
         req->cbdata = ms->cbdata;
@@ -597,23 +597,29 @@ void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata)
                     buf.pack_ptr = ((char*)buf.base_ptr) + buf.bytes_used;
                 }
                 msg->data = NULL;  // protect the data region
-                if (NULL != rcv->cbfunc) {
-                    rcv->cbfunc(msg->peer, &msg->hdr, &buf, rcv->cbdata);
-                }
+                rcv->cbfunc(msg->peer, &msg->hdr, &buf, rcv->cbdata);
                 PMIX_DESTRUCT(&buf);  // free's the msg data
-                /* also done with the recv, if not a wildcard or the error tag */
-                if (UINT32_MAX != rcv->tag && 0 != rcv->tag) {
-                    pmix_list_remove_item(&pmix_ptl_globals.posted_recvs, &rcv->super);
-                    PMIX_RELEASE(rcv);
-                }
-                PMIX_RELEASE(msg);
-                return;
             }
+            /* done with the recv if it is a dynamic tag */
+            if (PMIX_PTL_TAG_DYNAMIC <= rcv->tag && UINT_MAX != rcv->tag) {
+                pmix_list_remove_item(&pmix_ptl_globals.posted_recvs, &rcv->super);
+                PMIX_RELEASE(rcv);
+            }
+            PMIX_RELEASE(msg);
+            return;
         }
     }
 
-    /* we get here if no matching recv was found - this is an error */
-    pmix_output(0, "UNEXPECTED MESSAGE tag = %d", msg->hdr.tag);
-    PMIX_RELEASE(msg);
-    PMIX_REPORT_EVENT(PMIX_ERROR, _notify_complete);
+    /* if the tag in this message is above the dynamic marker, then
+     * that is an error */
+    if (PMIX_PTL_TAG_DYNAMIC <= msg->hdr.tag) {
+        pmix_output(0, "UNEXPECTED MESSAGE tag = %d", msg->hdr.tag);
+        PMIX_RELEASE(msg);
+        PMIX_REPORT_EVENT(PMIX_ERROR, _notify_complete);
+        return;
+    }
+
+    /* it is possible that someone may post a recv for this message
+     * at some point, so we have to hold onto it */
+    pmix_list_append(&pmix_ptl_globals.unexpected_msgs, &msg->super);
 }

opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -105,3 +105,92 @@ pmix_status_t pmix_ptl_stub_connect_to_peer(struct pmix_peer_t *peer,
 
     return PMIX_ERR_UNREACH;
 }
+
+static void post_recv(int fd, short args, void *cbdata)
+{
+    pmix_ptl_posted_recv_t *req = (pmix_ptl_posted_recv_t*)cbdata;
+    pmix_ptl_recv_t *msg, *nmsg;
+    pmix_buffer_t buf;
+
+    pmix_output_verbose(5, pmix_globals.debug_output,
+                        "posting recv on tag %d", req->tag);
+
+    /* add it to the list of recvs */
+    pmix_list_append(&pmix_ptl_globals.posted_recvs, &req->super);
+
+    /* now check the unexpected msg queue to see if we already
+     * recvd something for it */
+    PMIX_LIST_FOREACH_SAFE(msg, nmsg, &pmix_ptl_globals.unexpected_msgs, pmix_ptl_recv_t) {
+        if (msg->hdr.tag == req->tag || UINT_MAX == req->tag) {
+            if (NULL != req->cbfunc) {
+                /* construct and load the buffer */
+                PMIX_CONSTRUCT(&buf, pmix_buffer_t);
+                if (NULL != msg->data) {
+                    buf.base_ptr = (char*)msg->data;
+                    buf.bytes_allocated = buf.bytes_used = msg->hdr.nbytes;
+                    buf.unpack_ptr = buf.base_ptr;
+                    buf.pack_ptr = ((char*)buf.base_ptr) + buf.bytes_used;
+                }
+                msg->data = NULL;  // protect the data region
+                req->cbfunc(msg->peer, &msg->hdr, &buf, req->cbdata);
+                PMIX_DESTRUCT(&buf);  // free's the msg data
+            }
+            pmix_list_remove_item(&pmix_ptl_globals.unexpected_msgs, &msg->super);
+            PMIX_RELEASE(msg);
+        }
+    }
+}
+
+pmix_status_t pmix_ptl_stub_register_recv(struct pmix_peer_t *peer,
+                                          pmix_ptl_cbfunc_t cbfunc,
+                                          pmix_ptl_tag_t tag)
+{
+    pmix_ptl_posted_recv_t *req;
+
+    req = PMIX_NEW(pmix_ptl_posted_recv_t);
+    if (NULL == req) {
+        return PMIX_ERR_NOMEM;
+    }
+    req->tag = tag;
+    req->cbfunc = cbfunc;
+    /* have to push this into an event so we can add this
+     * to the list of posted recvs */
+    pmix_event_assign(&(req->ev), pmix_globals.evbase, -1,
+                      EV_WRITE, post_recv, req);
+    pmix_event_active(&(req->ev), EV_WRITE, 1);
+    return PMIX_SUCCESS;
+}
+
+static void cancel_recv(int fd, short args, void *cbdata)
+{
+    pmix_ptl_posted_recv_t *req = (pmix_ptl_posted_recv_t*)cbdata;
+    pmix_ptl_posted_recv_t *rcv;
+
+    PMIX_LIST_FOREACH(rcv, &pmix_ptl_globals.posted_recvs, pmix_ptl_posted_recv_t) {
+        if (rcv->tag == req->tag) {
+            pmix_list_remove_item(&pmix_ptl_globals.posted_recvs, &rcv->super);
+            PMIX_RELEASE(rcv);
+            PMIX_RELEASE(req);
+            return;
+        }
+    }
+    PMIX_RELEASE(req);
+}
+
+pmix_status_t pmix_ptl_stub_cancel_recv(struct pmix_peer_t *peer,
+                                        pmix_ptl_tag_t tag)
+{
+    pmix_ptl_posted_recv_t *req;
+
+    req = PMIX_NEW(pmix_ptl_posted_recv_t);
+    if (NULL == req) {
+        return PMIX_ERR_NOMEM;
+    }
+    req->tag = tag;
+    /* have to push this into an event so we can modify
+     * the list of posted recvs */
+    pmix_event_assign(&(req->ev), pmix_globals.evbase, -1,
+                      EV_WRITE, cancel_recv, req);
+    pmix_event_active(&(req->ev), EV_WRITE, 1);
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
@@ -110,6 +110,15 @@ typedef pmix_status_t (*pmix_ptl_send_fn_t)(struct pmix_peer_t *peer,
                                             pmix_buffer_t *bfr,
                                             pmix_ptl_tag_t tag);
 
+/* (ONE-WAY) register a persistent recv */
+typedef pmix_status_t (*pmix_ptl_recv_fn_t)(struct pmix_peer_t *peer,
+                                            pmix_ptl_cbfunc_t cbfunc,
+                                            pmix_ptl_tag_t tag);
+
+/* Cancel a persistent recv */
+typedef pmix_status_t (*pmix_ptl_cancel_fn_t)(struct pmix_peer_t *peer,
+                                              pmix_ptl_tag_t tag);
+
 /* connect to a peer - this is a blocking function
  * to establish a connection to a peer. It assigns
  * the corresponding module to the peer's compat
@@ -126,6 +135,8 @@ struct pmix_ptl_module_t {
     pmix_ptl_finalize_fn_t              finalize;
     pmix_ptl_send_recv_fn_t             send_recv;
     pmix_ptl_send_fn_t                  send;
+    pmix_ptl_recv_fn_t                  recv;
+    pmix_ptl_cancel_fn_t                cancel;
     pmix_ptl_connect_to_peer_fn_t       connect_to_peer;
 };
 typedef struct pmix_ptl_module_t pmix_ptl_module_t;
@@ -152,6 +163,8 @@ typedef struct {
     pmix_ptl_get_available_modules_fn_t     get_available_modules;
     pmix_ptl_send_recv_fn_t                 send_recv;
     pmix_ptl_send_fn_t                      send_oneway;
+    pmix_ptl_recv_fn_t                      recv;
+    pmix_ptl_cancel_fn_t                    cancel;
     pmix_ptl_connect_to_peer_fn_t           connect_to_peer;
     pmix_ptl_start_listening_fn_t           start_listening;
     pmix_ptl_stop_listening_fn_t            stop_listening;

opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h

@@ -63,6 +63,16 @@ struct pmix_ptl_module_t;
 
 /****    MESSAGING STRUCTURES    ****/
 typedef uint32_t pmix_ptl_tag_t;
+/* define a range of "reserved" tags - these
+ * are tags that are used for persistent recvs
+ * within the system */
+#define PMIX_PTL_TAG_NOTIFY           0
+#define PMIX_PTL_TAG_HEARTBEAT        1
+
+/* define the start of dynamic tags that are
+ * assigned for send/recv operations */
+#define PMIX_PTL_TAG_DYNAMIC        100
+
 
 /* header for messages */
 typedef struct {

opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -13,6 +13,11 @@
 
 #include "pmix_config.h"
 
+#include <pthread.h>
+#include PMIX_EVENT_HEADER
+
+#include "src/include/types.h"
+
 /**
  * Initialize a progress thread name; if a progress thread is not
  * already associated with that name, start a progress thread.

opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c

@@ -2345,6 +2345,18 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag,
         return rc;
     }
 
+    if (PMIX_JOB_CONTROL_CMD == cmd) {
+        PMIX_PEER_CADDY(cd, peer, tag);
+        rc = pmix_server_job_ctrl(peer, buf, query_cbfunc, cd);
+        return rc;
+    }
+
+    if (PMIX_MONITOR_CMD == cmd) {
+        PMIX_PEER_CADDY(cd, peer, tag);
+        rc = pmix_server_monitor(peer, buf, query_cbfunc, cd);
+        return rc;
+    }
+
     return PMIX_ERR_NOT_SUPPORTED;
 }
 

opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c

@@ -1562,6 +1562,134 @@ pmix_status_t pmix_server_alloc(pmix_peer_t *peer,
     return rc;
 }
 
+pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer,
+                                   pmix_buffer_t *buf,
+                                   pmix_info_cbfunc_t cbfunc,
+                                   void *cbdata)
+{
+    int32_t cnt;
+    pmix_status_t rc;
+    pmix_query_caddy_t *cd;
+    pmix_proc_t proc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "recvd job control request from client");
+
+    if (NULL == pmix_host_server.job_control) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    cd = PMIX_NEW(pmix_query_caddy_t);
+    cd->cbdata = cbdata;
+
+    /* unpack the number of targets */
+    cnt = 1;
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &cd->ntargets, &cnt, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+    if (0 < cd->ntargets) {
+        PMIX_PROC_CREATE(cd->targets, cd->ntargets);
+        cnt = cd->ntargets;
+        if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, cd->targets, &cnt, PMIX_PROC))) {
+            PMIX_ERROR_LOG(rc);
+            goto exit;
+        }
+    }
+    /* unpack the number of info objects */
+    cnt = 1;
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &cd->ninfo, &cnt, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+    /* unpack the info */
+    if (0 < cd->ninfo) {
+        PMIX_INFO_CREATE(cd->info, cd->ninfo);
+        cnt = cd->ninfo;
+        if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, cd->info, &cnt, PMIX_INFO))) {
+            PMIX_ERROR_LOG(rc);
+            goto exit;
+        }
+    }
+
+    /* setup the requesting peer name */
+    (void)strncpy(proc.nspace, peer->info->nptr->nspace, PMIX_MAX_NSLEN);
+    proc.rank = peer->info->rank;
+
+    /* ask the host to execute the request */
+    if (PMIX_SUCCESS != (rc = pmix_host_server.job_control(&proc,
+                                                           cd->targets, cd->ntargets,
+                                                           cd->info, cd->ninfo,
+                                                           cbfunc, cd))) {
+        goto exit;
+    }
+    return PMIX_SUCCESS;
+
+  exit:
+    PMIX_RELEASE(cd);
+    return rc;
+}
+
+pmix_status_t pmix_server_monitor(pmix_peer_t *peer,
+                                  pmix_buffer_t *buf,
+                                  pmix_info_cbfunc_t cbfunc,
+                                  void *cbdata)
+{
+    int32_t cnt;
+    pmix_status_t rc, error;
+    pmix_query_caddy_t *cd;
+    pmix_proc_t proc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "recvd monitor request from client");
+
+    if (NULL == pmix_host_server.monitor) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    cd = PMIX_NEW(pmix_query_caddy_t);
+    cd->cbdata = cbdata;
+
+    /* unpack the error code */
+    cnt = 1;
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &error, &cnt, PMIX_STATUS))) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+
+    /* unpack the number of directives */
+    cnt = 1;
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, &cd->ninfo, &cnt, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+    /* unpack the directives */
+    if (0 < cd->ninfo) {
+        PMIX_INFO_CREATE(cd->info, cd->ninfo);
+        cnt = cd->ninfo;
+        if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(buf, cd->info, &cnt, PMIX_INFO))) {
+            PMIX_ERROR_LOG(rc);
+            goto exit;
+        }
+    }
+
+    /* setup the requesting peer name */
+    (void)strncpy(proc.nspace, peer->info->nptr->nspace, PMIX_MAX_NSLEN);
+    proc.rank = peer->info->rank;
+
+    /* ask the host to execute the request */
+    if (PMIX_SUCCESS != (rc = pmix_host_server.monitor(&proc, error,
+                                                       cd->info, cd->ninfo,
+                                                       cbfunc, cd))) {
+        goto exit;
+    }
+    return PMIX_SUCCESS;
+
+  exit:
+    PMIX_RELEASE(cd);
+    return rc;
+}
+
 /*****    INSTANCE SERVER LIBRARY CLASSES    *****/
 static void tcon(pmix_server_trkr_t *t)
 {

opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h

@@ -218,6 +218,16 @@ pmix_status_t pmix_server_alloc(pmix_peer_t *peer,
                                 pmix_info_cbfunc_t cbfunc,
                                 void *cbdata);
 
+pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer,
+                                   pmix_buffer_t *buf,
+                                   pmix_info_cbfunc_t cbfunc,
+                                   void *cbdata);
+
+pmix_status_t pmix_server_monitor(pmix_peer_t *peer,
+                                  pmix_buffer_t *buf,
+                                  pmix_info_cbfunc_t cbfunc,
+                                  void *cbdata);
+
 pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer,
                                                   pmix_buffer_t *buf,
                                                   pmix_op_cbfunc_t cbfunc,

opal/mca/pmix/pmix2x/pmix/src/util/error.c

@@ -56,6 +56,8 @@ PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t errnum)
         return "INVALID-KEYVAL";
     case PMIX_ERR_INVALID_NUM_PARSED:
         return "INVALID-NUM-PARSED";
+    case PMIX_ERR_TAKE_NEXT_OPTION:
+        return "TAKE-NEXT-OPTION";
 
     case PMIX_ERR_INVALID_ARGS:
         return "INVALID-ARGS";
@@ -157,6 +159,14 @@ PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t errnum)
         return "PMIX_ERR_WILDCARD";
     case PMIX_NOTIFY_ALLOC_COMPLETE:
         return "PMIX ALLOC OPERATION COMPLETE";
+    case PMIX_JCTRL_CHECKPOINT:
+        return "PMIX JOB CONTROL CHECKPOINT";
+    case PMIX_JCTRL_PREEMPT_ALERT:
+        return "PMIX PRE-EMPTION ALERT";
+    case PMIX_MONITOR_HEARTBEAT_ALERT:
+        return "PMIX HEARTBEAT ALERT";
+    case PMIX_MONITOR_FILE_ALERT:
+        return "PMIX FILE MONITOR ALERT";
     case PMIX_SUCCESS:
         return "SUCCESS";
     default:

opal/mca/pmix/pmix2x/pmix/src/util/error.h

@@ -37,6 +37,7 @@
 #define PMIX_ERR_NETWORK_NOT_PARSEABLE                  (PMIX_INTERNAL_ERR_BASE - 33)
 #define PMIX_ERR_FILE_OPEN_FAILURE                      (PMIX_INTERNAL_ERR_BASE - 34)
 #define PMIX_ERR_FILE_READ_FAILURE                      (PMIX_INTERNAL_ERR_BASE - 35)
+#define PMIX_ERR_TAKE_NEXT_OPTION                       (PMIX_INTERNAL_ERR_BASE - 36)
 
 #define PMIX_ERROR_LOG(r)                                           \
  do {                                                               \

opal/mca/pmix/pmix2x/pmix2x.c

@@ -493,6 +493,12 @@ int pmix2x_convert_rc(pmix_status_t rc)
     case PMIX_QUERY_PARTIAL_SUCCESS:
         return OPAL_ERR_PARTIAL_SUCCESS;
 
+    case PMIX_MONITOR_HEARTBEAT_ALERT:
+        return OPAL_ERR_HEARTBEAT_ALERT;
+
+    case PMIX_MONITOR_FILE_ALERT:
+        return OPAL_ERR_FILE_ALERT;
+
     case PMIX_ERROR:
         return OPAL_ERROR;
     case PMIX_SUCCESS:
@@ -1333,6 +1339,22 @@ static void pmix2x_log(opal_list_t *info,
     OBJ_RELEASE(cd);
 }
 
+opal_pmix_alloc_directive_t pmix2x_convert_allocdir(pmix_alloc_directive_t dir)
+{
+    switch (dir) {
+        case PMIX_ALLOC_NEW:
+            return OPAL_PMIX_ALLOC_NEW;
+        case PMIX_ALLOC_EXTEND:
+            return OPAL_PMIX_ALLOC_EXTEND;
+        case PMIX_ALLOC_RELEASE:
+            return OPAL_PMIX_ALLOC_RELEASE;
+        case PMIX_ALLOC_REAQUIRE:
+            return OPAL_PMIX_ALLOC_REAQCUIRE;
+        default:
+            return OPAL_PMIX_ALLOC_UNDEF;
+    }
+}
+
 /****  INSTANTIATE INTERNAL CLASSES  ****/
 OBJ_CLASS_INSTANCE(opal_pmix2x_jobid_trkr_t,
                    opal_list_item_t,

opal/mca/pmix/pmix2x/pmix2x.h

@@ -279,6 +279,8 @@ OPAL_MODULE_DECLSPEC void pmix2x_value_load(pmix_value_t *v,
 OPAL_MODULE_DECLSPEC int pmix2x_value_unload(opal_value_t *kv,
                                             const pmix_value_t *v);
 
+OPAL_MODULE_DECLSPEC opal_pmix_alloc_directive_t pmix2x_convert_allocdir(pmix_alloc_directive_t dir);
+
 END_C_DECLS
 
 #endif /* MCA_PMIX_EXTERNAL_H */

opal/mca/pmix/pmix2x/pmix2x_server_north.c

@@ -45,63 +45,73 @@
 /* These are the interfaces used by the embedded PMIx server
  * to call up into ORTE for service requests */
 
- static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object,
-                                                 pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_client_finalized_fn(const pmix_proc_t *proc, void* server_object,
-                                                 pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_abort_fn(const pmix_proc_t *proc, void *server_object,
-                                      int status, const char msg[],
-                                      pmix_proc_t procs[], size_t nprocs,
-                                      pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs,
-                                        const pmix_info_t info[], size_t ninfo,
-                                        char *data, size_t ndata,
-                                        pmix_modex_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *proc,
-                                           const pmix_info_t info[], size_t ninfo,
-                                           pmix_modex_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_publish_fn(const pmix_proc_t *proc,
-                                        const pmix_info_t info[], size_t ninfo,
-                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_lookup_fn(const pmix_proc_t *proc,  char **keys,
+static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object,
+                                                pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_client_finalized_fn(const pmix_proc_t *proc, void* server_object,
+                                                pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_abort_fn(const pmix_proc_t *proc, void *server_object,
+                                     int status, const char msg[],
+                                     pmix_proc_t procs[], size_t nprocs,
+                                     pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs,
                                        const pmix_info_t info[], size_t ninfo,
-                                       pmix_lookup_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_unpublish_fn(const pmix_proc_t *proc, char **keys,
+                                       char *data, size_t ndata,
+                                       pmix_modex_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *proc,
+                                          const pmix_info_t info[], size_t ninfo,
+                                          pmix_modex_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_publish_fn(const pmix_proc_t *proc,
+                                       const pmix_info_t info[], size_t ninfo,
+                                       pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_lookup_fn(const pmix_proc_t *proc,  char **keys,
+                                      const pmix_info_t info[], size_t ninfo,
+                                      pmix_lookup_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_unpublish_fn(const pmix_proc_t *proc, char **keys,
+                                         const pmix_info_t info[], size_t ninfo,
+                                         pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_spawn_fn(const pmix_proc_t *proc,
+                                     const pmix_info_t job_info[], size_t ninfo,
+                                     const pmix_app_t apps[], size_t napps,
+                                     pmix_spawn_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs,
+                                       const pmix_info_t info[], size_t ninfo,
+                                       pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs,
                                           const pmix_info_t info[], size_t ninfo,
                                           pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_spawn_fn(const pmix_proc_t *proc,
-                                      const pmix_info_t job_info[], size_t ninfo,
-                                      const pmix_app_t apps[], size_t napps,
-                                      pmix_spawn_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs,
-                                        const pmix_info_t info[], size_t ninfo,
-                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs,
-                                           const pmix_info_t info[], size_t ninfo,
-                                           pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes,
-                                             const pmix_info_t info[], size_t ninfo,
-                                             pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes,
-                                               pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_notify_event(pmix_status_t code,
-                                          const pmix_proc_t *source,
-                                          pmix_data_range_t range,
-                                          pmix_info_t info[], size_t ninfo,
-                                          pmix_op_cbfunc_t cbfunc, void *cbdata);
- static pmix_status_t server_query(pmix_proc_t *proct,
-                                   pmix_query_t *queryies, size_t nqueries,
-                                   pmix_info_cbfunc_t cbfunc,
+static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes,
+                                            const pmix_info_t info[], size_t ninfo,
+                                            pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes,
+                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_notify_event(pmix_status_t code,
+                                         const pmix_proc_t *source,
+                                         pmix_data_range_t range,
+                                         pmix_info_t info[], size_t ninfo,
+                                         pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t server_query(pmix_proc_t *proct,
+                                  pmix_query_t *queryies, size_t nqueries,
+                                  pmix_info_cbfunc_t cbfunc,
+                                  void *cbdata);
+static void server_tool_connection(pmix_info_t *info, size_t ninfo,
+                                   pmix_tool_connection_cbfunc_t cbfunc,
                                    void *cbdata);
- static void server_tool_connection(pmix_info_t *info, size_t ninfo,
-                                    pmix_tool_connection_cbfunc_t cbfunc,
-                                    void *cbdata);
 static void server_log(const pmix_proc_t *client,
                        const pmix_info_t data[], size_t ndata,
                        const pmix_info_t directives[], size_t ndirs,
                        pmix_op_cbfunc_t cbfunc, void *cbdata);
 
-  pmix_server_module_t mymodule = {
+static pmix_status_t server_allocate(const pmix_proc_t *client,
+                                     pmix_alloc_directive_t directive,
+                                     const pmix_info_t data[], size_t ndata,
+                                     pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+static pmix_status_t server_job_control(const pmix_proc_t *requestor,
+                                        const pmix_proc_t targets[], size_t ntargets,
+                                        const pmix_info_t directives[], size_t ndirs,
+                                        pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+pmix_server_module_t mymodule = {
     .client_connected = server_client_connected_fn,
     .client_finalized = server_client_finalized_fn,
     .abort = server_abort_fn,
@@ -118,7 +128,11 @@ static void server_log(const pmix_proc_t *client,
     .notify_event = server_notify_event,
     .query = server_query,
     .tool_connected = server_tool_connection,
-    .log = server_log
+    .log = server_log,
+    .allocate = server_allocate,
+    .job_control = server_job_control
+    /* we do not support monitoring, but use the
+     * PMIx internal monitoring capability */
 };
 
 opal_pmix_server_module_t *host_module = NULL;
@@ -1052,3 +1066,117 @@ static void server_log(const pmix_proc_t *proct,
                      &opalcaddy->apps,
                      opal_opcbfunc, opalcaddy);
 }
+
+static pmix_status_t server_allocate(const pmix_proc_t *proct,
+                                     pmix_alloc_directive_t directive,
+                                     const pmix_info_t data[], size_t ndata,
+                                     pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix2x_opalcaddy_t *opalcaddy;
+    opal_process_name_t requestor;
+    int rc;
+    size_t n;
+    opal_value_t *oinfo;
+    opal_pmix_alloc_directive_t odir;
+
+    if (NULL == host_module || NULL == host_module->allocate) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* setup the caddy */
+    opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t);
+    opalcaddy->infocbfunc = cbfunc;
+    opalcaddy->cbdata = cbdata;
+
+    /* convert the requestor */
+    if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) {
+        OBJ_RELEASE(opalcaddy);
+        return pmix2x_convert_opalrc(rc);
+    }
+    requestor.vpid = pmix2x_convert_rank(proct->rank);
+
+    /* convert the directive */
+    odir = pmix2x_convert_allocdir(directive);
+
+    /* convert the data */
+    for (n=0; n < ndata; n++) {
+        oinfo = OBJ_NEW(opal_value_t);
+        opal_list_append(&opalcaddy->info, &oinfo->super);
+        if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &data[n].value))) {
+            OBJ_RELEASE(opalcaddy);
+            return pmix2x_convert_opalrc(rc);
+        }
+    }
+
+    /* pass the call upwards */
+    if (OPAL_SUCCESS != (rc = host_module->allocate(&requestor, odir,
+                                                    &opalcaddy->info,
+                                                    info_cbfunc, opalcaddy))) {
+        OBJ_RELEASE(opalcaddy);
+        return pmix2x_convert_opalrc(rc);
+    }
+
+    return PMIX_SUCCESS;
+
+}
+
+static pmix_status_t server_job_control(const pmix_proc_t *proct,
+                                        const pmix_proc_t targets[], size_t ntargets,
+                                        const pmix_info_t directives[], size_t ndirs,
+                                        pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix2x_opalcaddy_t *opalcaddy;
+    opal_process_name_t requestor;
+    int rc;
+    size_t n;
+    opal_value_t *oinfo;
+    opal_namelist_t *nm;
+
+    if (NULL == host_module || NULL == host_module->job_control) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* setup the caddy */
+    opalcaddy = OBJ_NEW(pmix2x_opalcaddy_t);
+    opalcaddy->infocbfunc = cbfunc;
+    opalcaddy->cbdata = cbdata;
+
+    /* convert the requestor */
+    if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) {
+        OBJ_RELEASE(opalcaddy);
+        return pmix2x_convert_opalrc(rc);
+    }
+    requestor.vpid = pmix2x_convert_rank(proct->rank);
+
+    /* convert the targets */
+    for (n=0; n < ntargets; n++) {
+        nm = OBJ_NEW(opal_namelist_t);
+        opal_list_append(&opalcaddy->procs, &nm->super);
+        if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, targets[n].nspace))) {
+            OBJ_RELEASE(opalcaddy);
+            return pmix2x_convert_opalrc(rc);
+        }
+        nm->name.vpid = pmix2x_convert_rank(targets[n].rank);
+    }
+
+    /* convert the directives */
+    for (n=0; n < ndirs; n++) {
+        oinfo = OBJ_NEW(opal_value_t);
+        opal_list_append(&opalcaddy->info, &oinfo->super);
+        if (OPAL_SUCCESS != (rc = pmix2x_value_unload(oinfo, &directives[n].value))) {
+            OBJ_RELEASE(opalcaddy);
+            return pmix2x_convert_opalrc(rc);
+        }
+    }
+
+    /* pass the call upwards */
+    if (OPAL_SUCCESS != (rc = host_module->job_control(&requestor,
+                                                       &opalcaddy->procs,
+                                                       &opalcaddy->info,
+                                                       info_cbfunc, opalcaddy))) {
+        OBJ_RELEASE(opalcaddy);
+        return pmix2x_convert_opalrc(rc);
+    }
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix_server.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -230,6 +230,19 @@ typedef void (*opal_pmix_connection_cbfunc_t)(int incoming_sd);
 typedef int (*opal_pmix_server_listener_fn_t)(int listening_sd,
                                               opal_pmix_connection_cbfunc_t cbfunc);
 
+/* Request allocation modifications on behalf of a client */
+typedef int (*opal_pmix_server_alloc_fn_t)(const opal_process_name_t *client,
+                                           opal_pmix_alloc_directive_t directive,
+                                           opal_list_t *data,
+                                           opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Execute a job control action on behalf of a client */
+typedef int (*opal_pmix_server_job_control_fn_t)(const opal_process_name_t *requestor,
+                                                 opal_list_t *targets, opal_list_t *directives,
+                                                 opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* we do not provide a monitoring capability */
+
 typedef struct opal_pmix_server_module_1_0_0_t {
     opal_pmix_server_client_connected_fn_t      client_connected;
     opal_pmix_server_client_finalized_fn_t      client_finalized;
@@ -249,6 +262,8 @@ typedef struct opal_pmix_server_module_1_0_0_t {
     opal_pmix_server_tool_connection_fn_t       tool_connected;
     opal_pmix_server_log_fn_t                   log;
     opal_pmix_server_listener_fn_t              listener;
+    opal_pmix_server_alloc_fn_t                 allocate;
+    opal_pmix_server_job_control_fn_t           job_control;
 } opal_pmix_server_module_t;
 
 

opal/mca/pmix/pmix_types.h

@@ -32,6 +32,11 @@ BEGIN_C_DECLS
  * that key */
 #define OPAL_PMIX_RANK_WILDCARD  UINT32_MAX-1
 
+/* other special rank values will be used to define
+ * groups of ranks for use in collectives */
+#define OPAL_PMIX_RANK_LOCAL_NODE    UINT32_MAX-2        // all ranks on local node
+
+
 /* define a set of "standard" attributes that can
  * be queried. Implementations (and users) are free to extend as
  * desired, so the get functions need to be capable
@@ -55,12 +60,15 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_CONNECT_TO_SYSTEM             "pmix.cnct.sys"         // (bool) The requestor requires that a connection be made only to
                                                                         //        a local system-level PMIx server
 #define OPAL_PMIX_CONNECT_SYSTEM_FIRST          "pmix.cnct.sys.first"   // (bool) Preferentially look for a system-level PMIx server first
+#define OPAL_PMIX_REGISTER_NODATA               "pmix.reg.nodata"       // (bool) Registration is for nspace only, do not copy job data
+#define OPAL_PMIX_SERVER_ENABLE_MONITORING      "pmix.srv.monitor"      // (bool) Enable PMIx internal monitoring by server
 
 
 /* identification attributes */
 #define OPAL_PMIX_USERID                        "pmix.euid"             // (uint32_t) effective user id
 #define OPAL_PMIX_GRPID                         "pmix.egid"             // (uint32_t) effective group id
 
+
 /* attributes for the rendezvous socket  */
 #define OPAL_PMIX_USOCK_DISABLE                 "pmix.usock.disable"    // (bool) disable legacy usock support
 #define OPAL_PMIX_SOCKET_MODE                   "pmix.sockmode"         // (uint32_t) POSIX mode_t (9 bits valid)
@@ -76,6 +84,7 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_TCP_DISABLE_IPV4              "pmix.tcp.disipv4"      // (bool) true to disable IPv4 family
 #define OPAL_PMIX_TCP_DISABLE_IPV6              "pmix.tcp.disipv6"      // (bool) true to disable IPv6 family
 
+
 /* general proc-level attributes */
 #define OPAL_PMIX_CPUSET                        "pmix.cpuset"           // (char*) hwloc bitmap applied to proc upon launch
 #define OPAL_PMIX_CREDENTIAL                    "pmix.cred"             // (char*) security credential assigned to proc
@@ -89,6 +98,7 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_PROCDIR                       "pmix.pdir"             // (char*) sub-nsdir assigned to proc
 #define OPAL_PMIX_TDIR_RMCLEAN                  "pmix.tdir.rmclean"     // (bool)  Resource Manager will clean session directories
 
+
 /* information about relative ranks as assigned by the RM */
 #define OPAL_PMIX_PROCID                        "pmix.procid"           // (opal_process_name_t) process identifier
 #define OPAL_PMIX_NSPACE                        "pmix.nspace"           // (char*) nspace of a job
@@ -104,25 +114,26 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_LOCALLDR                      "pmix.lldr"             // (uint64_t) opal_identifier of lowest rank on this node within this job
 #define OPAL_PMIX_APPLDR                        "pmix.aldr"             // (uint32_t) lowest rank in this app within this job
 #define OPAL_PMIX_PROC_PID                      "pmix.ppid"             // (pid_t) pid of specified proc
-
-/****  no PMIx equivalent ****/
-#define OPAL_PMIX_LOCALITY                      "pmix.loc"              // (uint16_t) relative locality of two procs
-#define OPAL_PMIX_TOPOLOGY_SIGNATURE            "pmix.toposig"          // (char*) topology signature string
-#define OPAL_PMIX_LOCALITY_STRING               "pmix.locstr"           // (char*) string describing a proc's location
-#define OPAL_PMIX_AVAIL_PHYS_MEMORY             "pmix.pmem"             // (uint64_t) total available physical memory on this node
-
+#define OPAL_PMIX_SESSION_ID                    "pmix.session.id"       // (uint32_t) session identifier
 #define OPAL_PMIX_NODE_LIST                     "pmix.nlist"            // (char*) comma-delimited list of nodes running procs for the specified nspace
 #define OPAL_PMIX_ALLOCATED_NODELIST            "pmix.alist"            // (char*) comma-delimited list of all nodes in this allocation regardless of
                                                                         //           whether or not they currently host procs.
 #define OPAL_PMIX_HOSTNAME                      "pmix.hname"            // (char*) name of the host the specified proc is on
 #define OPAL_PMIX_NODEID                        "pmix.nodeid"           // (uint32_t) node identifier
 #define OPAL_PMIX_LOCAL_PEERS                   "pmix.lpeers"           // (char*) comma-delimited string of ranks on this node within the specified nspace
+#define OPAL_PMIX_LOCAL_PROCS                   "pmix.lprocs"           // (opal_list_t*) list of opal_namelist_t of procs on the specified node
 #define OPAL_PMIX_LOCAL_CPUSETS                 "pmix.lcpus"            // (char*) colon-delimited cpusets of local peers within the specified nspace
 #define OPAL_PMIX_PROC_URI                      "opal.puri"             // (char*) URI containing contact info for proc - NOTE: this is published by procs and
                                                                         //            thus cannot be prefixed with "pmix"
+#define OPAL_PMIX_LOCALITY                      "pmix.loc"              // (uint16_t) relative locality of two procs
+
+
+/* Memory info */
+#define OPAL_PMIX_AVAIL_PHYS_MEMORY             "pmix.pmem"             // (uint64_t) total available physical memory on this node
 #define OPAL_PMIX_DAEMON_MEMORY                 "pmix.dmn.mem"          // (float) Mbytes of memory currently used by daemon
 #define OPAL_PMIX_CLIENT_AVG_MEMORY             "pmix.cl.mem.avg"       // (float) Average Mbytes of memory used by client processes
 
+
 /* size info */
 #define OPAL_PMIX_UNIV_SIZE                     "pmix.univ.size"        // (uint32_t) #procs in this nspace
 #define OPAL_PMIX_JOB_SIZE                      "pmix.job.size"         // (uint32_t) #procs in this job
@@ -133,11 +144,15 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_MAX_PROCS                     "pmix.max.size"         // (uint32_t) max #procs for this job
 #define OPAL_PMIX_NUM_NODES                     "pmix.num.nodes"        // (uint32_t) #nodes in this nspace
 
+
 /* topology info */
 #define OPAL_PMIX_NET_TOPO                      "pmix.ntopo"            // (char*) xml-representation of network topology
 #define OPAL_PMIX_LOCAL_TOPO                    "pmix.ltopo"            // (char*) xml-representation of local node topology
 #define OPAL_PMIX_NODE_LIST                     "pmix.nlist"            // (char*) comma-delimited list of nodes running procs for this job
 #define OPAL_PMIX_TOPOLOGY                      "pmix.topo"             // (hwloc_topology_t) pointer to the PMIx client's internal topology object
+#define OPAL_PMIX_TOPOLOGY_SIGNATURE            "pmix.toposig"          // (char*) topology signature string
+#define OPAL_PMIX_LOCALITY_STRING               "pmix.locstr"           // (char*) string describing a proc's location
+
 
 /* request-related info */
 #define OPAL_PMIX_COLLECT_DATA                  "pmix.collect"          // (bool) collect data and return it at the end of the operation
@@ -156,16 +171,19 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_EMBED_BARRIER                 "pmix.embed.barrier"    // (bool) execute a blocking fence operation before executing the
                                                                         //        specified operation
 
+
 /* attribute used by host server to pass data to the server convenience library - the
  * data will then be parsed and provided to the local clients */
 #define OPAL_PMIX_PROC_DATA                     "pmix.pdata"            // (pmix_value_array_t) starts with rank, then contains more data
 #define OPAL_PMIX_NODE_MAP                      "pmix.nmap"             // (char*) regex of nodes containing procs for this job
 #define OPAL_PMIX_PROC_MAP                      "pmix.pmap"             // (char*) regex describing procs on each node within this job
 
+
 /* attributes used internally to communicate data from the server to the client */
 #define OPAL_PMIX_PROC_BLOB                     "pmix.pblob"            // (pmix_byte_object_t) packed blob of process data
 #define OPAL_PMIX_MAP_BLOB                      "pmix.mblob"            // (pmix_byte_object_t) packed blob of process location
 
+
 /* error handler registration  and notification info keys */
 #define OPAL_PMIX_EVENT_HDLR_NAME               "pmix.evname"           // (char*) string name identifying this handler
 #define OPAL_PMIX_EVENT_JOB_LEVEL               "pmix.evjob"            // (bool) register for job-specific events only
@@ -187,7 +205,7 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_EVENT_ACTION_TIMEOUT          "pmix.evtimeout"        // (int) time in sec before RM will execute error response
 
 
-/* attributes used to describe "spawm" attributes */
+/* attributes used to describe "spawn" attributes */
 #define OPAL_PMIX_PERSONALITY                   "pmix.pers"             // (char*) name of personality to use
 #define OPAL_PMIX_HOST                          "pmix.host"             // (char*) comma-delimited list of hosts to use for spawned procs
 #define OPAL_PMIX_HOSTFILE                      "pmix.hostfile"         // (char*) hostfile to use for spawned procs
@@ -229,19 +247,89 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_QUERY_LOCAL_ONLY              "pmix.qry.local"        // constrain the query to local information only
 #define OPAL_PMIX_QUERY_REPORT_AVG              "pmix.qry.avg"          // report average values
 #define OPAL_PMIX_QUERY_REPORT_MINMAX           "pmix.qry.minmax"       // report minimum and maximum value
+#define OPAL_PMIX_QUERY_ALLOC_STATUS            "pmix.query.alloc"      // (char*) string identifier of the allocation whose status
+                                                                        //         is being requested
+#define OPAL_PMIX_TIME_REMAINING                "pmix.time.remaining"   // (char*) query number of seconds (uint32_t) remaining in allocation
+                                                                        //         for the specified nspace
 
 /* log attributes */
-#define OPAL_PMIX_LOG_STDERR                    "pmix.log.stderr"        // (char*) log string to stderr
-#define OPAL_PMIX_LOG_STDOUT                    "pmix.log.stdout"        // (char*) log string to stdout
-#define OPAL_PMIX_LOG_SYSLOG                    "pmix.log.syslog"        // (char*) log data to syslog - defaults to ERROR priority unless
-#define OPAL_PMIX_LOG_MSG                       "pmix.log.msg"           // (pmix_byte_object_t) message blob to be sent somewhere
+#define OPAL_PMIX_LOG_STDERR                    "pmix.log.stderr"       // (char*) log string to stderr
+#define OPAL_PMIX_LOG_STDOUT                    "pmix.log.stdout"       // (char*) log string to stdout
+#define OPAL_PMIX_LOG_SYSLOG                    "pmix.log.syslog"       // (char*) log data to syslog - defaults to ERROR priority unless
+#define OPAL_PMIX_LOG_MSG                       "pmix.log.msg"          // (pmix_byte_object_t) message blob to be sent somewhere
+#define OPAL_PMIX_LOG_EMAIL                     "pmix.log.email"        // (pmix_data_array_t) log via email based on pmix_info_t containing directives
+#define OPAL_PMIX_LOG_EMAIL_ADDR                "pmix.log.emaddr"       // (char*) comma-delimited list of email addresses that are to recv msg
+#define OPAL_PMIX_LOG_EMAIL_SUBJECT             "pmix.log.emsub"        // (char*) subject line for email
+#define OPAL_PMIX_LOG_EMAIL_MSG                 "pmix.log.emmsg"        // (char*) msg to be included in email
+
 
 /* debugger attributes */
-#define OPAL_PMIX_DEBUG_STOP_ON_EXEC            "pmix.dbg.exec"          // (bool) job is being spawned under debugger - instruct it to pause on start
-#define OPAL_PMIX_DEBUG_STOP_IN_INIT            "pmix.dbg.init"          // (bool) instruct job to stop during PMIx init
-#define OPAL_PMIX_DEBUG_WAIT_FOR_NOTIFY         "pmix.dbg.notify"        // (bool) block at desired point until receiving debugger release notification
-#define OPAL_PMIX_DEBUG_JOB                     "pmix.dbg.job"           // (char*) nspace of the job to be debugged - the RM/PMIx server are
-#define OPAL_PMIX_DEBUG_WAITING_FOR_NOTIFY      "pmix.dbg.waiting"       // (bool) job to be debugged is waiting for a release
+#define OPAL_PMIX_DEBUG_STOP_ON_EXEC            "pmix.dbg.exec"         // (bool) job is being spawned under debugger - instruct it to pause on start
+#define OPAL_PMIX_DEBUG_STOP_IN_INIT            "pmix.dbg.init"         // (bool) instruct job to stop during PMIx init
+#define OPAL_PMIX_DEBUG_WAIT_FOR_NOTIFY         "pmix.dbg.notify"       // (bool) block at desired point until receiving debugger release notification
+#define OPAL_PMIX_DEBUG_JOB                     "pmix.dbg.job"          // (char*) nspace of the job to be debugged - the RM/PMIx server are
+#define OPAL_PMIX_DEBUG_WAITING_FOR_NOTIFY      "pmix.dbg.waiting"      // (bool) job to be debugged is waiting for a release
+
+
+/* Resource Manager identification */
+#define OPAL_PMIX_RM_NAME                       "pmix.rm.name"          // (char*) string name of the resource manager
+#define OPAL_PMIX_RM_VERSION                    "pmix.rm.version"       // (char*) RM version string
+
+
+/* attributes for setting envars */
+#define OPAL_PMIX_SET_ENVAR                     "pmix.set.envar"        // (char*) string "key=value" value shall be put into the environment
+#define OPAL_PMIX_UNSET_ENVAR                   "pmix.unset.envar"      // (char*) unset envar specified in string
+
+
+/* attributes relating to allocations */
+#define OPAL_PMIX_ALLOC_ID                      "pmix.alloc.id"         // (char*) provide a string identifier for this allocation request
+                                                                        //         which can later be used to query status of the request
+#define OPAL_PMIX_ALLOC_NUM_NODES               "pmix.alloc.nnodes"     // (uint64_t) number of nodes
+#define OPAL_PMIX_ALLOC_NODE_LIST               "pmix.alloc.nlist"      // (char*) regex of specific nodes
+#define OPAL_PMIX_ALLOC_NUM_CPUS                "pmix.alloc.ncpus"      // (uint64_t) number of cpus
+#define OPAL_PMIX_ALLOC_NUM_CPU_LIST            "pmix.alloc.ncpulist"   // (char*) regex of #cpus for each node
+#define OPAL_PMIX_ALLOC_CPU_LIST                "pmix.alloc.cpulist"    // (char*) regex of specific cpus indicating the cpus involved.
+#define OPAL_PMIX_ALLOC_MEM_SIZE                "pmix.alloc.msize"      // (float) number of Mbytes
+#define OPAL_PMIX_ALLOC_NETWORK                 "pmix.alloc.net"        // (array) array of pmix_info_t describing network resources. If not
+                                                                        //         given as part of an info struct that identifies the
+                                                                        //         impacted nodes, then the description will be applied
+                                                                        //         across all nodes in the requestor's allocation
+#define OPAL_PMIX_ALLOC_NETWORK_ID              "pmix.alloc.netid"      // (char*) name of network
+#define OPAL_PMIX_ALLOC_BANDWIDTH               "pmix.alloc.bw"         // (float) Mbits/sec
+#define OPAL_PMIX_ALLOC_NETWORK_QOS             "pmix.alloc.netqos"     // (char*) quality of service level
+#define OPAL_PMIX_ALLOC_TIME                    "pmix.alloc.time"       // (uint32_t) time in seconds
+
+
+/* job control attributes */
+#define OPAL_PMIX_JOB_CTRL_ID                   "pmix.jctrl.id"         // (char*) provide a string identifier for this request
+#define OPAL_PMIX_JOB_CTRL_PAUSE                "pmix.jctrl.pause"      // (bool) pause the specified processes
+#define OPAL_PMIX_JOB_CTRL_RESUME               "pmix.jctrl.resume"     // (bool) "un-pause" the specified processes
+#define OPAL_PMIX_JOB_CTRL_CANCEL               "pmix.jctrl.cancel"     // (char*) cancel the specified request
+                                                                        //         (NULL => cancel all requests from this requestor)
+#define OPAL_PMIX_JOB_CTRL_KILL                 "pmix.jctrl.kill"       // (bool) forcibly terminate the specified processes and cleanup
+#define OPAL_PMIX_JOB_CTRL_RESTART              "pmix.jctrl.restart"    // (char*) restart the specified processes using the given checkpoint ID
+#define OPAL_PMIX_JOB_CTRL_CHECKPOINT           "pmix.jctrl.ckpt"       // (char*) checkpoint the specified processes and assign the given ID to it
+#define OPAL_PMIX_JOB_CTRL_CHECKPOINT_EVENT     "pmix.jctrl.ckptev"     // (bool) use event notification to trigger process checkpoint
+#define OPAL_PMIX_JOB_CTRL_CHECKPOINT_SIGNAL    "pmix.jctrl.ckptsig"    // (int) use the given signal to trigger process checkpoint
+#define OPAL_PMIX_JOB_CTRL_CHECKPOINT_TIMEOUT   "pmix.jctrl.ckptsig"    // (int) time in seconds to wait for checkpoint to complete
+#define OPAL_PMIX_JOB_CTRL_SIGNAL               "pmix.jctrl.sig"        // (int) send given signal to specified processes
+#define OPAL_PMIX_JOB_CTRL_PROVISION            "pmix.jctrl.pvn"        // (char*) regex identifying nodes that are to be provisioned
+#define OPAL_PMIX_JOB_CTRL_PROVISION_IMAGE      "pmix.jctrl.pvnimg"     // (char*) name of the image that is to be provisioned
+#define OPAL_PMIX_JOB_CTRL_PREEMPTIBLE          "pmix.jctrl.preempt"    // (bool) job can be pre-empted
+
+/* monitoring attributes */
+#define OPAL_PMIX_MONITOR_HEARTBEAT             "pmix.monitor.mbeat"    // (void) register to have the server monitor the requestor for heartbeats
+#define OPAL_PMIX_SEND_HEARTBEAT                "pmix.monitor.beat"     // (void) send heartbeat to local server
+#define OPAL_PMIX_MONITOR_HEARTBEAT_TIME        "pmix.monitor.btime"    // (uint32_t) time in seconds before declaring heartbeat missed
+#define OPAL_PMIX_MONITOR_HEARTBEAT_DROPS       "pmix.monitor.bdrop"    // (uint32_t) number of heartbeats that can be missed before taking
+                                                                        //            specified action
+#define OPAL_PMIX_MONITOR_FILE                  "pmix.monitor.fmon"     // (char*) register to monitor file for signs of life
+#define OPAL_PMIX_MONITOR_FILE_SIZE             "pmix.monitor.fsize"    // (bool) monitor size of given file is growing to determine app is running
+#define OPAL_PMIX_MONITOR_FILE_ACCESS           "pmix.monitor.faccess"  // (char*) monitor time since last access of given file to determine app is running
+#define OPAL_PMIX_MONITOR_FILE_MODIFY           "pmix.monitor.fmod"     // (char*) monitor time since last modified of given file to determine app is running
+#define OPAL_PMIX_MONITOR_FILE_CHECK_TIME       "pmix.monitor.ftime"    // (uint32_t) time in seconds between checking file
+#define OPAL_PMIX_MONITOR_FILE_DROPS            "pmix.monitor.fdrop"    // (uint32_t) number of file checks that can be missed before taking
+                                                                        //            specified action
 
 
 /* define a scope for data "put" by PMI per the following:
@@ -285,6 +373,16 @@ typedef enum {
 } opal_pmix_persistence_t;
 
 
+/* define allocation request flags */
+typedef enum {
+    OPAL_PMIX_ALLOC_UNDEF = 0,
+    OPAL_PMIX_ALLOC_NEW,
+    OPAL_PMIX_ALLOC_EXTEND,
+    OPAL_PMIX_ALLOC_RELEASE,
+    OPAL_PMIX_ALLOC_REAQCUIRE
+} opal_pmix_alloc_directive_t;
+
+
 /****    PMIX INFO STRUCT    ****/
 
 /* NOTE: the pmix_info_t is essentially equivalent to the opal_value_t

opal/runtime/opal_init.c

@@ -292,6 +292,12 @@ opal_err2str(int errnum, const char **errmsg)
     case OPAL_ERR_EVENT_REGISTRATION:
         retval = "Event registration";
         break;
+    case OPAL_ERR_HEARTBEAT_ALERT:
+        retval = "Heartbeat not received";
+        break;
+    case OPAL_ERR_FILE_ALERT:
+        retval = "File alert - proc may have stalled";
+        break;
     default:
         retval = "UNRECOGNIZED";
     }

orte/mca/schizo/base/base.h

@@ -76,7 +76,7 @@ ORTE_DECLSPEC int orte_schizo_base_setup_child(orte_job_t *jobdat,
                                                orte_app_context_t *app,
                                                char ***env);
 ORTE_DECLSPEC orte_schizo_launch_environ_t orte_schizo_base_check_launch_environment(void);
-ORTE_DECLSPEC long orte_schizo_base_get_remaining_time(void);
+ORTE_DECLSPEC int orte_schizo_base_get_remaining_time(uint32_t *timeleft);
 ORTE_DECLSPEC void orte_schizo_base_finalize(void);
 
 END_C_DECLS

orte/mca/schizo/base/schizo_base_stubs.c

@@ -162,20 +162,20 @@ orte_schizo_launch_environ_t orte_schizo_base_check_launch_environment(void)
     return ORTE_SCHIZO_UNDETERMINED;
 }
 
-long orte_schizo_base_get_remaining_time(void)
+int orte_schizo_base_get_remaining_time(uint32_t *timeleft)
 {
-    long rc;
+    int rc;
     orte_schizo_base_active_module_t *mod;
 
     OPAL_LIST_FOREACH(mod, &orte_schizo_base.active_modules, orte_schizo_base_active_module_t) {
         if (NULL != mod->module->get_remaining_time) {
-            rc = mod->module->get_remaining_time();
+            rc = mod->module->get_remaining_time(timeleft);
             if (ORTE_ERR_TAKE_NEXT_OPTION != rc) {
                 return rc;
             }
         }
     }
-    return -1;
+    return ORTE_ERR_NOT_SUPPORTED;
 }
 
 void orte_schizo_base_finalize(void)

orte/mca/schizo/schizo.h

@@ -118,7 +118,7 @@ typedef void (*orte_schizo_base_module_finalize_fn_t)(void);
  * and decides it cannot provide the info in the current situation,
  * then it can return ORTE_ERR_TAKE_NEXT_OPTION to indicate that
  * another module should be tried */
-typedef long (*orte_schizo_base_module_get_rem_time_fn_t)(void);
+typedef int (*orte_schizo_base_module_get_rem_time_fn_t)(uint32_t *timeleft);
 
 /*
  * schizo module version 1.3.0

orte/mca/schizo/slurm/schizo_slurm.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies Ltd.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -29,10 +29,12 @@
 #include "schizo_slurm.h"
 
 static orte_schizo_launch_environ_t check_launch_environment(void);
+static int get_remaining_time(uint32_t *timeleft);
 static void finalize(void);
 
 orte_schizo_base_module_t orte_schizo_slurm_module = {
     .check_launch_environment = check_launch_environment,
+    .get_remaining_time = get_remaining_time,
     .finalize = finalize
 };
 
@@ -123,6 +125,58 @@ static orte_schizo_launch_environ_t check_launch_environment(void)
     return myenv;
 }
 
+static int get_remaining_time(uint32_t *timeleft)
+{
+    char output[256], *cmd, *jobid, **res;
+    FILE *fp;
+    uint32_t tleft;
+    size_t cnt;
+
+    /* set the default */
+    *timeleft = UINT32_MAX;
+
+    if (NULL == (jobid = getenv("SLURM_JOBID"))) {
+        return ORTE_ERR_TAKE_NEXT_OPTION;
+    }
+    if (0 > asprintf(&cmd, "squeue -h -j %s -o %%L", jobid)) {
+        return ORTE_ERR_OUT_OF_RESOURCE;
+    }
+    fp = popen(cmd, "r");
+    if (NULL == fp) {
+        free(cmd);
+        return ORTE_ERR_FILE_OPEN_FAILURE;
+    }
+    if (NULL == fgets(output, 256, fp)) {
+        free(cmd);
+        return ORTE_ERR_FILE_READ_FAILURE;
+    }
+    free(cmd);
+    /* the output is returned in a colon-delimited set of fields */
+    res = opal_argv_split(output, ':');
+    cnt =  opal_argv_count(res);
+    tleft = strtol(res[cnt-1], NULL, 10); // has to be at least one field
+    /* the next field would be minutes */
+    if (1 < cnt) {
+        tleft += 60 * strtol(res[cnt-2], NULL, 10);
+    }
+    /* next field would be hours */
+    if (2 < cnt) {
+        tleft += 3600 * strtol(res[cnt-3], NULL, 10);
+    }
+    /* next field is days */
+    if (3 < cnt) {
+        tleft += 24*3600 * strtol(res[cnt-4], NULL, 10);
+    }
+    /* if there are more fields than that, then it is infinite */
+    if (4 < cnt) {
+        tleft = UINT32_MAX;
+    }
+    opal_argv_free(res);
+
+    *timeleft = tleft;
+    return ORTE_SUCCESS;
+}
+
 static void finalize(void)
 {
     int i;

orte/mca/schizo/slurm/schizo_slurm_component.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,8 +38,8 @@ orte_schizo_base_component_t mca_schizo_slurm_component = {
 
 static int component_query(mca_base_module_t **module, int *priority)
 {
-    /* disqualify ourselves if we are not an app or under slurm */
-    if (!ORTE_PROC_IS_APP) {
+    /* disqualify ourselves if we are not under slurm */
+    if (NULL == getenv("SLURM_JOBID")) {
         *priority = 0;
         *module = NULL;
         return OPAL_ERROR;
@@ -49,4 +49,3 @@ static int component_query(mca_base_module_t **module, int *priority)
     *priority = 50;
     return ORTE_SUCCESS;
 }
-

orte/mca/sensor/base/base.h

@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file:
- */
-
-#ifndef MCA_SENSOR_BASE_H
-#define MCA_SENSOR_BASE_H
-
-/*
- * includes
- */
-#include "orte_config.h"
-
-#include "opal/class/opal_list.h"
-#include "opal/mca/base/base.h"
-
-#include "orte/mca/sensor/sensor.h"
-
-BEGIN_C_DECLS
-
-/*
- * MCA Framework
- */
-ORTE_DECLSPEC extern mca_base_framework_t orte_sensor_base_framework;
-/* select a component */
-ORTE_DECLSPEC int orte_sensor_base_select(void);
-
-
-END_C_DECLS
-#endif

orte/mca/sensor/base/sensor_base_fns.c

@@ -1,158 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#include "opal/dss/dss.h"
-#include "opal/mca/event/event.h"
-
-#include "orte/mca/sensor/base/base.h"
-#include "orte/mca/sensor/base/sensor_private.h"
-
-static bool mods_active = false;
-
-void orte_sensor_base_start(orte_jobid_t job)
-{
-    orte_sensor_active_module_t *i_module;
-    int i;
-
-    if (0 < orte_sensor_base.rate.tv_sec) {
-        opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                            "%s sensor:base: starting sensors",
-                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
-        /* call the start function of all modules in priority order */
-        for (i=0; i < orte_sensor_base.modules.size; i++) {
-            if (NULL == (i_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&orte_sensor_base.modules, i))) {
-                continue;
-            }
-            mods_active = true;
-            if (NULL != i_module->module->start) {
-                i_module->module->start(job);
-            }
-        }
-
-        if (mods_active && !orte_sensor_base.active) {
-            /* setup a buffer to collect samples */
-            orte_sensor_base.samples = OBJ_NEW(opal_buffer_t);
-            /* startup a timer to wake us up periodically
-             * for a data sample
-             */
-            orte_sensor_base.active = true;
-            opal_event_evtimer_set(orte_event_base, &orte_sensor_base.sample_ev,
-                                   orte_sensor_base_sample, NULL);
-            opal_event_evtimer_add(&orte_sensor_base.sample_ev, &orte_sensor_base.rate);
-        }
-    }
-    return;    
-}
-
-void orte_sensor_base_stop(orte_jobid_t job)
-{
-    orte_sensor_active_module_t *i_module;
-    int i;
-
-    if (!mods_active) {
-        return;
-    }
-
-    opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                        "%s sensor:base: stopping sensors",
-                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
-
-    if (orte_sensor_base.active) {
-        opal_event_del(&orte_sensor_base.sample_ev);
-        orte_sensor_base.active = false;
-    }
-
-    /* call the stop function of all modules in priority order */
-    for (i=0; i < orte_sensor_base.modules.size; i++) {
-        if (NULL == (i_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&orte_sensor_base.modules, i))) {
-            continue;
-        }
-        if (NULL != i_module->module->stop) {
-            i_module->module->stop(job);
-        }
-    }
-
-    return;
-}
-
-void orte_sensor_base_sample(int fd, short args, void *cbdata)
-{
-    orte_sensor_active_module_t *i_module;
-    int i;
-    
-    if (!mods_active) {
-        return;
-    }
-
-    /* see if we were ordered to stop */
-    if (!orte_sensor_base.active) {
-        return;
-    }
-
-    opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                        "%s sensor:base: sampling sensors",
-                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
-
-    /* call the sample function of all modules in priority order from
-     * highest to lowest - the heartbeat should always be the lowest
-     * priority, so it will send any collected data
-     */
-    for (i=0; i < orte_sensor_base.modules.size; i++) {
-        if (NULL == (i_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&orte_sensor_base.modules, i))) {
-            continue;
-        }
-        if (NULL != i_module->module->sample) {
-            opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                                "%s sensor:base: sampling component %s",
-                                ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                i_module->component->base_version.mca_component_name);
-            i_module->module->sample();
-        }
-    }
-
-    /* restart the timer */
-    opal_event_evtimer_add(&orte_sensor_base.sample_ev, &orte_sensor_base.rate);
-
-    return;
-}
-
-void orte_sensor_base_log(char *comp, opal_buffer_t *data)
-{
-    int i;
-    orte_sensor_active_module_t *i_module;
-
-    if (NULL == comp) {
-        /* nothing we can do */
-        return;
-    }
-
-    opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                        "%s sensor:base: logging sensor %s",
-                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), comp);
-
-    /* find the specified module  */
-    for (i=0; i < orte_sensor_base.modules.size; i++) {
-        if (NULL == (i_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&orte_sensor_base.modules, i))) {
-            continue;
-        }
-        if (0 == strcmp(comp, i_module->component->base_version.mca_component_name)) {
-            if (NULL != i_module->module->log) {
-                i_module->module->log(data);
-            }
-            return;
-        }
-    }
-}

orte/mca/sensor/base/sensor_base_frame.c

@@ -1,133 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * Copyright (c) 2017      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#include "opal/mca/mca.h"
-#include "opal/util/argv.h"
-#include "opal/util/output.h"
-#include "opal/mca/base/base.h"
-#include "opal/class/opal_pointer_array.h"
-
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-
-#include "orte/mca/sensor/base/base.h"
-#include "orte/mca/sensor/base/sensor_private.h"
-
-/*
- * The following file was created by configure.  It contains extern
- * statements and the definition of an array of pointers to each
- * component's public mca_base_component_t struct.
- */
-
-#include "orte/mca/sensor/base/static-components.h"
-
-/*
- * Global variables
- */
-orte_sensor_base_API_module_t orte_sensor = {
-    orte_sensor_base_start,
-    orte_sensor_base_stop
-};
-orte_sensor_base_t orte_sensor_base = {{{0}}};
-
-/*
- * Local variables
- */
-static int orte_sensor_base_sample_rate = 0;
-
-static int orte_sensor_base_register(mca_base_register_flag_t flags)
-{
-    int var_id;
-
-    orte_sensor_base_sample_rate = 0;
-    var_id = mca_base_var_register("orte", "sensor", "base", "sample_rate",
-                                   "Sample rate in seconds",
-                                   MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                   OPAL_INFO_LVL_9,
-                                   MCA_BASE_VAR_SCOPE_READONLY,
-                                   &orte_sensor_base_sample_rate);
-    mca_base_var_register_synonym(var_id, "orte", "sensor", NULL, "sample_rate",
-                                  MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
-  
-    /* see if we want samples logged */
-    orte_sensor_base.log_samples = false;
-    var_id = mca_base_var_register("orte", "sensor", "base", "log_samples",
-                                   "Log samples to database",
-                                   MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                   OPAL_INFO_LVL_9,
-                                   MCA_BASE_VAR_SCOPE_READONLY,
-                                   &orte_sensor_base.log_samples);
-    mca_base_var_register_synonym(var_id, "orte", "sensor", NULL, "log_samples",
-                                  MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
-
-    return ORTE_SUCCESS;
-}
-
-static int orte_sensor_base_close(void)
-{
-    orte_sensor_active_module_t *i_module;
-    int i;
-    
-    for (i=0; i < orte_sensor_base.modules.size; i++) {
-        if (NULL == (i_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&orte_sensor_base.modules, i))) {
-            continue;
-        }
-        if (NULL != i_module->module->finalize) {
-            i_module->module->finalize();
-        }
-    }
-    OBJ_DESTRUCT(&orte_sensor_base.modules);
-    
-    /* Close all remaining available components */
-    return mca_base_framework_components_close(&orte_sensor_base_framework, NULL);
-}
-
-/**
- * Function for finding and opening either all MCA components, or the one
- * that was specifically requested via a MCA parameter.
- */
-static int orte_sensor_base_open(mca_base_open_flag_t flags)
-{
-    /* initialize globals */
-    orte_sensor_base.active = false;
-
-    /* construct the array of modules */
-    OBJ_CONSTRUCT(&orte_sensor_base.modules, opal_pointer_array_t);
-    opal_pointer_array_init(&orte_sensor_base.modules, 3, INT_MAX, 1);
-    
-    /* get the sample rate */
-    orte_sensor_base.rate.tv_sec = orte_sensor_base_sample_rate;
-    orte_sensor_base.rate.tv_usec = 0;
-
-    /* Open up all available components */
-    return mca_base_framework_components_open(&orte_sensor_base_framework, flags);
-}
-
-MCA_BASE_FRAMEWORK_DECLARE(orte, sensor, "ORTE Monitoring Sensors",
-                           orte_sensor_base_register,
-                           orte_sensor_base_open, orte_sensor_base_close,
-                           mca_sensor_base_static_components, 0);
-
-static void cons(orte_sensor_active_module_t *t)
-{
-    t->sampling = true;
-}
-OBJ_CLASS_INSTANCE(orte_sensor_active_module_t,
-                   opal_object_t,
-                   cons, NULL);

orte/mca/sensor/base/sensor_base_select.c

@@ -1,219 +0,0 @@
-/*
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved.
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "orte_config.h"
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-
-#include "orte/constants.h"
-
-#include "opal/mca/mca.h"
-#include "opal/mca/base/base.h"
-#include "opal/util/argv.h"
-#include "opal/util/output.h"
-#include "opal/class/opal_pointer_array.h"
-
-#include "orte/mca/errmgr/errmgr.h"
-#include "orte/mca/sensor/base/base.h"
-#include "orte/mca/sensor/base/sensor_private.h"
-
-
-static bool selected = false;
-
-/**
- * Function for weeding out sensor components that don't want to run.
- *
- * Call the init function on all available components to find out if
- * they want to run.  Select all components that don't fail.  Failing
- * components will be closed and unloaded.  The selected modules will
- * be returned to the caller in a opal_list_t.
- */
-int orte_sensor_base_select(void)
-{
-    mca_base_component_list_item_t *cli = NULL;
-    orte_sensor_base_component_t *component = NULL;
-    mca_base_module_t *module = NULL;
-    orte_sensor_active_module_t *i_module;
-    int priority = 0, i, j, low_i;
-    opal_pointer_array_t tmp_array;
-    bool none_found;
-    orte_sensor_active_module_t *tmp_module = NULL, *tmp_module_sw = NULL;
-    bool duplicate;
-
-    if (selected) {
-        return ORTE_SUCCESS;
-    }
-    selected = true;
-
-    OBJ_CONSTRUCT(&tmp_array, opal_pointer_array_t);
-
-    opal_output_verbose(10, orte_sensor_base_framework.framework_output,
-                        "sensor:base:select: Auto-selecting components");
-
-    /*
-     * Traverse the list of available components.
-     * For each call their 'query' functions to determine relative priority.
-     */
-    none_found = true;
-    OPAL_LIST_FOREACH(cli, &orte_sensor_base_framework.framework_components, mca_base_component_list_item_t) {
-        component = (orte_sensor_base_component_t *) cli->cli_component;
-
-        /*
-         * If there is a query function then use it.
-         */
-        if (NULL == component->base_version.mca_query_component) {
-            opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                                "sensor:base:select Skipping component [%s]. It does not implement a query function",
-                                component->base_version.mca_component_name );
-            continue;
-        }
-
-        /*
-         * Query this component for the module and priority
-         */
-        opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                            "sensor:base:select Querying component [%s]",
-                            component->base_version.mca_component_name);
-
-        component->base_version.mca_query_component(&module, &priority);
-
-        /*
-         * If no module was returned or negative priority, then skip component
-         */
-        if (NULL == module || priority < 0) {
-            opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                                "sensor:base:select Skipping component [%s]. Query failed to return a module",
-                                component->base_version.mca_component_name );
-            continue;
-        }
-
-        /* check to see if we already have someone who senses the
-         * same things - if so, take the higher priority one
-         */
-        duplicate = false;
-        for (i=0; i < tmp_array.size; i++) {
-            tmp_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&tmp_array, i);
-            if (NULL == tmp_module) {
-                continue;
-            }
-            if (0 == strcmp(component->data_measured, tmp_module->component->data_measured)) {
-                if (tmp_module->priority < priority) {
-                    opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                                        "sensor:base:select Replacing component %s with %s - both measure %s",
-                                        tmp_module->component->base_version.mca_component_name,
-                                        component->base_version.mca_component_name,
-                                        component->data_measured);
-                    OBJ_RELEASE(tmp_module);
-                    opal_pointer_array_set_item(&tmp_array, i, NULL);
-                    break;
-                } else {
-                    duplicate = true;
-                }
-            }
-        }
-        if (duplicate) {
-            /* ignore this component */
-            opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                                "sensor:base:select Ignoring component %s - duplicate with higher priority measures %s",
-                                component->base_version.mca_component_name,
-                                component->data_measured);
-            continue;
-        }
-
-        /*
-         * Append them to the temporary list, we will sort later
-         */
-        opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                            "sensor:base:select Query of component [%s] set priority to %d",
-                            component->base_version.mca_component_name, priority);
-        tmp_module = OBJ_NEW(orte_sensor_active_module_t);
-        tmp_module->component = component;
-        tmp_module->module    = (orte_sensor_base_module_t*)module;
-        tmp_module->priority  = priority;
-
-        opal_pointer_array_add(&tmp_array, (void*)tmp_module);
-        none_found = false;
-    }
-
-    if (none_found) {
-        /* okay for no modules to be found */
-        return ORTE_SUCCESS;
-    }
-
-    /*
-     * Sort the list by decending priority
-     */
-    priority = 0;
-    for(j = 0; j < tmp_array.size; ++j) {
-        tmp_module_sw = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&tmp_array, j);
-        if( NULL == tmp_module_sw ) {
-            continue;
-        }
-
-        low_i   = -1;
-        priority = tmp_module_sw->priority;
-
-        for(i = 0; i < tmp_array.size; ++i) {
-            tmp_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&tmp_array, i);
-            if( NULL == tmp_module ) {
-                continue;
-            }
-            if( tmp_module->priority > priority ) {
-                low_i = i;
-                priority = tmp_module->priority;
-            }
-        }
-
-        if( low_i >= 0 ) {
-            tmp_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&tmp_array, low_i);
-            opal_pointer_array_set_item(&tmp_array, low_i, NULL);
-            j--; /* Try this entry again, if it is not the lowest */
-        } else {
-            tmp_module = tmp_module_sw;
-            opal_pointer_array_set_item(&tmp_array, j, NULL);
-        }
-        opal_output_verbose(5, orte_sensor_base_framework.framework_output,
-                            "sensor:base:select Add module with priority [%s] %d",
-                            tmp_module->component->base_version.mca_component_name, tmp_module->priority);
-        opal_pointer_array_add(&orte_sensor_base.modules, tmp_module);
-    }
-    OBJ_DESTRUCT(&tmp_array);
-
-    /*
-     * Initialize each of the modules in priority order from
-     * highest to lowest
-     */
-    for(i = 0; i < orte_sensor_base.modules.size; ++i) {
-        i_module = (orte_sensor_active_module_t*)opal_pointer_array_get_item(&orte_sensor_base.modules, i);
-        if( NULL == i_module ) {
-            continue;
-        }
-        if( NULL != i_module->module->init ) {
-            if (ORTE_SUCCESS != i_module->module->init()) {
-                /* can't sample - however, if we are the HNP,
-                 * then we need this module
-                 * anyway so we can log incoming data
-                 */
-                if (ORTE_PROC_IS_HNP) {
-                    i_module->sampling = false;
-                } else {
-                    opal_pointer_array_set_item(&orte_sensor_base.modules, i, NULL);
-                    OBJ_RELEASE(i_module);
-                }
-            }
-        }
-    }
-
-    return ORTE_SUCCESS;
-}

orte/mca/sensor/base/sensor_private.h

@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file:
- */
-
-#ifndef MCA_SENSOR_PRIVATE_H
-#define MCA_SENSOR_PRIVATE_H
-
-/*
- * includes
- */
-#include "orte_config.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
-
-#include "opal/class/opal_pointer_array.h"
-#include "opal/mca/event/event.h"
-
-#include "orte/runtime/orte_globals.h"
-
-#include "orte/mca/sensor/sensor.h"
-
-
-/*
- * Global functions for MCA overall collective open and close
- */
-BEGIN_C_DECLS
-
-/* define a struct to hold framework-global values */
-typedef struct {
-    opal_pointer_array_t modules;
-    bool log_samples;
-    bool active;
-    struct timeval rate;
-    opal_event_t sample_ev;
-    opal_buffer_t *samples;
-} orte_sensor_base_t;
-
-typedef struct {
-    opal_object_t super;
-    orte_sensor_base_component_t *component;
-    orte_sensor_base_module_t *module;
-    int priority;
-    bool sampling;
-} orte_sensor_active_module_t;
-OBJ_CLASS_DECLARATION(orte_sensor_active_module_t);
-
-
-ORTE_DECLSPEC extern orte_sensor_base_t orte_sensor_base;
-ORTE_DECLSPEC void orte_sensor_base_start(orte_jobid_t job);
-ORTE_DECLSPEC void orte_sensor_base_stop(orte_jobid_t job);
-ORTE_DECLSPEC void orte_sensor_base_sample(int fd, short args, void *cbdata);
-ORTE_DECLSPEC void orte_sensor_base_log(char *comp, opal_buffer_t *data);
-
-END_C_DECLS
-#endif

orte/mca/sensor/file/configure.m4

@@ -1,24 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
-# Copyright (c) 2011-2013 Los Alamos National Security, LLC.
-#                         All rights reserved.
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-# MCA_sensor_file_CONFIG([action-if-found], [action-if-not-found])
-# -----------------------------------------------------------
-AC_DEFUN([MCA_orte_sensor_file_CONFIG], [
-    AC_CONFIG_FILES([orte/mca/sensor/file/Makefile])
-
-    # if we don't want sensors, don't compile
-    # this component
-    AS_IF([test "$orte_want_sensors" = "1"],
-        [$1], [$2])
-])dnl
-

orte/mca/sensor/file/sensor_file.c

@@ -1,354 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
- *                         All rights reserved.
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-#include "orte/types.h"
-
-#include <stdio.h>
-#include <stddef.h>
-#include <ctype.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-#include <fcntl.h>
-#include <errno.h>
-#include <signal.h>
-#ifdef HAVE_TIME_H
-#include <time.h>
-#endif
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#include "opal_stdint.h"
-#include "opal/util/output.h"
-
-#include "orte/util/show_help.h"
-#include "orte/mca/errmgr/errmgr.h"
-#include "orte/mca/state/state.h"
-#include "orte/util/name_fns.h"
-#include "orte/runtime/orte_globals.h"
-
-#include "orte/mca/sensor/base/base.h"
-#include "orte/mca/sensor/base/sensor_private.h"
-#include "sensor_file.h"
-
-/* declare the API functions */
-static int init(void);
-static void finalize(void);
-static void start(orte_jobid_t job);
-static void stop(orte_jobid_t job);
-static void file_sample(void);
-static void file_log(opal_buffer_t *sample);
-
-/* instantiate the module */
-orte_sensor_base_module_t orte_sensor_file_module = {
-    init,
-    finalize,
-    start,
-    stop,
-    file_sample,
-    file_log
-};
-
-/* define a tracking object */
-typedef struct {
-    opal_list_item_t super;
-    orte_jobid_t jobid;
-    orte_vpid_t vpid;
-    char *file;
-    int tick;
-    bool check_size;
-    bool check_access;
-    bool check_mod;
-    int32_t file_size;
-    time_t last_access;
-    time_t last_mod;
-    int limit;
-} file_tracker_t;
-static void ft_constructor(file_tracker_t *ft)
-{
-    ft->file = NULL;
-    ft->tick = 0;
-    ft->file_size = 0;
-    ft->last_access = 0;
-    ft->last_mod = 0;
-    ft->limit = 0;
-}
-static void ft_destructor(file_tracker_t *ft)
-{
-    if (NULL != ft->file) {
-        free(ft->file);
-    }
-}
-OBJ_CLASS_INSTANCE(file_tracker_t,
-                   opal_list_item_t,
-                   ft_constructor, ft_destructor);
-
-/* local globals */
-static opal_list_t jobs; 
-
-static int init(void)
-{
-    OBJ_CONSTRUCT(&jobs, opal_list_t);
-    return ORTE_SUCCESS;
-}
-
-static void finalize(void)
-{
-    opal_list_item_t *item;
-    
-    while (NULL != (item = opal_list_remove_first(&jobs))) {
-        OBJ_RELEASE(item);
-    }
-    OBJ_DESTRUCT(&jobs);
-    
-    return;
-}
-
-static bool find_value(orte_app_context_t *app,
-                       char *pattern, char **value)
-{
-    int i;
-    char *ptr;
-
-    for (i=0; NULL != app->env[i]; i++) {
-        if (0 == strncmp(app->env[i], pattern, strlen(pattern))) {
-            ptr = strchr(app->env[i], '=');
-            ptr++;
-            if (NULL != value) {
-                *value = strdup(ptr);
-            }
-            return true;
-        }
-    }
-    return false;
-}
-
-/*
- * Start monitoring of local processes
- */
-static void start(orte_jobid_t jobid)
-{
-    orte_job_t *jobdat;
-    orte_app_context_t *app, *aptr;
-    int i;
-    char *filename;
-    file_tracker_t *ft;
-    char *ptr;
-
-    /* cannot monitor my own job */
-    if (jobid == ORTE_PROC_MY_NAME->jobid && ORTE_JOBID_WILDCARD != jobid) {
-        return;
-    }
-    
-    OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                         "%s starting file monitoring for job %s",
-                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                         ORTE_JOBID_PRINT(jobid)));
-    
-    /* get the local jobdat for this job */
-    if (NULL == (jobdat = orte_get_job_data_object(jobid))) {
-        ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
-        return;
-    }
-
-    /* must be at least one app_context, so use the first one found */
-    app = NULL;
-    for (i=0; i < jobdat->apps->size; i++) {
-        if (NULL != (aptr = (orte_app_context_t*)opal_pointer_array_get_item(jobdat->apps, i))) {
-            app = aptr;
-            break;
-        }
-    }
-    if (NULL == app) {
-        /* got a problem */
-        ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
-        return;
-    }
-            
-    /* search the environ to get the filename */
-    if (!find_value(app, "OMPI_MCA_sensor_file_filename", &filename)) {
-        /* was a default file given */
-        if (NULL == mca_sensor_file_component.file) {
-            /* can't do anything without a file */
-            OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                                 "%s sensor:file no file for job %s",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 ORTE_JOBID_PRINT(jobid)));
-            return;
-        }
-        filename = mca_sensor_file_component.file;
-    }
-            
-    /* create the tracking object */
-    ft = OBJ_NEW(file_tracker_t);
-    ft->jobid = jobid;
-    ft->file = strdup(filename);
-    
-    /* search the environ to see what we are checking */
-    if (!find_value(app, "OMPI_MCA_sensor_file_check_size", &ptr)) {
-        /* was a default value given */
-        if (0 < mca_sensor_file_component.check_size) {
-            ft->check_size = OPAL_INT_TO_BOOL(mca_sensor_file_component.check_size);
-        }
-    } else {
-        ft->check_size = OPAL_INT_TO_BOOL(strtol(ptr, NULL, 10));
-        free(ptr);
-    }
-
-    if (!find_value(app, "OMPI_MCA_sensor_file_check_access", &ptr)) {
-        /* was a default value given */
-        if (0 < mca_sensor_file_component.check_access) {
-            ft->check_access = OPAL_INT_TO_BOOL(mca_sensor_file_component.check_access);
-        }
-    } else {
-        ft->check_access = OPAL_INT_TO_BOOL(strtol(ptr, NULL, 10));
-        free(ptr);
-    }
-
-    if (!find_value(app, "OMPI_MCA_sensor_file_check_mod", &ptr)) {
-        /* was a default value given */
-        if (0 < mca_sensor_file_component.check_mod) {
-            ft->check_mod = OPAL_INT_TO_BOOL(mca_sensor_file_component.check_mod);
-        }
-    } else {
-        ft->check_mod = OPAL_INT_TO_BOOL(strtol(ptr, NULL, 10));
-        free(ptr);
-    }
-
-    if (!find_value(app, "OMPI_MCA_sensor_file_limit", &ptr)) {
-        ft->limit = mca_sensor_file_component.limit;
-    } else {
-        ft->limit = strtol(ptr, NULL, 10);
-        free(ptr);
-    }
-    opal_list_append(&jobs, &ft->super);
-    OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                         "%s file %s monitored for %s%s%s with limit %d",
-                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                         ft->file, ft->check_size ? "SIZE:" : " ",
-                         ft->check_access ? "ACCESS TIME:" : " ",
-                         ft->check_mod ? "MOD TIME" : " ", ft->limit));
-    return;
-}
-
-
-static void stop(orte_jobid_t jobid)
-{
-    opal_list_item_t *item;
-    file_tracker_t *ft;
-    
-    /* cannot monitor my own job */
-    if (jobid == ORTE_PROC_MY_NAME->jobid && ORTE_JOBID_WILDCARD != jobid) {
-        return;
-    }
-    
-    for (item = opal_list_get_first(&jobs);
-         item != opal_list_get_end(&jobs);
-         item = opal_list_get_next(item)) {
-        ft = (file_tracker_t*)item;
-        if (jobid == ft->jobid || ORTE_JOBID_WILDCARD == jobid) {
-            opal_list_remove_item(&jobs, item);
-            OBJ_RELEASE(item);
-        }
-    }
-    return;
-}
-
-static void file_sample(void)
-{
-    struct stat buf;
-    opal_list_item_t *item;
-    file_tracker_t *ft;
-    orte_job_t *jdata;
-
-    OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                         "%s sampling files",
-                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-    
-    for (item = opal_list_get_first(&jobs);
-         item != opal_list_get_end(&jobs);
-         item = opal_list_get_next(item)) {
-        ft = (file_tracker_t*)item;
-        
-        /* stat the file and get its size */
-        if (0 > stat(ft->file, &buf)) {
-            /* cannot stat file */
-            OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                                 "%s could not stat %s",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 ft->file));
-            continue;
-        }
-        
-        OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                             "%s size %lu access %s\tmod %s",
-                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                             (unsigned long)buf.st_size, ctime(&buf.st_atime), ctime(&buf.st_mtime)));
-
-        if (ft->check_size) {
-            if (buf.st_size == ft->file_size) {
-                ft->tick++;
-                goto CHECK;
-            } else {
-                ft->tick = 0;
-                ft->file_size = buf.st_size;
-            }
-        }
-        if (ft->check_access) {
-            if (buf.st_atime == ft->last_access) {
-                ft->tick++;
-                goto CHECK;
-            } else {
-                ft->tick = 0;
-                ft->last_access = buf.st_atime;
-            }
-        }
-        if (ft->check_mod) {
-            if (buf.st_mtime == ft->last_mod) {
-                ft->tick++;
-                goto CHECK;
-            } else {
-                ft->tick = 0;
-                ft->last_mod = buf.st_mtime;
-            }
-        }
-        
-    CHECK:
-        OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                             "%s sampled file %s tick %d",
-                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                             ft->file, ft->tick));
-
-        if (ft->tick == ft->limit) {
-            orte_show_help("help-orte-sensor-file.txt", "file-stalled", true,
-                           ft->file, ft->file_size, ctime(&ft->last_access), ctime(&ft->last_mod));
-            jdata = orte_get_job_data_object(ft->jobid);
-            ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_SENSOR_BOUND_EXCEEDED);
-        }
-    }
-}
-
-static void file_log(opal_buffer_t *sample)
-{
-}

orte/mca/sensor/file/sensor_file.h

@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-/**
- * @file
- *
- * File movement sensor 
- */
-#ifndef ORTE_SENSOR_FILE_H
-#define ORTE_SENSOR_FILE_H
-
-#include "orte_config.h"
-
-#include "orte/mca/sensor/sensor.h"
-
-BEGIN_C_DECLS
-
-struct orte_sensor_file_component_t {
-    orte_sensor_base_component_t super;
-    int sample_rate;
-    char *file;
-    bool check_size;
-    bool check_access;
-    bool check_mod;
-    int limit;
-};
-typedef struct orte_sensor_file_component_t orte_sensor_file_component_t;
-
-ORTE_MODULE_DECLSPEC extern orte_sensor_file_component_t mca_sensor_file_component;
-extern orte_sensor_base_module_t orte_sensor_file_module;
-
-
-END_C_DECLS
-
-#endif

orte/mca/sensor/file/sensor_file_component.c

@@ -1,120 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#include "opal/mca/base/base.h"
-#include "opal/util/output.h"
-#include "opal/class/opal_pointer_array.h"
-
-#include "orte/util/proc_info.h"
-#include "orte/util/show_help.h"
-
-#include "sensor_file.h"
-
-/*
- * Local functions
- */
-static int orte_sensor_file_register (void);
-static int orte_sensor_file_open(void);
-static int orte_sensor_file_close(void);
-static int orte_sensor_file_query(mca_base_module_t **module, int *priority);
-
-orte_sensor_file_component_t mca_sensor_file_component = {
-    {
-        {
-            ORTE_SENSOR_BASE_VERSION_1_0_0,
-            
-            "file", /* MCA component name */
-            ORTE_MAJOR_VERSION,  /* MCA component major version */
-            ORTE_MINOR_VERSION,  /* MCA component minor version */
-            ORTE_RELEASE_VERSION,  /* MCA component release version */
-            orte_sensor_file_open,  /* component open  */
-            orte_sensor_file_close, /* component close */
-            orte_sensor_file_query,  /* component query */
-            orte_sensor_file_register
-        },
-        {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-        "filemods"  // data being sensed
-    }
-};
-
-
-/**
-  * component register/open/close/init function
-  */
-static int orte_sensor_file_register (void)
-{
-    mca_base_component_t *c = &mca_sensor_file_component.super.base_version;
-
-    /* lookup parameters */
-    mca_sensor_file_component.file = NULL;
-    (void) mca_base_component_var_register (c, "filename", "File to be monitored",
-                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                            &mca_sensor_file_component.file);
-
-    mca_sensor_file_component.check_size = false;
-    (void) mca_base_component_var_register (c, "check_size", "Check the file size",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                            &mca_sensor_file_component.check_size);
-
-    mca_sensor_file_component.check_access = false;
-    (void) mca_base_component_var_register (c, "check_access", "Check access time",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                            &mca_sensor_file_component.check_access);
-
-    mca_sensor_file_component.check_mod = false;
-    (void) mca_base_component_var_register (c, "check_mod", "Check modification time",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                            &mca_sensor_file_component.check_mod);
-
-    mca_sensor_file_component.limit = 3;
-    (void) mca_base_component_var_register (c, "limit",
-                                            "Number of times the sensor can detect no motion before declaring error (default=3)",
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                            &mca_sensor_file_component.limit);
-    return ORTE_SUCCESS;
-}
-
-static int orte_sensor_file_open(void)
-{
-    return ORTE_SUCCESS;
-}
-
-
-static int orte_sensor_file_query(mca_base_module_t **module, int *priority)
-{
-    *priority = 20;  /* higher than heartbeat */
-    *module = (mca_base_module_t *)&orte_sensor_file_module;
-    return ORTE_SUCCESS;
-}
-
-/**
- *  Close all subsystems.
- */
-
-static int orte_sensor_file_close(void)
-{
-    return ORTE_SUCCESS;
-}

orte/mca/sensor/ft_tester/Makefile.am

@@ -1,36 +0,0 @@
-#
-# Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved. 
-#
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-sources = \
-        sensor_ft_tester.c \
-        sensor_ft_tester.h \
-        sensor_ft_tester_component.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_orte_sensor_ft_tester_DSO
-component_noinst =
-component_install = mca_sensor_ft_tester.la
-else
-component_noinst = libmca_sensor_ft_tester.la
-component_install =
-endif
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_sensor_ft_tester_la_SOURCES = $(sources)
-mca_sensor_ft_tester_la_LDFLAGS = -module -avoid-version
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_sensor_ft_tester_la_SOURCES =$(sources)
-libmca_sensor_ft_tester_la_LDFLAGS = -module -avoid-version

orte/mca/sensor/ft_tester/configure.m4

@@ -1,24 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2011      Cisco Systems, Inc. All rights reserved.
-# Copyright (c) 2011-2013 Los Alamos National Security, LLC.
-#                         All rights reserved.
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-# MCA_sensor_ft_tester_CONFIG([action-if-found], [action-if-not-found])
-# -----------------------------------------------------------
-AC_DEFUN([MCA_orte_sensor_ft_tester_CONFIG], [
-    AC_CONFIG_FILES([orte/mca/sensor/ft_tester/Makefile])
-
-    # if we don't want sensors, don't compile
-    # this component
-    AS_IF([test "$orte_want_sensors" = "1"],
-        [$1], [$2])
-])dnl
-

orte/mca/sensor/ft_tester/sensor_ft_tester.h

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved. 
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-/**
- * @file
- *
- * Process Resource Utilization sensor 
- */
-#ifndef ORTE_SENSOR_FT_TESTER_H
-#define ORTE_SENSOR_FT_TESTER_H
-
-#include "orte_config.h"
-
-#include "orte/mca/sensor/sensor.h"
-#include "opal/util/alfg.h"
-
-BEGIN_C_DECLS
-
-struct orte_sensor_ft_tester_component_t {
-    orte_sensor_base_component_t super;
-    float fail_prob;
-    float daemon_fail_prob;
-    bool multi_fail;
-};
-typedef struct orte_sensor_ft_tester_component_t orte_sensor_ft_tester_component_t;
-
-ORTE_MODULE_DECLSPEC extern orte_sensor_ft_tester_component_t mca_sensor_ft_tester_component;
-extern orte_sensor_base_module_t orte_sensor_ft_tester_module;
-
-extern opal_rng_buff_t orte_sensor_ft_rng_buff;
-
-END_C_DECLS
-
-#endif

orte/mca/sensor/ft_tester/sensor_ft_tester_component.c

@@ -1,141 +0,0 @@
-/*
- * Copyright (c) 2010-2011 Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#include "opal/mca/base/base.h"
-#include "opal/util/output.h"
-#include "opal/class/opal_pointer_array.h"
-
-#include "orte/util/proc_info.h"
-#include "orte/util/show_help.h"
-
-#include "sensor_ft_tester.h"
-
-/*
- * Local functions
- */
-static int orte_sensor_ft_tester_register (void);
-static int orte_sensor_ft_tester_open(void);
-static int orte_sensor_ft_tester_close(void);
-static int orte_sensor_ft_tester_query(mca_base_module_t **module, int *priority);
-
-orte_sensor_ft_tester_component_t mca_sensor_ft_tester_component = {
-    {
-        {
-            ORTE_SENSOR_BASE_VERSION_1_0_0,
-            
-            "ft_tester", /* MCA component name */
-            ORTE_MAJOR_VERSION,  /* MCA component major version */
-            ORTE_MINOR_VERSION,  /* MCA component minor version */
-            ORTE_RELEASE_VERSION,  /* MCA component release version */
-            orte_sensor_ft_tester_open,  /* component open  */
-            orte_sensor_ft_tester_close, /* component close */
-            orte_sensor_ft_tester_query, /* component query */
-            orte_sensor_ft_tester_register
-        },
-        {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-        NULL
-    }
-};
-
-static char *daemon_fail_prob = NULL;
-static char *fail_prob = NULL;
-opal_rng_buff_t orte_sensor_ft_rng_buff;
-
-/**
-  * component register/open/close/init function
-  */
-static int orte_sensor_ft_tester_register (void)
-{
-    mca_base_component_t *c = &mca_sensor_ft_tester_component.super.base_version;
-
-    fail_prob = NULL;
-    (void) mca_base_component_var_register (c, "fail_prob", "Probability of killing a single executable",
-                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &fail_prob);
-
-    mca_sensor_ft_tester_component.multi_fail = false;
-    (void) mca_base_component_var_register (c, "multi_allowed", "Allow multiple executables to be killed at one time",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &mca_sensor_ft_tester_component.multi_fail);
-
-    daemon_fail_prob = NULL;
-    (void) mca_base_component_var_register (c, "daemon_fail_prob", "Probability of killing a daemon",
-                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &daemon_fail_prob);
-
-    return ORTE_SUCCESS;
-}
-
-static int orte_sensor_ft_tester_open(void)
-{
-    /* lookup parameters */
-    if (NULL != fail_prob) {
-        mca_sensor_ft_tester_component.fail_prob = strtof(fail_prob, NULL);
-        if (1.0 < mca_sensor_ft_tester_component.fail_prob) {
-            /* given in percent */
-            mca_sensor_ft_tester_component.fail_prob /= 100.0;
-        }
-    } else {
-        mca_sensor_ft_tester_component.fail_prob = 0.0;
-    }
-
-    if (NULL != daemon_fail_prob) {
-        mca_sensor_ft_tester_component.daemon_fail_prob = strtof(daemon_fail_prob, NULL);
-        if (1.0 < mca_sensor_ft_tester_component.daemon_fail_prob) {
-            /* given in percent */
-            mca_sensor_ft_tester_component.daemon_fail_prob /= 100.0;
-        }
-    } else {
-        mca_sensor_ft_tester_component.daemon_fail_prob = 0.0;
-    }
-    
-    return ORTE_SUCCESS;
-}
-
-
-static int orte_sensor_ft_tester_query(mca_base_module_t **module, int *priority)
-{
-    if (0.0 < mca_sensor_ft_tester_component.fail_prob ||
-        0.0 < mca_sensor_ft_tester_component.daemon_fail_prob) {
-        *priority = 1;  /* at the bottom */
-        *module = (mca_base_module_t *)&orte_sensor_ft_tester_module;
-        /* seed the RNG --- Not sure if we should assume all procs use 
-         * the same seed? 
-         */
-        opal_srand(&orte_sensor_ft_rng_buff, (uint32_t) getpid());
-        return ORTE_SUCCESS;
-    }
-    *priority = 0;
-    *module = NULL;
-    return ORTE_ERROR;
-
-}
-
-/**
- *  Close all subsystems.
- */
-
-static int orte_sensor_ft_tester_close(void)
-{
-    return ORTE_SUCCESS;
-}

orte/mca/sensor/heartbeat/Makefile.am

@@ -1,38 +0,0 @@
-#
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
-#
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-dist_ompidata_DATA = help-orte-sensor-heartbeat.txt
-
-sources = \
-        sensor_heartbeat.c \
-        sensor_heartbeat.h \
-        sensor_heartbeat_component.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_orte_sensor_heartbeat_DSO
-component_noinst =
-component_install = mca_sensor_heartbeat.la
-else
-component_noinst = libmca_sensor_heartbeat.la
-component_install =
-endif
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_sensor_heartbeat_la_SOURCES = $(sources)
-mca_sensor_heartbeat_la_LDFLAGS = -module -avoid-version
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_sensor_heartbeat_la_SOURCES =$(sources)
-libmca_sensor_heartbeat_la_LDFLAGS = -module -avoid-version

orte/mca/sensor/heartbeat/configure.m4

@@ -1,24 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
-# Copyright (c) 2011-2013 Los Alamos National Security, LLC.
-#                         All rights reserved.
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-# MCA_sensor_heartbeat_CONFIG([action-if-found], [action-if-not-found])
-# -----------------------------------------------------------
-AC_DEFUN([MCA_orte_sensor_heartbeat_CONFIG], [
-    AC_CONFIG_FILES([orte/mca/sensor/heartbeat/Makefile])
-
-    # if we don't want sensors, don't compile
-    # this component
-    AS_IF([test "$orte_want_sensors" = "1"],
-        [$1], [$2])
-])dnl
-

orte/mca/sensor/heartbeat/sensor_heartbeat.c

@@ -1,279 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC.  All rights
- *                         reserved. 
-  *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-#include "orte/types.h"
-
-#include <errno.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif  /* HAVE_STRING_H */
-#include <stdio.h>
-
-#include "opal_stdint.h"
-#include "opal/util/argv.h"
-#include "opal/util/output.h"
-#include "opal/mca/event/event.h"
-
-#include "orte/util/show_help.h"
-#include "orte/util/proc_info.h"
-#include "orte/util/name_fns.h"
-#include "orte/mca/errmgr/errmgr.h"
-#include "orte/mca/rml/rml.h"
-#include "orte/mca/state/state.h"
-#include "orte/runtime/orte_wait.h"
-#include "orte/runtime/orte_globals.h"
-
-#include "orte/mca/sensor/base/base.h"
-#include "orte/mca/sensor/base/sensor_private.h"
-#include "sensor_heartbeat.h"
-
-/* declare the API functions */
-static int init(void);
-static void finalize(void);
-static void start(orte_jobid_t job);
-static void sample(void);
-
-/* instantiate the module */
-orte_sensor_base_module_t orte_sensor_heartbeat_module = {
-    init,
-    finalize,
-    start,
-    NULL,
-    sample,
-    NULL
-};
-
-/* declare the local functions */
-static void check_heartbeat(int fd, short event, void *arg);
-static void recv_beats(int status, orte_process_name_t* sender,
-                       opal_buffer_t *buffer,
-                       orte_rml_tag_t tag, void *cbdata);
-
-/* local globals */
-static orte_job_t *daemons=NULL;
-static opal_event_t check_ev;
-static bool check_active = false;
-static struct timeval check_time;
-
-static int init(void)
-{
-    OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                         "%s initializing heartbeat recvs",
-                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-
-    /* setup to receive heartbeats */
-    if (ORTE_PROC_IS_HNP || ORTE_PROC_IS_AGGREGATOR) {
-        orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD,
-                                ORTE_RML_TAG_HEARTBEAT,
-                                ORTE_RML_PERSISTENT,
-                                recv_beats, NULL);
-    }
-
-    if (ORTE_PROC_IS_HNP) {
-        daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
-    }
-
-    return ORTE_SUCCESS;
-}
-
-static void finalize(void)
-{
-    orte_rml.recv_cancel(ORTE_NAME_WILDCARD, ORTE_RML_TAG_HEARTBEAT);
-    if (check_active) {
-        opal_event_del(&check_ev);
-        check_active = false;
-    }
-    return;
-}
-
-static void start(orte_jobid_t job)
-{
-    if (!check_active && NULL != daemons) {
-        /* setup the check event */
-        check_time.tv_sec = 3 * orte_sensor_base.rate.tv_sec;
-        check_time.tv_usec = 0;
-        opal_event_evtimer_set(orte_event_base, &check_ev, check_heartbeat, &check_ev);
-        opal_event_evtimer_add(&check_ev, &check_time);
-        check_active = true;
-    }
-}
-
-static void sample(void)
-{
-    opal_buffer_t *buf;
-    int rc;
-    orte_process_name_t *tgt;
-
-    /* if we are aborting or shutting down, ignore this */
-    if (orte_abnormal_term_ordered || orte_finalizing || !orte_initialized) {
-        return;
-    }
-
-    if (ORTE_PROC_IS_CM) {
-        /* we send to our daemon */
-        tgt = ORTE_PROC_MY_DAEMON;
-    } else {
-        tgt = ORTE_PROC_MY_HNP;
-    }
-    /* if my target hasn't been defined yet, ignore - nobody listening yet */
-    if (ORTE_JOBID_INVALID ==tgt->jobid ||
-        ORTE_VPID_INVALID == tgt->vpid) {
-        opal_output_verbose(1, orte_sensor_base_framework.framework_output,
-                            "%s sensor:heartbeat: HNP is not defined",
-                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
-        return;
-    }
-
-    OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                         "%s sending heartbeat",
-                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-
-    /* if we want sampled data included, point to the bucket */
-    buf = OBJ_NEW(opal_buffer_t);
-    if (orte_sensor_base.log_samples) {
-        opal_dss.copy_payload(buf, orte_sensor_base.samples);
-        OBJ_RELEASE(orte_sensor_base.samples);
-        /* start a new sample bucket */
-        orte_sensor_base.samples = OBJ_NEW(opal_buffer_t);
-    }
-
-    /* send heartbeat */
-    if (ORTE_SUCCESS != (rc = orte_rml.send_buffer_nb(tgt, buf,
-                                                      ORTE_RML_TAG_HEARTBEAT,
-                                                      orte_rml_send_callback, NULL))) {
-        ORTE_ERROR_LOG(rc);
-        OBJ_RELEASE(buf);
-    }
-}
-
-/* this function automatically gets periodically called
- * by the event library so we can check on the state
- * of the various orteds
- */
-static void check_heartbeat(int fd, short dummy, void *arg)
-{
-    int v;
-    orte_proc_t *proc;
-    opal_event_t *tmp = (opal_event_t*)arg;
-
-    OPAL_OUTPUT_VERBOSE((3, orte_sensor_base_framework.framework_output,
-                         "%s sensor:check_heartbeat",
-                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-    
-    /* if we are aborting or shutting down, ignore this */
-    if (orte_abnormal_term_ordered || orte_finalizing || !orte_initialized) {
-        OPAL_OUTPUT_VERBOSE((3,  orte_sensor_base_framework.framework_output,
-                             "%s IGNORING CHECK abnorm_term %s fin %s init %s",
-                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                             orte_abnormal_term_ordered ? "TRUE" : "FALSE",
-                             orte_finalizing ? "TRUE" : "FALSE",
-                             orte_initialized ? "TRUE" : "FALSE"));
-        check_active = false;
-        return;
-    }
-    
-    for (v=0; v < daemons->procs->size; v++) {
-        if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(daemons->procs, v))) {
-            continue;
-        }
-        /* ignore myself */
-        if (proc->name.vpid == ORTE_PROC_MY_NAME->vpid) {
-            continue;
-        }
-        if (ORTE_PROC_STATE_RUNNING != proc->state) {
-            OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                                 "%s sensor:heartbeat DAEMON %s IS NOT RUNNING",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 ORTE_NAME_PRINT(&proc->name)));
-            continue;
-        }
-
-        if (0 == proc->beat) {
-            /* no heartbeat recvd in last window */
-            OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                                 "%s sensor:check_heartbeat FAILED for daemon %s",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 ORTE_NAME_PRINT(&proc->name)));
-            ORTE_ACTIVATE_PROC_STATE(&proc->name, ORTE_PROC_STATE_HEARTBEAT_FAILED);
-        } else {
-            OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                                 "%s HEARTBEAT DETECTED FOR %s: NUM BEATS %d",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 ORTE_NAME_PRINT(&proc->name), proc->beat));
-        }
-        /* reset for next period */
-        proc->beat = 0;
-    }
-
-    /* reset the timer */
-    opal_event_evtimer_add(tmp, &check_time);
-}
-
-static void recv_beats(int status, orte_process_name_t* sender,
-                       opal_buffer_t *buffer,
-                       orte_rml_tag_t tag, void *cbdata)
-{
-    orte_proc_t *proc;
-    int rc, n;
-    char *component=NULL;
-    opal_buffer_t *buf;
-
-    opal_output_verbose(1, orte_sensor_base_framework.framework_output,
-                        "%s received beat from %s",
-                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                        ORTE_NAME_PRINT(sender));
-
-    /* if we are aborting or shutting down, ignore this */
-    if (orte_abnormal_term_ordered || orte_finalizing || !orte_initialized) {
-        return;
-    }
-
-    /* get this daemon's object */
-    if (NULL != daemons) {
-        if (NULL != (proc = (orte_proc_t*)opal_pointer_array_get_item(daemons->procs, sender->vpid))) {
-            OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                                 "%s marked beat from %s",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 ORTE_NAME_PRINT(sender)));
-            proc->beat++;
-            /* if this daemon has reappeared, reset things */
-            if (ORTE_PROC_STATE_HEARTBEAT_FAILED == proc->state) {
-                proc->state = ORTE_PROC_STATE_RUNNING;
-            }
-        }
-    }
-
-    /* unload any sampled data */
-    n=1;
-    while (OPAL_SUCCESS == (rc = opal_dss.unpack(buffer, &buf, &n, OPAL_BUFFER))) {
-        if (NULL != buf) {
-            n=1;
-            if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &component, &n, OPAL_STRING))) {
-                ORTE_ERROR_LOG(rc);
-                break;
-            }
-            orte_sensor_base_log(component, buf);
-            OBJ_RELEASE(buf);
-            free(component);
-            n=1;
-        }
-    }
-    if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
-        ORTE_ERROR_LOG(rc);
-    }
-}

orte/mca/sensor/heartbeat/sensor_heartbeat.h

@@ -1,32 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-/**
- * @file
- *
- * Heartbeat sensor 
- */
-#ifndef ORTE_SENSOR_HEARTBEAT_H
-#define ORTE_SENSOR_HEARTBEAT_H
-
-#include "orte_config.h"
-
-#include "orte/mca/sensor/sensor.h"
-
-BEGIN_C_DECLS
-
-ORTE_MODULE_DECLSPEC extern orte_sensor_base_component_t mca_sensor_heartbeat_component;
-extern orte_sensor_base_module_t orte_sensor_heartbeat_module;
-
-
-END_C_DECLS
-
-#endif

orte/mca/sensor/heartbeat/sensor_heartbeat_component.c

@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#include "opal/mca/base/base.h"
-#include "opal/util/output.h"
-#include "opal/class/opal_pointer_array.h"
-
-#include "orte/util/proc_info.h"
-#include "orte/util/show_help.h"
-
-#include "sensor_heartbeat.h"
-
-/*
- * Local functions
- */
-
-static int orte_sensor_heartbeat_open(void);
-static int orte_sensor_heartbeat_close(void);
-static int orte_sensor_heartbeat_query(mca_base_module_t **module, int *priority);
-
-orte_sensor_base_component_t mca_sensor_heartbeat_component = {
-    {
-        ORTE_SENSOR_BASE_VERSION_1_0_0,
-            
-        "heartbeat", /* MCA component name */
-        ORTE_MAJOR_VERSION,  /* MCA component major version */
-        ORTE_MINOR_VERSION,  /* MCA component minor version */
-        ORTE_RELEASE_VERSION,  /* MCA component release version */
-        orte_sensor_heartbeat_open,  /* component open  */
-        orte_sensor_heartbeat_close, /* component close */
-        orte_sensor_heartbeat_query  /* component query */
-    },
-    {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-    "heartbeat"
-};
-
-
-/**
-  * component open/close/init function
-  */
-static int orte_sensor_heartbeat_open(void)
-{
-    return ORTE_SUCCESS;
-}
-
-
-static int orte_sensor_heartbeat_query(mca_base_module_t **module, int *priority)
-{
-    *priority = 5;  /* lower than all other samplers so that their data gets included in heartbeat */
-    *module = (mca_base_module_t *)&orte_sensor_heartbeat_module;
-    return ORTE_SUCCESS;
-}
-
-/**
- *  Close all subsystems.
- */
-
-static int orte_sensor_heartbeat_close(void)
-{
-    return ORTE_SUCCESS;
-}

orte/mca/sensor/resusage/Makefile.am

@@ -1,38 +0,0 @@
-#
-# Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved. 
-#
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-dist_ompidata_DATA = help-orte-sensor-resusage.txt
-
-sources = \
-        sensor_resusage.c \
-        sensor_resusage.h \
-        sensor_resusage_component.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_orte_sensor_resusage_DSO
-component_noinst =
-component_install = mca_sensor_resusage.la
-else
-component_noinst = libmca_sensor_resusage.la
-component_install =
-endif
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_sensor_resusage_la_SOURCES = $(sources)
-mca_sensor_resusage_la_LDFLAGS = -module -avoid-version
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_sensor_resusage_la_SOURCES =$(sources)
-libmca_sensor_resusage_la_LDFLAGS = -module -avoid-version

orte/mca/sensor/resusage/configure.m4

@@ -1,24 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
-# Copyright (c) 2011-2013 Los Alamos National Security, LLC.
-#                         All rights reserved.
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-# MCA_sensor_resusage_CONFIG([action-if-found], [action-if-not-found])
-# -----------------------------------------------------------
-AC_DEFUN([MCA_orte_sensor_resusage_CONFIG], [
-    AC_CONFIG_FILES([orte/mca/sensor/resusage/Makefile])
-
-    # if we don't want sensors, don't compile
-    # this component
-    AS_IF([test "$orte_want_sensors" = "1"],
-        [$1], [$2])
-])dnl
-

orte/mca/sensor/resusage/help-orte-sensor-resusage.txt

@@ -1,21 +0,0 @@
-# -*- text -*-
-#
-# Copyright (c) 2010-2011 Cisco Systems, Inc.  All rights reserved.
-#
-# Copyright (c) 2017      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-# This is the US/English general help file for the memory usage sensor
-#
-[mem-limit-exceeded]
-A process has exceeded the specified limit on memory usage:
-
-Node:         %s
-Process rank: %s
-Memory used:  %luGbytes
-Memory limit: %luGbytes
-

orte/mca/sensor/resusage/sensor_resusage.c

@@ -1,478 +0,0 @@
-/*
- * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC.  All rights
- *                         reserved. 
-  *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-#include "orte/types.h"
-
-#include <errno.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif  /* HAVE_STRING_H */
-#include <stdio.h>
-
-#include "opal_stdint.h"
-#include "opal/class/opal_pointer_array.h"
-#include "opal/class/opal_ring_buffer.h"
-#include "opal/dss/dss.h"
-#include "opal/util/output.h"
-#include "opal/mca/pstat/pstat.h"
-#include "opal/mca/db/db.h"
-
-#include "orte/util/proc_info.h"
-#include "orte/util/name_fns.h"
-#include "orte/mca/errmgr/errmgr.h"
-#include "orte/mca/odls/odls_types.h"
-#include "orte/mca/odls/base/odls_private.h"
-#include "orte/mca/rml/rml.h"
-#include "orte/mca/state/state.h"
-#include "orte/runtime/orte_globals.h"
-#include "orte/orted/orted.h"
-
-#include "orte/mca/sensor/base/base.h"
-#include "orte/mca/sensor/base/sensor_private.h"
-#include "sensor_resusage.h"
-
-/* declare the API functions */
-static int init(void);
-static void finalize(void);
-static void sample(void);
-static void res_log(opal_buffer_t *sample);
-
-/* instantiate the module */
-orte_sensor_base_module_t orte_sensor_resusage_module = {
-    init,
-    finalize,
-    NULL,
-    NULL,
-    sample,
-    res_log
-};
-
-static bool log_enabled = true;
-static orte_node_t *my_node;
-static orte_proc_t *my_proc;
-
-static int init(void)
-{
-    orte_job_t *jdata;
-
-    /* ensure my_proc and my_node are available on the global arrays */
-    if (NULL == (jdata = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
-        my_proc = OBJ_NEW(orte_proc_t);
-        my_node = OBJ_NEW(orte_node_t);
-    } else {
-        if (NULL == (my_proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, ORTE_PROC_MY_NAME->vpid))) {
-            ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
-            return ORTE_ERR_NOT_FOUND;
-        }
-        if (NULL == (my_node = my_proc->node)) {
-            ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
-            return ORTE_ERR_NOT_FOUND;
-        }
-        /* protect the objects */
-        OBJ_RETAIN(my_proc);
-        OBJ_RETAIN(my_node);
-    }
-
-    return ORTE_SUCCESS;
-}
-
-static void finalize(void)
-{
-    if (NULL != my_proc) {
-        OBJ_RELEASE(my_proc);
-    }
-    if (NULL != my_node) {
-        OBJ_RELEASE(my_node);
-    }
-    return;
-}
-
-static void sample(void)
-{
-    opal_pstats_t *stats, *st;
-    opal_node_stats_t *nstats, *nst;
-    int rc, i;
-    orte_proc_t *child, *hog=NULL;
-    float in_use, max_mem;
-    opal_buffer_t buf, *bptr;
-    char *comp;
-
-    OPAL_OUTPUT_VERBOSE((1, orte_sensor_base_framework.framework_output,
-                         "sample:resusage sampling resource usage"));
-    
-    /* setup a buffer for our stats */
-    OBJ_CONSTRUCT(&buf, opal_buffer_t);
-    /* pack our name */
-    comp = strdup("resusage");
-    if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &comp, 1, OPAL_STRING))) {
-        ORTE_ERROR_LOG(rc);
-        OBJ_DESTRUCT(&buf);
-        return;
-    }
-    free(comp);
-
-    /* update stats on ourself and the node */
-    stats = OBJ_NEW(opal_pstats_t);
-    nstats = OBJ_NEW(opal_node_stats_t);
-    if (ORTE_SUCCESS != (rc = opal_pstat.query(orte_process_info.pid, stats, nstats))) {
-        ORTE_ERROR_LOG(rc);
-        OBJ_DESTRUCT(stats);
-        OBJ_RELEASE(nstats);
-        OBJ_DESTRUCT(&buf);
-        return;
-    }
-
-    /* the stats framework can't know nodename or rank */
-    strncpy(stats->node, orte_process_info.nodename, OPAL_PSTAT_MAX_STRING_LEN);
-    stats->rank = ORTE_PROC_MY_NAME->vpid;
-    /* locally save the stats */
-    if (NULL != (st = (opal_pstats_t*)opal_ring_buffer_push(&my_proc->stats, stats))) {
-        OBJ_RELEASE(st);
-    }
-    if (NULL != (nst = (opal_node_stats_t*)opal_ring_buffer_push(&my_node->stats, nstats))) {
-        /* release the popped value */
-        OBJ_RELEASE(nst);
-    }
-
-    /* pack them */
-    if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &orte_process_info.nodename, 1, OPAL_STRING))) {
-        ORTE_ERROR_LOG(rc);
-        OBJ_DESTRUCT(&buf);
-        return;
-    }
-    if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &nstats, 1, OPAL_NODE_STAT))) {
-        ORTE_ERROR_LOG(rc);
-        OBJ_DESTRUCT(&buf);
-        return;
-    }
-    if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &stats, 1, OPAL_PSTAT))) {
-        ORTE_ERROR_LOG(rc);
-        OBJ_DESTRUCT(&buf);
-        return;
-    }
-
-    /* loop through our children and update their stats */
-    if (NULL != orte_local_children) {
-        for (i=0; i < orte_local_children->size; i++) {
-            if (NULL == (child = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i))) {
-                continue;
-            }
-            if (!child->alive) {
-                continue;
-            }
-            if (0 == child->pid) {
-                /* race condition */
-                continue;
-            }
-            stats = OBJ_NEW(opal_pstats_t);
-            if (ORTE_SUCCESS != opal_pstat.query(child->pid, stats, NULL)) {
-                /* may hit a race condition where the process has
-                 * terminated, so just ignore any error
-                 */
-                OBJ_RELEASE(stats);
-                continue;
-            }
-            /* the stats framework can't know nodename or rank */
-            strncpy(stats->node, orte_process_info.nodename, OPAL_PSTAT_MAX_STRING_LEN);
-            stats->rank = child->name.vpid;
-            /* store it */
-            if (NULL != (st = (opal_pstats_t*)opal_ring_buffer_push(&child->stats, stats))) {
-                OBJ_RELEASE(st);
-            }
-            /* pack them */
-            if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &stats, 1, OPAL_PSTAT))) {
-                ORTE_ERROR_LOG(rc);
-                OBJ_DESTRUCT(&buf);
-                return;
-            }
-        }
-    }
-
-    /* xfer any data for transmission */
-    if (0 < buf.bytes_used) {
-        bptr = &buf;
-        if (OPAL_SUCCESS != (rc = opal_dss.pack(orte_sensor_base.samples, &bptr, 1, OPAL_BUFFER))) {
-            ORTE_ERROR_LOG(rc);
-            OBJ_DESTRUCT(&buf);
-            return;
-        }
-    }
-    OBJ_DESTRUCT(&buf);
-
-    /* are there any issues with node-level usage? */
-    nst = (opal_node_stats_t*)opal_ring_buffer_poke(&my_node->stats, -1);
-    if (NULL != nst && 0.0 < mca_sensor_resusage_component.node_memory_limit) {
-        OPAL_OUTPUT_VERBOSE((2, orte_sensor_base_framework.framework_output,
-                             "%s CHECKING NODE MEM",
-                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-        /* compute the percentage of node memory in-use */
-        in_use = 1.0 - (nst->free_mem / nst->total_mem);
-        OPAL_OUTPUT_VERBOSE((2, orte_sensor_base_framework.framework_output,
-                             "%s PERCENT USED: %f LIMIT: %f",
-                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                             in_use, mca_sensor_resusage_component.node_memory_limit));
-        if (mca_sensor_resusage_component.node_memory_limit <= in_use) {
-            /* loop through our children and find the biggest hog */
-            hog = NULL;
-            max_mem = 0.0;
-            for (i=0; i < orte_local_children->size; i++) {
-                if (NULL == (child = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i))) {
-                    continue;
-                }
-                if (!child->alive) {
-                    continue;
-                }
-                if (0 == child->pid) {
-                    /* race condition */
-                    continue;
-                }
-                if (NULL == (st = (opal_pstats_t*)opal_ring_buffer_poke(&child->stats, -1))) {
-                    continue;
-                }
-                OPAL_OUTPUT_VERBOSE((5, orte_sensor_base_framework.framework_output,
-                                     "%s PROC %s AT VSIZE %f",
-                                     ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                     ORTE_NAME_PRINT(&child->name), st->vsize));
-                if (max_mem < st->vsize) {
-                    hog = child;
-                    max_mem = st->vsize;
-                }
-            }
-            if (NULL == hog) {
-                /* if all children dead and we are still too big,
-                 * then we must be the culprit - abort
-                 */
-                OPAL_OUTPUT_VERBOSE((2, orte_sensor_base_framework.framework_output,
-                                     "%s NO CHILD: COMMITTING SUICIDE",
-                                     ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-                orte_errmgr.abort(ORTE_ERR_MEM_LIMIT_EXCEEDED, NULL);
-            } else {
-                /* report the problem */
-                OPAL_OUTPUT_VERBOSE((2, orte_sensor_base_framework.framework_output,
-                                     "%s REPORTING %s TO ERRMGR FOR EXCEEDING LIMITS",
-                                     ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                     ORTE_NAME_PRINT(&hog->name)));
-                ORTE_ACTIVATE_PROC_STATE(&hog->name, ORTE_PROC_STATE_SENSOR_BOUND_EXCEEDED);
-            }
-            /* since we have ordered someone to die, we've done enough for this
-             * time around - don't check proc limits as well
-             */
-            return;
-        }
-    }
-
-    /* check proc limits */
-    if (0.0 < mca_sensor_resusage_component.proc_memory_limit) {
-        OPAL_OUTPUT_VERBOSE((2, orte_sensor_base_framework.framework_output,
-                             "%s CHECKING PROC MEM",
-                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-        /* check my children first */
-        for (i=0; i < orte_local_children->size; i++) {
-            if (NULL == (child = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i))) {
-                continue;
-            }
-            if (!child->alive) {
-                continue;
-            }
-            if (0 == child->pid) {
-                /* race condition */
-                continue;
-            }
-            if (NULL == (st = (opal_pstats_t*)opal_ring_buffer_poke(&child->stats, -1))) {
-                continue;
-            }
-            OPAL_OUTPUT_VERBOSE((5, orte_sensor_base_framework.framework_output,
-                                 "%s PROC %s AT VSIZE %f",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 ORTE_NAME_PRINT(&child->name), st->vsize));
-            if (mca_sensor_resusage_component.proc_memory_limit <= st->vsize) {
-                /* report the problem */
-                ORTE_ACTIVATE_PROC_STATE(&child->name, ORTE_PROC_STATE_SENSOR_BOUND_EXCEEDED);
-            }
-        }
-    }
-}
-
-static void res_log(opal_buffer_t *sample)
-{
-    opal_pstats_t *st=NULL;
-    opal_node_stats_t *nst=NULL;
-    int rc, n, i;
-    opal_value_t kv[14];
-    char *node;
-
-    if (!log_enabled) {
-        return;
-    }
-
-    /* unpack the node name */
-    n=1;
-    if (OPAL_SUCCESS != (rc = opal_dss.unpack(sample, &node, &n, OPAL_STRING))) {
-        ORTE_ERROR_LOG(rc);
-        return;
-    }
-
-    /* unpack the node stats */
-    n=1;
-    if (OPAL_SUCCESS != (rc = opal_dss.unpack(sample, &nst, &n, OPAL_NODE_STAT))) {
-        ORTE_ERROR_LOG(rc);
-        return;
-    }
-
-    if (mca_sensor_resusage_component.log_node_stats) {
-        /* convert this into an array of opal_value_t's - no clean way
-         * to do this, so have to just manually map each field
-         */
-        for (i=0; i < 13; i++) {
-            OBJ_CONSTRUCT(&kv[i], opal_value_t);
-        }
-        i=0;
-        kv[i].key = strdup("ctime");
-        kv[i].type = OPAL_TIMEVAL;
-        kv[i].data.tv.tv_sec = nst->sample_time.tv_sec;
-        kv[i++].data.tv.tv_usec = nst->sample_time.tv_usec;
-
-        kv[i].key = "hostname";
-        kv[i].type = OPAL_STRING;
-        kv[i++].data.string = strdup(node);
-
-        kv[i].key = strdup("total_mem");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->total_mem;
-
-        kv[i].key = strdup("free_mem");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->free_mem;
-
-        kv[i].key = strdup("buffers");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->buffers;
-
-        kv[i].key = strdup("cached");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->cached;
-
-        kv[i].key = strdup("swap_total");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->swap_total;
-
-        kv[i].key = strdup("swap_free");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->swap_free;
-
-        kv[i].key = strdup("mapped");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->mapped;
-
-        kv[i].key = strdup("swap_cached");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->swap_cached;
-
-        kv[i].key = strdup("la");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->la;
-
-        kv[i].key = strdup("la5");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->la5;
-
-        kv[i].key = strdup("la15");
-        kv[i].type = OPAL_FLOAT;
-        kv[i++].data.fval = nst->la15;
-
-        /* store it */
-        if (ORTE_SUCCESS != (rc = opal_db.add_log("nodestats", kv, 12))) {
-            /* don't bark about it - just quietly disable the log */
-            log_enabled = false;
-        }
-        for (i=0; i < 12; i++) {
-            OBJ_DESTRUCT(&kv[i]);
-        }
-    }
-
-    OBJ_RELEASE(nst);
-
-    if (mca_sensor_resusage_component.log_process_stats) {
-        /* unpack all process stats */
-        n=1;
-        while (OPAL_SUCCESS == (rc = opal_dss.unpack(sample, &st, &n, OPAL_PSTAT))) {
-            for (i=0; i < 14; i++) {
-                OBJ_CONSTRUCT(&kv[i], opal_value_t);
-            }
-            kv[0].key = strdup("node");
-            kv[0].type = OPAL_STRING;
-            kv[0].data.string = strdup(st->node);
-            kv[1].key = strdup("rank");
-            kv[1].type = OPAL_INT32;
-            kv[1].data.int32 = st->rank;
-            kv[2].key = strdup("pid");
-            kv[2].type = OPAL_PID;
-            kv[2].data.pid = st->pid;
-            kv[3].key = strdup("cmd");
-            kv[3].type = OPAL_STRING;
-            kv[3].data.string = strdup(st->cmd);
-            kv[4].key = strdup("state");
-            kv[4].type = OPAL_STRING;
-            kv[4].data.string = (char*)malloc(3 * sizeof(char));
-            kv[4].data.string[0] = st->state[0];
-            kv[4].data.string[1] = st->state[1];
-            kv[4].data.string[2] = '\0';
-            kv[5].key = strdup("time");
-            kv[5].type = OPAL_TIMEVAL;
-            kv[5].data.tv.tv_sec = st->time.tv_sec;
-            kv[5].data.tv.tv_usec = st->time.tv_usec;
-            kv[6].key = strdup("percent_cpu");
-            kv[6].type = OPAL_FLOAT;
-            kv[6].data.fval = st->percent_cpu;
-            kv[7].key = strdup("priority");
-            kv[7].type = OPAL_INT32;
-            kv[7].data.int32 = st->priority;
-            kv[8].key = strdup("num_threads");
-            kv[8].type = OPAL_INT16;
-            kv[8].data.int16 = st->num_threads;
-            kv[9].key = strdup("vsize");
-            kv[9].type = OPAL_FLOAT;
-            kv[9].data.fval = st->vsize;
-            kv[10].key = strdup("rss");
-            kv[10].type = OPAL_FLOAT;
-            kv[10].data.fval = st->rss;
-            kv[11].key = strdup("peak_vsize");
-            kv[11].type = OPAL_FLOAT;
-            kv[11].data.fval = st->peak_vsize;
-            kv[12].key = strdup("processor");
-            kv[12].type = OPAL_INT16;
-            kv[12].data.int16 = st->processor;
-            kv[13].key = strdup("sample_time");
-            kv[13].type = OPAL_TIMEVAL;
-            kv[13].data.tv.tv_sec = st->sample_time.tv_sec;
-            kv[13].data.tv.tv_usec = st->sample_time.tv_usec;
-            /* store it */
-            if (ORTE_SUCCESS != (rc = opal_db.add_log("procstats", kv, 14))) {
-                log_enabled = false;
-            }
-            for (i=0; i < 14; i++) {
-                OBJ_DESTRUCT(&kv[i]);
-            }
-            OBJ_RELEASE(st);
-            n=1;
-        }
-        if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
-            ORTE_ERROR_LOG(rc);
-        }
-    }
-}

orte/mca/sensor/resusage/sensor_resusage.h

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved. 
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-/**
- * @file
- *
- * Process Resource Utilization sensor 
- */
-#ifndef ORTE_SENSOR_RESUSAGE_H
-#define ORTE_SENSOR_RESUSAGE_H
-
-#include "orte_config.h"
-
-#include "orte/mca/sensor/sensor.h"
-
-BEGIN_C_DECLS
-
-struct orte_sensor_resusage_component_t {
-    orte_sensor_base_component_t super;
-    int sample_rate;
-    float node_memory_limit;
-    float proc_memory_limit;
-    bool log_node_stats;
-    bool log_process_stats;
-};
-typedef struct orte_sensor_resusage_component_t orte_sensor_resusage_component_t;
-
-ORTE_MODULE_DECLSPEC extern orte_sensor_resusage_component_t mca_sensor_resusage_component;
-extern orte_sensor_base_module_t orte_sensor_resusage_module;
-
-
-END_C_DECLS
-
-#endif

orte/mca/sensor/resusage/sensor_resusage_component.c

@@ -1,138 +0,0 @@
-/*
- * Copyright (c) 2010-2011 Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#include "opal/mca/base/base.h"
-#include "opal/util/output.h"
-#include "opal/class/opal_pointer_array.h"
-
-#include "orte/util/proc_info.h"
-#include "orte/util/show_help.h"
-
-#include "sensor_resusage.h"
-
-/*
- * Local functions
- */
-static int orte_sensor_resusage_register (void);
-static int orte_sensor_resusage_open(void);
-static int orte_sensor_resusage_close(void);
-static int orte_sensor_resusage_query(mca_base_module_t **module, int *priority);
-
-orte_sensor_resusage_component_t mca_sensor_resusage_component = {
-    {
-        {
-            ORTE_SENSOR_BASE_VERSION_1_0_0,
-            
-            "resusage", /* MCA component name */
-            ORTE_MAJOR_VERSION,  /* MCA component major version */
-            ORTE_MINOR_VERSION,  /* MCA component minor version */
-            ORTE_RELEASE_VERSION,  /* MCA component release version */
-            orte_sensor_resusage_open,  /* component open  */
-            orte_sensor_resusage_close, /* component close */
-            orte_sensor_resusage_query, /* component query */
-            orte_sensor_resusage_register
-        },
-        {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-        "procresource,noderesource"
-    }
-};
-
-static int node_memory_limit;
-static int proc_memory_limit;
-
-/**
-  * component open/close/init function
-  */
-static int orte_sensor_resusage_register (void)
-{
-    mca_base_component_t *c = &mca_sensor_resusage_component.super.base_version;
-
-    mca_sensor_resusage_component.sample_rate = 0;
-    (void) mca_base_component_var_register (c, "sample_rate", "Sample rate in seconds (default: 0)",
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &mca_sensor_resusage_component.sample_rate);
-    if (mca_sensor_resusage_component.sample_rate < 0) {
-        opal_output(0, "Illegal value %d - must be > 0", mca_sensor_resusage_component.sample_rate);
-        return ORTE_ERR_BAD_PARAM;
-    }
-
-    node_memory_limit = 0;
-    (void) mca_base_component_var_register (c, "node_memory_limit",
-                                            "Percentage of total memory that can be in-use",
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &node_memory_limit);
-    mca_sensor_resusage_component.node_memory_limit = (float)node_memory_limit/100.0;
-
-    proc_memory_limit = 0;
-    (void) mca_base_component_var_register (c, "proc_memory_limit",
-                                            "Max virtual memory size in MBytes",
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &proc_memory_limit);
-    mca_sensor_resusage_component.proc_memory_limit = (float) proc_memory_limit;
-    
-    mca_sensor_resusage_component.log_node_stats = false;
-    (void) mca_base_component_var_register (c, "log_node_stats", "Log the node stats",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &mca_sensor_resusage_component.log_node_stats);
-
-    mca_sensor_resusage_component.log_process_stats = false;
-    (void) mca_base_component_var_register (c, "log_process_stats", "Log the process stats",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &mca_sensor_resusage_component.log_process_stats);
-
-    return ORTE_SUCCESS;
-}
-
-static int orte_sensor_resusage_open(void)
-{
-    if (mca_sensor_resusage_component.sample_rate < 0) {
-        opal_output(0, "Illegal value %d - must be > 0", mca_sensor_resusage_component.sample_rate);
-        return ORTE_ERR_FATAL;
-    }
-
-    mca_sensor_resusage_component.node_memory_limit = (float) node_memory_limit/100.0;
-    mca_sensor_resusage_component.proc_memory_limit = (float) proc_memory_limit;
-
-    return ORTE_SUCCESS;
-}
-
-
-static int orte_sensor_resusage_query(mca_base_module_t **module, int *priority)
-{    
-    *priority = 100;  /* ahead of heartbeat */
-    *module = (mca_base_module_t *)&orte_sensor_resusage_module;
-    
-    return ORTE_SUCCESS;
-}
-
-/**
- *  Close all subsystems.
- */
-
-static int orte_sensor_resusage_close(void)
-{
-    return ORTE_SUCCESS;
-}

orte/mca/sensor/sensor.h

@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved. 
- * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * @file:
- *
- */
-
-#ifndef MCA_SENSOR_H
-#define MCA_SENSOR_H
-
-/*
- * includes
- */
-
-#include "orte_config.h"
-#include "orte/types.h"
-
-#include "opal/mca/mca.h"
-
-BEGIN_C_DECLS
-
-/*
- * Component functions - all MUST be provided!
- */
-
-/* start collecting data */
-typedef void (*orte_sensor_API_module_start_fn_t)(orte_jobid_t job);
-
-/* stop collecting data */
-typedef void (*orte_sensor_API_module_stop_fn_t)(orte_jobid_t job);
-
-/* API module */
-/*
- * Ver 1.0
- */
-struct orte_sensor_base_API_module_1_0_0_t {
-    orte_sensor_API_module_start_fn_t      start;
-    orte_sensor_API_module_stop_fn_t       stop;
-};
-
-typedef struct orte_sensor_base_API_module_1_0_0_t orte_sensor_base_API_module_1_0_0_t;
-typedef orte_sensor_base_API_module_1_0_0_t orte_sensor_base_API_module_t;
-
-/* initialize the module */
-typedef int (*orte_sensor_base_module_init_fn_t)(void);
-    
-/* finalize the module */
-typedef void (*orte_sensor_base_module_finalize_fn_t)(void);
-
-/* tell the module to sample its sensor */
-typedef void (*orte_sensor_base_module_sample_fn_t)(void);
-
-/* pass a buffer to the module for logging */
-typedef void (*orte_sensor_base_module_log_fn_t)(opal_buffer_t *sample);
-
-/*
- * Component modules Ver 1.0
- */
-struct orte_sensor_base_module_1_0_0_t {
-    orte_sensor_base_module_init_fn_t       init;
-    orte_sensor_base_module_finalize_fn_t   finalize;
-    orte_sensor_API_module_start_fn_t       start;
-    orte_sensor_API_module_stop_fn_t        stop;
-    orte_sensor_base_module_sample_fn_t     sample;
-    orte_sensor_base_module_log_fn_t        log;
-};
-
-typedef struct orte_sensor_base_module_1_0_0_t orte_sensor_base_module_1_0_0_t;
-typedef orte_sensor_base_module_1_0_0_t orte_sensor_base_module_t;
-
-/*
- * the standard component data structure
- */
-struct orte_sensor_base_component_1_0_0_t {
-    mca_base_component_t base_version;
-    mca_base_component_data_t base_data;
-    char *data_measured;
-};
-typedef struct orte_sensor_base_component_1_0_0_t orte_sensor_base_component_1_0_0_t;
-typedef orte_sensor_base_component_1_0_0_t orte_sensor_base_component_t;
-
-
-
-/*
- * Macro for use in components that are of type sensor v1.0.0
- */
-#define ORTE_SENSOR_BASE_VERSION_1_0_0 \
-  /* sensor v1.0 is chained to MCA v2.0 */ \
-  MCA_BASE_VERSION_2_0_0, \
-  /* sensor v1.0 */ \
-  "sensor", 1, 0, 0
-
-/* Global structure for accessing sensor functions
- */
-ORTE_DECLSPEC extern orte_sensor_base_API_module_t orte_sensor;  /* holds API function pointers */
-
-END_C_DECLS
-
-#endif /* MCA_SENSOR_H */

orte/mca/sensor/sensor_types.h

@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
- *
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-/** @file:
- */
-
-#ifndef ORTE_MCA_SENSOR_TYPES_H
-#define ORTE_MCA_SENSOR_TYPES_H
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif  /* HAVE_SYS_TIME_H */
-
-#include "opal/dss/dss_types.h"
-
-/*
- * General SENSOR types - instanced in runtime/orte_globals.c
- */
-
-BEGIN_C_DECLS
-
-enum {
-    ORTE_SENSOR_SCALE_LINEAR,
-    ORTE_SENSOR_SCALE_LOG,
-    ORTE_SENSOR_SCALE_SIGMOID
-};
-
-/*
- * Structure for passing data from sensors
- */
-typedef struct {
-    opal_object_t super;
-    char *sensor;
-    struct timeval timestamp;
-    opal_byte_object_t data;
-} orte_sensor_data_t;
-ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_sensor_data_t);
-
-END_C_DECLS
-
-#endif

orte/mca/state/state.h

@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -85,27 +86,19 @@ ORTE_DECLSPEC extern mca_base_framework_t orte_state_base_framework;
                             ORTE_JOBID_PRINT(shadow->jobid),		\
                             orte_job_state_to_str((s)),                 \
                             __FILE__, __LINE__);			\
-        /* sanity check */                                              \
-        if ((s) < 0) {                                                  \
-            assert(0);                                                  \
-        }                                                               \
         orte_state.activate_job_state(shadow, (s));                     \
     } while(0);
 
 #define ORTE_ACTIVATE_PROC_STATE(p, s)                                  \
     do {                                                                \
         orte_process_name_t *shadow=(p);                                \
-	opal_output_verbose(1, orte_state_base_framework.framework_output, \
+	    opal_output_verbose(1, orte_state_base_framework.framework_output, \
                             "%s ACTIVATE PROC %s STATE %s AT %s:%d",	\
                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),         \
                             (NULL == shadow) ? "NULL" :                 \
                             ORTE_NAME_PRINT(shadow),			\
                             orte_proc_state_to_str((s)),		\
                             __FILE__, __LINE__);			\
-        /* sanity check */                                              \
-        if ((s) < 0) {                                                  \
-            assert(0);                                                  \
-        }                                                               \
         orte_state.activate_proc_state(shadow, (s));                    \
     } while(0);
 

orte/orted/ft_tester.c

@@ -1,13 +1,13 @@
 /*
- * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved. 
+ * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
- * 
+ *
  * Additional copyrights may follow
- * 
+ *
  * $HEADER$
  */
 

orte/orted/pmix/pmix_server.c

@@ -102,7 +102,9 @@ static opal_pmix_server_module_t pmix_server = {
     .notify_event = pmix_server_notify_event,
     .query = pmix_server_query_fn,
     .tool_connected = pmix_tool_connected_fn,
-    .log = pmix_server_log_fn
+    .log = pmix_server_log_fn,
+    .allocate = pmix_server_alloc_fn,
+    .job_control = pmix_server_job_ctrl_fn
 };
 
 void pmix_server_register_params(void)
@@ -265,6 +267,12 @@ int pmix_server_init(void)
     kv->type = OPAL_BOOL;
     kv->data.flag = true;
     opal_list_append(&info, &kv->super);
+    /* tell the server to use its own internal monitoring */
+    kv = OBJ_NEW(opal_value_t);
+    kv->key = strdup(OPAL_PMIX_SERVER_ENABLE_MONITORING);
+    kv->type = OPAL_BOOL;
+    kv->data.flag = true;
+    opal_list_append(&info, &kv->super);
 
     /* setup the local server */
     if (ORTE_SUCCESS != (rc = opal_pmix.server_init(&pmix_server, &info))) {

orte/orted/pmix/pmix_server_dyn.c

@@ -511,3 +511,13 @@ int pmix_server_disconnect_fn(opal_list_t *procs, opal_list_t *info,
 
     return rc;
 }
+
+int pmix_server_alloc_fn(const opal_process_name_t *requestor,
+                         opal_pmix_alloc_directive_t dir,
+                         opal_list_t *info,
+                         opal_pmix_info_cbfunc_t cbfunc,
+                         void *cbdata)
+{
+    /* ORTE currently has no way of supporting allocation requests */
+    return ORTE_ERR_NOT_SUPPORTED;
+}

orte/orted/pmix/pmix_server_gen.c

@@ -40,10 +40,12 @@
 #include "orte/mca/errmgr/errmgr.h"
 #include "orte/mca/iof/iof.h"
 #include "orte/mca/rmaps/rmaps_types.h"
+#include "orte/mca/schizo/schizo.h"
 #include "orte/mca/state/state.h"
 #include "orte/util/name_fns.h"
 #include "orte/runtime/orte_globals.h"
 #include "orte/mca/rml/rml.h"
+#include "orte/mca/plm/plm.h"
 #include "orte/mca/plm/base/plm_private.h"
 
 #include "pmix_server_internal.h"
@@ -611,7 +613,15 @@ static void _query(int sd, short args, void *cbdata)
                      * and ask directly for the info - if rank=wildcard, then
                      * we need to xcast the request and collect the results */
                 }
-
+            } else if (0 == strcmp(q->keys[n], OPAL_PMIX_TIME_REMAINING)) {
+                kv = OBJ_NEW(opal_value_t);
+                kv->key = strdup(OPAL_PMIX_TIME_REMAINING);
+                kv->type = OPAL_UINT32;
+                if (ORTE_SUCCESS != orte_schizo.get_remaining_time(&kv->data.uint32)) {
+                    OBJ_RELEASE(kv);
+                } else {
+                    opal_list_append(results, &kv->super);
+                }
             }
         }
     }
@@ -813,3 +823,62 @@ void pmix_server_log_fn(opal_process_name_t *requestor,
         cbfunc(OPAL_SUCCESS, cbdata);
     }
 }
+
+int pmix_server_job_ctrl_fn(const opal_process_name_t *requestor,
+                            opal_list_t *targets,
+                            opal_list_t *info,
+                            opal_pmix_info_cbfunc_t cbfunc,
+                            void *cbdata)
+{
+    opal_value_t *val;
+    int rc, n;
+    orte_proc_t *proc;
+    opal_pointer_array_t parray, *ptrarray;
+    opal_namelist_t *nm;
+
+    opal_output_verbose(2, orte_pmix_server_globals.output,
+                        "%s job control request from %s",
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                        ORTE_NAME_PRINT(requestor));
+
+    OPAL_LIST_FOREACH(val, info, opal_value_t) {
+        if (NULL == val->key) {
+            ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
+            continue;
+        }
+
+        if (0 == strcmp(val->key, OPAL_PMIX_JOB_CTRL_KILL)) {
+            /* convert the list of targets to a pointer array */
+            if (NULL == targets) {
+                ptrarray = NULL;
+            } else {
+                OBJ_CONSTRUCT(&parray, opal_pointer_array_t);
+                OPAL_LIST_FOREACH(nm, targets, opal_namelist_t) {
+                    /* get the proc object for this proc */
+                    if (NULL == (proc = orte_get_proc_object(&nm->name))) {
+                        ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
+                        continue;
+                    }
+                    OBJ_RETAIN(proc);
+                    opal_pointer_array_add(&parray, proc);
+                }
+                ptrarray = &parray;
+            }
+            if (ORTE_SUCCESS != (rc = orte_plm.terminate_procs(ptrarray))) {
+                ORTE_ERROR_LOG(rc);
+            }
+            if (NULL != ptrarray) {
+                /* cleanup the array */
+                for (n=0; n < parray.size; n++) {
+                    if (NULL != (proc = (orte_proc_t*)opal_pointer_array_get_item(&parray, n))) {
+                        OBJ_RELEASE(proc);
+                    }
+                }
+                OBJ_DESTRUCT(&parray);
+            }
+            continue;
+        }
+    }
+
+    return ORTE_SUCCESS;
+}

orte/orted/pmix/pmix_server_internal.h

@@ -12,7 +12,7 @@
  * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2010-2011 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
@@ -206,6 +206,18 @@ extern void pmix_server_log_fn(opal_process_name_t *requestor,
                                opal_pmix_op_cbfunc_t cbfunc,
                                void *cbdata);
 
+extern int pmix_server_alloc_fn(const opal_process_name_t *requestor,
+                                opal_pmix_alloc_directive_t dir,
+                                opal_list_t *info,
+                                opal_pmix_info_cbfunc_t cbfunc,
+                                void *cbdata);
+
+extern int pmix_server_job_ctrl_fn(const opal_process_name_t *requestor,
+                                   opal_list_t *targets,
+                                   opal_list_t *info,
+                                   opal_pmix_info_cbfunc_t cbfunc,
+                                   void *cbdata);
+
 /* declare the RML recv functions for responses */
 extern void pmix_server_launch_resp(int status, orte_process_name_t* sender,
                                     opal_buffer_t *buffer,

orte/util/nidmap.c

@@ -705,7 +705,7 @@ int orte_util_decode_daemon_nodemap(opal_buffer_t *buffer)
     char *ndnames, *rmndr, **tmp;
     opal_list_t dids, slts, flgs;;
     opal_buffer_t *bptr=NULL;
-    orte_topology_t *t;
+    orte_topology_t *t2;
     orte_regex_range_t *rng, *drng, *srng, *frng;
     uint8_t ui8;
 
@@ -978,14 +978,13 @@ int orte_util_decode_daemon_nodemap(opal_buffer_t *buffer)
 
     /* if no topology info was passed, then everyone shares our topology */
     if (NULL == bptr) {
-        orte_topology_t *t;
         /* our topology is first in the array */
-        t = (orte_topology_t*)opal_pointer_array_get_item(orte_node_topologies, 0);
+        t2 = (orte_topology_t*)opal_pointer_array_get_item(orte_node_topologies, 0);
         for (n=0; n < orte_node_pool->size; n++) {
             if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, n))) {
                 if (NULL == node->topology) {
-                    OBJ_RETAIN(t);
-                    node->topology = t;
+                    OBJ_RETAIN(t2);
+                    node->topology = t2;
                 }
             }
         }
@@ -1004,6 +1003,13 @@ int orte_util_decode_daemon_nodemap(opal_buffer_t *buffer)
                 OBJ_RELEASE(bptr);
                 goto cleanup;
             }
+            if (NULL == sig) {
+                rc = ORTE_ERR_BAD_PARAM;
+                ORTE_ERROR_LOG(rc);
+                opal_argv_free(tmp);
+                OBJ_RELEASE(bptr);
+                goto cleanup;
+            }
             n = 1;
             if (ORTE_SUCCESS != (rc = opal_dss.unpack(bptr, &topo, &n, OPAL_HWLOC_TOPO))) {
                 ORTE_ERROR_LOG(rc);
@@ -1013,11 +1019,12 @@ int orte_util_decode_daemon_nodemap(opal_buffer_t *buffer)
                 goto cleanup;
             }
             /* see if we already have this topology - could be an update */
+            t2 = NULL;
             for (n=0; n < orte_node_topologies->size; n++) {
-                if (NULL == (t = (orte_topology_t*)opal_pointer_array_get_item(orte_node_topologies, n))) {
+                if (NULL == (t2 = (orte_topology_t*)opal_pointer_array_get_item(orte_node_topologies, n))) {
                     continue;
                 }
-                if (0 == strcmp(t->sig, sig)) {
+                if (0 == strcmp(t2->sig, sig)) {
                     /* found a match */
                     free(sig);
                     opal_hwloc_base_free_topology(topo);
@@ -1025,11 +1032,12 @@ int orte_util_decode_daemon_nodemap(opal_buffer_t *buffer)
                     break;
                 }
             }
-            if (NULL != sig) {
+            if (NULL != sig || NULL == t2) {
                 /* new topology - record it */
-                t = OBJ_NEW(orte_topology_t);
-                t->sig = sig;
-                t->topo = topo;
+                t2 = OBJ_NEW(orte_topology_t);
+                t2->sig = sig;
+                t2->topo = topo;
+                opal_pointer_array_add(orte_node_topologies, t2);
             }
             /* point each of the nodes in the regex to this topology */
             start = strtoul(tmp[nn], &rmndr, 10);
@@ -1043,8 +1051,8 @@ int orte_util_decode_daemon_nodemap(opal_buffer_t *buffer)
             for (k=start; k <= endpt; k++) {
                 if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, k))) {
                     if (NULL == node->topology) {
-                        OBJ_RETAIN(t);
-                        node->topology = t;
+                        OBJ_RETAIN(t2);
+                        node->topology = t2;
                     }
                 }
             }