framework for scheduling service - subscribes w/ registry to track

the list of available daemons - actual scheduling logic needs to be
filled in.

This commit was SVN r2602.

src/mca/svc/sched/Makefile.am

@@ -8,4 +8,6 @@ noinst_LTLIBRARIES = libmca_svc_sched.la
 libmca_svc_sched_la_SOURCES = \
 	svc_sched.c \
     svc_sched.h \
-	svc_sched_component.c
+	svc_sched_component.c \
+    svc_sched_node.c \
+    svc_sched_node.h

src/mca/svc/sched/svc_sched.c

@@ -1,7 +1,11 @@
 #include "ompi_config.h"
+#include "util/output.h"
 #include "mca/oob/oob.h"
 #include "mca/oob/base/base.h"
+#include "mca/gpr/gpr.h"
+#include "mca/gpr/base/base.h"
 #include "svc_sched.h"
+#include "svc_sched_node.h"
 
 
 mca_svc_base_module_t mca_svc_sched_module = {
@@ -21,6 +25,114 @@ static void mca_svc_sched_recv(
     int tag,
     void* cbdata)
 {
+    /* unpack the request */
+    ompi_buffer_t response;
+    int32_t jobid;
+    int32_t num_nodes;
+    int32_t num_procs;
+    mca_svc_sched_node_t* node;
+    int rc;
+   
+    ompi_unpack(buffer, &jobid, 1, OMPI_INT32);
+    ompi_unpack(buffer, &num_nodes, 1, OMPI_INT32);
+    ompi_unpack(buffer, &num_procs, 1, OMPI_INT32);
+ 
+    ompi_buffer_init(&response, 128);
+
+    /* iterate through the available daemons and attempt to allocate processes
+     * in a round-robin fashion - building the response to the requestor.
+    */   
+    OMPI_THREAD_LOCK(&mca_svc_sched_component.sched_lock);
+
+    /* start with the last used node */
+    node = mca_svc_sched_component.sched_node_next;
+    if (node == (mca_svc_sched_node_t*)ompi_list_get_end(&mca_svc_sched_component.sched_node_list))
+        node = (mca_svc_sched_node_t*)ompi_list_get_first(&mca_svc_sched_component.sched_node_list);
+
+
+    OMPI_THREAD_UNLOCK(&mca_svc_sched_component.sched_lock);
+   
+    /* send the response back to the peer */
+    rc = mca_oob_send_packed(peer, response, tag, 0);
+    if(rc < 0) {
+        ompi_output(0, "mca_svc_sched_recv: mca_oob_send_packed failed with status=%d\n", rc);
+    }
+    ompi_buffer_free(response);
+}
+
+
+/**
+ *  Registry callback on change to list of registered daemons.
+ */
+
+static void mca_svc_sched_registry_callback(
+    ompi_registry_notify_message_t* msg,
+    void* cbdata)
+{
+    ompi_list_item_t* item;
+    if(mca_svc_sched_component.sched_debug > 1) {
+        ompi_output(0, "[%d,%d,%d] mca_svc_sched_registry_callback\n",
+            OMPI_NAME_ARGS(mca_oob_name_self));
+    }
+    OMPI_THREAD_LOCK(&mca_svc_sched_component.sched_lock);
+    for(item =  ompi_list_get_first(&msg->data);
+        item != ompi_list_get_end(&msg->data);
+        item =  ompi_list_get_next(item)) {
+
+        ompi_registry_value_t* value = (ompi_registry_value_t*)item;
+        mca_svc_sched_node_t* node;
+        char* hostname = NULL;
+        ompi_process_name_t proc_name;
+        char* contact_info = NULL;
+        int32_t proc_slots;
+
+        /* unpack the daemon information */
+        ompi_buffer_t buffer;
+        ompi_buffer_init_preallocated(&buffer, value->object, value->object_size);
+        ompi_unpack_string(buffer, &hostname);
+        ompi_unpack(buffer, &proc_name, 1, OMPI_NAME);
+        ompi_unpack_string(buffer, &contact_info);
+        ompi_unpack(buffer, &proc_slots, 1, OMPI_INT32);
+        ompi_buffer_free(buffer);
+
+        /* lookup the corresponding node */
+        node = (mca_svc_sched_node_t*)ompi_rb_tree_find(
+            &mca_svc_sched_component.sched_node_tree,
+            &proc_name);
+
+        /* do the right thing based on the trigger type */
+        switch(msg->trig_action) {
+            case OMPI_REGISTRY_NOTIFY_MODIFICATION:
+            case OMPI_REGISTRY_NOTIFY_ADD_ENTRY:
+                /* create or modify the corresponding daemon entry */
+                if(node == NULL) {
+                    node = OBJ_NEW(mca_svc_sched_node_t);
+                    node->node_name = proc_name;
+                    ompi_rb_tree_insert(&mca_svc_sched_component.sched_node_tree, &node->node_name, node);
+                    ompi_list_append(&mca_svc_sched_component.sched_node_list, &node->node_item);
+                } 
+                mca_svc_sched_node_set(node,hostname,contact_info,proc_slots);
+                break;
+            case OMPI_REGISTRY_NOTIFY_DELETE_ENTRY:
+                /* delete the corresponding deamon entry */
+                if(node != NULL) {
+                    ompi_list_item_t* next;
+                    ompi_rb_tree_delete(&mca_svc_sched_component.sched_node_tree, &proc_name);
+                    next = ompi_list_remove_item(&mca_svc_sched_component.sched_node_list, &node->node_item);
+                    OBJ_RELEASE(node);
+                    if(mca_svc_sched_component.sched_node_next == node)
+                        mca_svc_sched_component.sched_node_next = (mca_svc_sched_node_t*)next;
+                }
+                break;
+        }
+
+        /* cleanup */
+        if(hostname != NULL)
+            free(hostname);
+        if(contact_info != NULL)
+            free(contact_info);
+    }
+    OMPI_THREAD_UNLOCK(&mca_svc_sched_component.sched_lock);
 }
 
 
@@ -30,12 +142,43 @@ static void mca_svc_sched_recv(
 
 int mca_svc_sched_module_init(mca_svc_base_module_t* module)
 {
-    return mca_oob_recv_packed_nb(
+    /* register */
+    int rc;
+    if(mca_svc_sched_component.sched_debug > 0) {
+        ompi_output(0, "[%d,%d,%d] mca_svc_sched_module_init: calling ompi_registry.subscribe(\"vm\")");
+    }
+    /* initialize next node for scheduling */
+    mca_svc_sched_component.sched_node_next = 
+        (mca_svc_sched_node_t*)ompi_list_get_end(&mca_svc_sched_component.sched_node_list);
+
+    rc = ompi_registry.subscribe(
+        OMPI_REGISTRY_NONE,
+        OMPI_REGISTRY_NOTIFY_MODIFICATION|
+        OMPI_REGISTRY_NOTIFY_ADD_ENTRY|
+        OMPI_REGISTRY_NOTIFY_DELETE_ENTRY|
+        OMPI_REGISTRY_NOTIFY_PRE_EXISTING,
+        "vm", /* segment */
+        NULL, /* keys */
+        mca_svc_sched_registry_callback,
+        NULL);
+    if(rc != OMPI_SUCCESS) {
+        ompi_output(0, "[%d,%d,%d] mca_svc_sched_module_init: ompi_registry.subscribe failed, error=%d\n", 
+            OMPI_NAME_ARGS(mca_oob_name_self), rc);
+        return rc;
+    }
+
+    rc = mca_oob_recv_packed_nb(
         MCA_OOB_NAME_ANY,  
         MCA_OOB_TAG_SCHED, 
         MCA_OOB_ALLOC, 
         mca_svc_sched_recv,
         NULL);
+    if(rc != OMPI_SUCCESS) {
+        ompi_output(0, "[%d,%d,%d] mca_svc_sched_module_init: mca_oob_recv_packed_nb failed, error=%d\n", 
+            OMPI_NAME_ARGS(mca_oob_name_self), rc);
+        return rc;
+    }
+    return OMPI_SUCCESS;
 }
 
 

src/mca/svc/sched/svc_sched.h

@@ -8,7 +8,10 @@
 #define _MCA_SVC_SCHED_
 
 #include "mca/svc/svc.h"
+#include "class/ompi_list.h"
+#include "class/ompi_rb_tree.h"
 
+#define OMPI_NAME_ARGS(n)  (n).cellid,(n).jobid,(n).vpid
 
 /**
  * Component open/close/init
@@ -23,8 +26,19 @@ mca_svc_base_module_t* mca_svc_sched_component_init(void);
 int mca_svc_sched_module_init(mca_svc_base_module_t*);
 int mca_svc_sched_module_fini(mca_svc_base_module_t*);
 
+struct mca_svc_sched_component_t {
+    mca_svc_base_component_t sched_base;
+    int sched_debug;                               /* debug flag */
+    ompi_list_t sched_node_list;                   /* list of nodes for round-robin scheduling */
+    ompi_rb_tree_t sched_node_tree;                /* tree of nodes for quick lookup by name */
+    struct mca_svc_sched_node_t* sched_node_next;  /* iterator for round-robin scheduling */
+    ompi_mutex_t sched_lock;                       /* lock to protected global variables */
+};
+typedef struct mca_svc_sched_component_t mca_svc_sched_component_t;
+
 extern mca_svc_base_module_t mca_svc_sched_module;
-extern mca_svc_base_component_t mca_svc_sched_component;
+extern mca_svc_sched_component_t mca_svc_sched_component;
+
 
 #endif
 

src/mca/svc/sched/svc_sched_component.c

@@ -2,7 +2,8 @@
 #include "util/proc_info.h"
 
 
-mca_svc_base_component_t mca_svc_sched_component = {
+mca_svc_sched_component_t mca_svc_sched_component = {
+    {
       /* First, the mca_base_module_t struct containing meta
          information about the module itself */
       {
@@ -28,26 +29,75 @@ mca_svc_base_component_t mca_svc_sched_component = {
       },
                                                                                                                   
       mca_svc_sched_component_init
+    },
 };
 
 
 /**
- *
+ * Utility function to register parameters
+ */
+
+static inline int mca_svc_sched_param_register_int(
+    const char* param_name,
+    int default_value)
+{
+    int id = mca_base_param_register_int("svc","sched",param_name,NULL,default_value);
+    int param_value = default_value;
+    mca_base_param_lookup_int(id,&param_value);
+    return param_value;
+}
+
+/**
+ * Sched component open - Initialize global data and register
+ * any MCA parameters.
  */
 
 int mca_svc_sched_component_open(void)
 {
+    OBJ_CONSTRUCT(&mca_svc_sched_component.sched_lock, ompi_mutex_t);
+    OBJ_CONSTRUCT(&mca_svc_sched_component.sched_node_list, ompi_list_t);
+    OBJ_CONSTRUCT(&mca_svc_sched_component.sched_node_tree, ompi_rb_tree_t);
+
+    mca_svc_sched_component.sched_debug =
+        mca_svc_sched_param_register_int("debug", 0);
     return OMPI_SUCCESS;
 }
 
+
 /**
- *
+ * compare function for tree insert
+ */
+
+static int mca_svc_sched_name_compare(const ompi_process_name_t* n1, const ompi_process_name_t* n2)
+{
+   if(n1->cellid < n2->cellid)
+       return -1;
+   else if(n1->cellid > n2->cellid)
+       return 1;
+   else if(n1->jobid < n2->jobid)
+       return -1;
+   else if(n1->jobid > n2->jobid)
+       return 1;
+   else if(n1->vpid < n2->vpid)
+       return -1;
+   else if(n1->vpid > n2->vpid)
+       return 1;
+   return(0);
+}
+                                                                                                                                               
+/**
+ * Sched component initialization.
  */
 
 mca_svc_base_module_t* mca_svc_sched_component_init(void)
 {
+    /* only run in the seed daemon */
     if(ompi_process_info.seed == false)
         return NULL;
+
+    /* initialize data structures */
+    ompi_rb_tree_init(&mca_svc_sched_component.sched_node_tree, 
+        (ompi_rb_tree_comp_fn_t)mca_svc_sched_name_compare);
     return &mca_svc_sched_module;
 }
 
@@ -58,6 +108,9 @@ mca_svc_base_module_t* mca_svc_sched_component_init(void)
 
 int mca_svc_sched_component_close(void)
 {
+    OBJ_DESTRUCT(&mca_svc_sched_component.sched_node_list);
+    OBJ_DESTRUCT(&mca_svc_sched_component.sched_node_tree);
+    OBJ_DESTRUCT(&mca_svc_sched_component.sched_lock);
     return OMPI_SUCCESS;
 }
 

src/mca/svc/sched/svc_sched_node.c

@@ -0,0 +1,44 @@
+#include "svc_sched_node.h"
+
+
+static void mca_svc_sched_node_construct(mca_svc_sched_node_t* node)
+{
+    node->node_hostname = NULL;
+    node->node_contactinfo = NULL;
+    node->node_proc_slots = 0;
+    node->node_proc_avail = 0;
+}
+
+static void mca_svc_sched_node_destruct(mca_svc_sched_node_t* node)
+{
+    if(node->node_hostname != NULL)
+        free(node->node_hostname);
+    if(node->node_contactinfo != NULL)
+        free(node->node_contactinfo);
+}
+
+OBJ_CLASS_INSTANCE(
+    mca_svc_sched_node_t,
+    ompi_list_item_t,
+    mca_svc_sched_node_construct,
+    mca_svc_sched_node_destruct);
+
+/**
+ * Setup node attributes
+ */
+                                                                                                                    
+void mca_svc_sched_node_set(
+    mca_svc_sched_node_t* node,
+    const char* hostname,
+    const char* contactinfo,
+    int32_t proc_slots)
+{
+    if(node->node_hostname != NULL)
+        free(node->node_hostname);
+    node->node_hostname = strdup(hostname);
+    if(node->node_contactinfo != NULL)
+        free(node->node_contactinfo);
+    node->node_contactinfo = strdup(contactinfo);
+    node->node_proc_slots = proc_slots;
+}
+

src/mca/svc/sched/svc_sched_node.h

@@ -0,0 +1,35 @@
+/*
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef _MCA_SVC_SCHED_NODE_
+#define _MCA_SVC_SCHED_NODE_
+
+#include "mca/svc/svc.h"
+
+struct mca_svc_sched_node_t {
+    ompi_list_item_t node_item;
+    ompi_process_name_t node_name;
+    char* node_hostname;
+    char* node_contactinfo;
+    int32_t node_proc_slots;
+    int32_t node_proc_avail;
+};
+typedef struct mca_svc_sched_node_t mca_svc_sched_node_t;
+
+OBJ_CLASS_DECLARATION(mca_svc_sched_node_t);
+
+/**
+ * Setup node attributes
+ */
+
+void mca_svc_sched_node_set(
+    mca_svc_sched_node_t* node, 
+    const char* hostname, 
+    const char* contactinfo,
+    int32_t proc_slots);
+
+#endif
+