/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "orte_config.h" #include "orte/constants.h" #include "orte/types.h" #include #ifdef HAVE_UNISTD_H #include #endif /* HAVE_UNISTD_H */ #ifdef HAVE_STRING_H #include #endif /* HAVE_STRING_H */ #include "opal/mca/base/mca_base_param.h" #include "opal/util/opal_sos.h" #include "orte/util/show_help.h" #include "orte/mca/errmgr/errmgr.h" #include "orte/util/error_strings.h" #include "orte/mca/rmaps/base/rmaps_private.h" #include "orte/mca/rmaps/base/base.h" #include "rmaps_rr.h" static orte_node_t* get_starting_point(opal_list_t *node_list, orte_job_t *jdata); /* * Create a round-robin mapping for the job. */ static int orte_rmaps_rr_map(orte_job_t *jdata) { orte_app_context_t *app; int i; opal_list_t node_list; opal_list_item_t *item; orte_std_cntr_t num_nodes, num_slots; int rc; mca_base_component_t *c = &mca_rmaps_round_robin_component.base_version; /* this mapper can only handle initial launch * when rr mapping is desired - allow * restarting of failed apps */ if (ORTE_JOB_STATE_INIT != jdata->state) { opal_output_verbose(5, orte_rmaps_base.rmaps_output, "mca:rmaps:rr: job %s in state %s - rr cannot map", ORTE_JOBID_PRINT(jdata->jobid), orte_job_state_to_str(jdata->state)); return ORTE_ERR_TAKE_NEXT_OPTION; } if (NULL != jdata->map->req_mapper && 0 != strcasecmp(jdata->map->req_mapper, c->mca_component_name)) { /* a mapper has been specified, and it isn't me */ opal_output_verbose(5, orte_rmaps_base.rmaps_output, "mca:rmaps:rr: job %s not using rr mapper", ORTE_JOBID_PRINT(jdata->jobid)); return ORTE_ERR_TAKE_NEXT_OPTION; } if (ORTE_MAPPING_RR < ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { /* I don't know how to do these - defer */ opal_output_verbose(5, orte_rmaps_base.rmaps_output, "mca:rmaps:rr: job %s not using rr mapper", ORTE_JOBID_PRINT(jdata->jobid)); return ORTE_ERR_TAKE_NEXT_OPTION; } opal_output_verbose(5, orte_rmaps_base.rmaps_output, "mca:rmaps:rr: mapping job %s", ORTE_JOBID_PRINT(jdata->jobid)); /* flag that I did the mapping */ if (NULL != jdata->map->last_mapper) { free(jdata->map->last_mapper); } jdata->map->last_mapper = strdup(c->mca_component_name); /* start at the beginning... */ jdata->num_procs = 0; /* cycle through the app_contexts, mapping them sequentially */ for(i=0; i < jdata->apps->size; i++) { if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) { continue; } /* setup the nodelist here in case we jump to error */ OBJ_CONSTRUCT(&node_list, opal_list_t); /* if the number of processes wasn't specified, then we know there can be only * one app_context allowed in the launch, and that we are to launch it across * all available slots. We'll double-check the single app_context rule first */ if (0 == app->num_procs && 1 < jdata->num_apps) { orte_show_help("help-orte-rmaps-rr.txt", "orte-rmaps-rr:multi-apps-and-zero-np", true, jdata->num_apps, NULL); rc = ORTE_ERR_SILENT; goto error; } /* for each app_context, we have to get the list of nodes that it can * use since that can now be modified with a hostfile and/or -host * option */ if(ORTE_SUCCESS != (rc = orte_rmaps_base_get_target_nodes(&node_list, &num_slots, app, jdata->map->mapping))) { ORTE_ERROR_LOG(rc); goto error; } num_nodes = (orte_std_cntr_t)opal_list_get_size(&node_list); /* if a bookmark exists from some prior mapping, set us to start there */ jdata->bookmark = get_starting_point(&node_list, jdata); if (0 == app->num_procs) { /* set the num_procs to equal the number of slots on these mapped nodes */ app->num_procs = num_slots; } /* Make assignments */ if (ORTE_MAPPING_BYNODE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_bynode(jdata, app, &node_list, num_slots, app->num_procs); } else if (ORTE_MAPPING_BYSLOT == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots, app->num_procs); #if OPAL_HAVE_HWLOC } else if (ORTE_MAPPING_BYHWTHREAD == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_PU, 0); } else if (ORTE_MAPPING_BYCORE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CORE, 0); } else if (ORTE_MAPPING_BYL1CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CACHE, 1); } else if (ORTE_MAPPING_BYL2CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CACHE, 2); } else if (ORTE_MAPPING_BYL3CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CACHE, 3); } else if (ORTE_MAPPING_BYSOCKET == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_SOCKET, 0); } else if (ORTE_MAPPING_BYNUMA == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_NODE, 0); #endif } else { /* unrecognized mapping directive */ orte_show_help("help-orte-rmaps-base.txt", "unrecognized-policy", true, "mapping", orte_rmaps_base_print_mapping(jdata->map->mapping)); rc = ORTE_ERR_SILENT; goto error; } if (ORTE_SUCCESS != rc) { ORTE_ERROR_LOG(rc); goto error; } /* track the total number of processes we mapped */ jdata->num_procs += app->num_procs; /* cleanup the node list - it can differ from one app_context * to another, so we have to get it every time */ while (NULL != (item = opal_list_remove_first(&node_list))) { OBJ_RELEASE(item); } OBJ_DESTRUCT(&node_list); /* compute vpids and add proc objects to the job - do this after * each app_context so that the ranks within each context are * contiguous */ if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_vpids(jdata))) { ORTE_ERROR_LOG(rc); return rc; } } return ORTE_SUCCESS; error: while(NULL != (item = opal_list_remove_first(&node_list))) { OBJ_RELEASE(item); } OBJ_DESTRUCT(&node_list); return rc; } /* * determine the proper starting point for the next mapping operation */ static orte_node_t* get_starting_point(opal_list_t *node_list, orte_job_t *jdata) { opal_list_item_t *item, *cur_node_item; orte_node_t *node, *nd1, *ndmin; int overload; /* if a bookmark exists from some prior mapping, set us to start there */ if (NULL != jdata->bookmark) { cur_node_item = NULL; /* find this node on the list */ for (item = opal_list_get_first(node_list); item != opal_list_get_end(node_list); item = opal_list_get_next(item)) { node = (orte_node_t*)item; if (node->index == jdata->bookmark->index) { cur_node_item = item; break; } } /* see if we found it - if not, just start at the beginning */ if (NULL == cur_node_item) { cur_node_item = opal_list_get_first(node_list); } } else { /* if no bookmark, then just start at the beginning of the list */ cur_node_item = opal_list_get_first(node_list); } /* is this node fully subscribed? If so, then the first * proc we assign will oversubscribe it, so let's look * for another candidate */ node = (orte_node_t*)cur_node_item; ndmin = node; overload = ndmin->slots_inuse - ndmin->slots_alloc; if (node->slots_inuse >= node->slots_alloc) { /* work down the list - is there another node that * would not be oversubscribed? */ if (cur_node_item != opal_list_get_last(node_list)) { item = opal_list_get_next(cur_node_item); } else { item = opal_list_get_first(node_list); } while (item != cur_node_item) { nd1 = (orte_node_t*)item; if (nd1->slots_inuse < nd1->slots_alloc) { /* this node is not oversubscribed! use it! */ return (orte_node_t*)item; } /* this one was also oversubscribed, keep track of the * node that has the least usage - if we can't * find anyone who isn't fully utilized, we will * start with the least used node */ if (overload >= (nd1->slots_inuse - nd1->slots_alloc)) { ndmin = nd1; overload = ndmin->slots_inuse - ndmin->slots_alloc; } if (item == opal_list_get_last(node_list)) { item = opal_list_get_first(node_list); } else { item= opal_list_get_next(item); } } /* if we get here, then we cycled all the way around the * list without finding a better answer - just use the node * that is minimally overloaded */ cur_node_item = (opal_list_item_t*)ndmin; } return (orte_node_t*)cur_node_item; } orte_rmaps_base_module_t orte_rmaps_round_robin_module = { orte_rmaps_rr_map };